Commit 577a4c87 authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'master' into master+ZODB4-wc2

* master: (112 commits)
  ninja: make sure it's built using SlapOS Python
  proxysql: stop using old CMake
  version up: proxysql 2.3.2
  proxysql: bundled libinjection needs Python 2
  proxysql: small clean up
  proxysql: clean up extends
  libiconv, libmicrohttpd, proxysql: make parts shared
  java-jdk: simplify a little
  fixup! component/nxdtest: v↑  (namespaces, /tmp-on-tmpfs, cancellation)
  erp5 and erp5testnode: add software-type to software.cfg.json so that it works in panel GUI
  Revert "component/libxml2: Version up"
  Revert "component/meson: Version up"
  software/theia: Fix EPR5 resiliency test
  component/libxml2: Version up
  component/meson: Version up
  component/m4: Version up
  erp5: Change default timerserver-interval from 5s to 1s.
  slapos/obs.cfg: supports the newer version of cmmi and add some comments
  pyprog: Wrap entry-point call with sys.exit()
  Generalize how nxdtest python script is generated into pyprog recipe macro
  ...
parents c398531e ad04d19a
#!/bin/sh
. "$(dirname "$0")/_/husky.sh"
npx lint-staged
......@@ -109,7 +109,6 @@ template = inline:
[versions]
mock = 3.0.5
random2 = 1.0.1
zope.testing = 4.7
zope.testrunner = 5.2
......
......@@ -125,4 +125,3 @@ environment =
recipe = slapos.recipe.build:download
shared = true
url = ${:_profile_base_location_}/${:filename}
mode = 640
[buildout]
parts =
bash
extends =
../gettext/buildout.cfg
../ncurses/buildout.cfg
../patch/buildout.cfg
parts =
bash
[bash]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/bash/bash-4.3.tar.gz
md5sum = 81348932d5da294953e15d4814c74dd1
url-prefix = https://ftp.gnu.org/pub/gnu/bash/bash-5.1
url = ${:url-prefix}.tar.gz
md5sum = bb91a17fd6c9032c26d0b2b78b50aff5
patch-binary = ${patch:location}/bin/patch
patch-prefix = ${:url-prefix}-patches/bash51
patches =
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-001#1ab682b4e36afa4cf1b426aa7ac81c0d
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-002#8fc22cf50ec85da00f6af3d66f7ddc1b
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-003#a41728eca78858758e26b5dea64ae506
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-004#bf8d53d227829d67235927689a03cc7a
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-005#c0c00935c8b8ffff76e8ab77e7be7d15
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-006#6f01e364cd092faa28dd7119f47ddb5f
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-007#dcf471d222bcd83283d3094e6ceeb6f8
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-008#f7553416646dc26c266454c78a916d36
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-009#7e73d2151f4064b484a4ba2c4b09960e
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-010#a275463d21735bb6d7161f9fbd320d8f
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-011#c17103ee20420d77e46b224c8d3fceda
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-012#3e2a057a19d02b3f92a3a09eacbc03ae
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-013#fb377143a996d4ff087a2771bc8332f9
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-014#1a1aaecc99a9d0cbc310e8e247dcc8b6
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-015#4f04387458a3c1b4d460d199f49991a8
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-016#90e759709720c4f877525bebc9d5dc06
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-017#11e4046e1b86070f6adbb7ffc89641be
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-018#cd5a9b46f5bea0dc0248c93c7dfac011
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-019#cff4dc024d9d3456888aaaf8a36ca774
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-020#167839c5f147347f4a03d88ab97ff787
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-021#1d350671c48dec30b34d8b81f09cd79d
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-022#11c349af66a55481a3215ef2520bec36
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-023#b3cb0d80fd0c47728264405cbb3b23c7
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-024#b5ea5600942acceb4b6f07313d2de74e
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-025#193c06f578d38ffdbaebae9c51a7551f
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-026#922578e2be7ed03729454e92ee8d3f3a
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-027#8ff6948b16f2db5c29b1b9ae1085bbe7
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-028#dd51fa67913b5dca45a702b672b3323f
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-029#0729364c977ef4271e9f8dfafadacf67
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-030#efb709fdb1368945513de23ccbfae053
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-031#236df1ac1130a033ed0dbe2d2115f28f
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-032#2360f7e79cfb28526f80021025ea5909
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-033#b551c4ee7b8713759e4143499d0bbd48
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-034#c9a56fbe0348e05a886dff97f2872b74
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-035#e564e8ab44ed1ca3a4e315a9f6cabdc9
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-036#b00ff66c41a7c0f06e191200981980b0
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-037#be2a7b05f6ae560313f3c9d5f7127bda
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-038#61e0522830b24fbe8c0d1b010f132470
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-039#a4775487abe958536751c8ce53cdf6f9
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-040#80d3587c58854e226055ef099ffeb535
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-041#20bf63eef7cb441c0b1cc49ef3191d03
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-042#70790646ae61e207c995e44931390e50
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-043#855a46955cb251534e80b4732b748e37
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-044#29623d3282fcbb37e1158136509b5bb8
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-045#4473244ca5abfd4b018ea26dc73e7412
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-046#7e5fb09991c077076b86e0e057798913
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-047#8483153bad1a6f52cadc3bd9a8df7835
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-048#e9f5dc12a32b2e0d3961344e794f92b3
${:patch-prefix}-001#57641ddbf92fca25df92a443e36f285a
${:patch-prefix}-002#aed44842ed1a05fcfc3ef146991fdaef
${:patch-prefix}-003#bf96455600a86420d69f5166575192dd
${:patch-prefix}-004#d2c524dba0eea5dc5f00849cc84376a0
${:patch-prefix}-005#5081278e6c35154e28d09f582251c529
${:patch-prefix}-006#f4a8bcda4b7bd2c72b29c107027608a3
${:patch-prefix}-007#bf7816d63ee0476054bf18a488d8bb1b
${:patch-prefix}-008#7e5a30d864f834953b22a55c01c8690b
${:patch-prefix}-009#8e35f11cbfcefe2c07c64d00601fd713
${:patch-prefix}-010#d78ad19986c0355a8d67c9a0e82ad4aa
${:patch-prefix}-011#2416386b5ee94e499ccbf71f6fd4aebd
${:patch-prefix}-012#879b2d8a03162faebb7234c4cd57c5cd
configure-options =
--with-curses
environment =
CPPFLAGS=-I${ncurses:location}/include
LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
PATH=${patch:location}/bin:%(PATH)s
PATH=${gettext:location}/bin:%(PATH)s
......@@ -9,16 +9,14 @@ parts =
do-not-strip-path = ${buildout:parts-directory}/bazel/bin/bazel
[zulu]
recipe = hexagonit.recipe.download
ignore-existing = true
recipe = slapos.recipe.build:download-unpacked
shared = true
url = http://cdn.azul.com/zulu/bin/zulu8.20.0.5-jdk8.0.121-linux_x64.tar.gz
md5sum = e5f4b1d997e50ffe4998c68c8ec45403
strip-top-level-dir = true
[template-bazel-crosstool]
recipe = slapos.recipe.template:jinja2
location = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 640
filename = bazel_tools_cpp_CROSSTOOL
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${:location}/${:filename}
......@@ -29,7 +27,6 @@ context =
[template-bazel-src-main-cpp-build]
recipe = slapos.recipe.template:jinja2
location = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 640
filename = bazel_src_main_cpp_BUILD
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${:location}/${:filename}
......
[buildout]
extends =
../patch/buildout.cfg
parts = boa
[boa-patch-ENOSYS]
recipe = hexagonit.recipe.download
ignore-existing = true
url = http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/www-servers/boa/files/boa-0.94.14_rc21-ENOSYS.patch?revision=1.1
filename = boa-0.94.14_rc21-ENOSYS.patch
md5sum = 7206b342195961501ed1eae38486e5db
download-only = true
[boa]
recipe = slapos.recipe.build
url = http://www.boa.org/boa-0.94.14rc21.tar.gz
md5sum = e24b570bd767a124fcfb40a34d148ba9
patches =
${boa-patch-ENOSYS:location}/${boa-patch-ENOSYS:filename}
install =
import shutil
import os
url = self.download(self.options['url'], self.options['md5sum'])
extract_dir = self.extract(url)
workdir = guessworkdir(extract_dir)
self.applyPatchList(self.options.get('patches'), '-p1', cwd=workdir)
call(['./configure'], cwd=workdir)
call(['make'], cwd=workdir)
# Installation of boa. Manually, no make install
bindir = os.path.join(location, 'bin')
os.makedirs(bindir)
for name in 'boa', 'boa_indexer':
path = os.path.join(bindir, name)
shutil.copyfile(os.path.join(workdir, 'src', name), path)
os.chmod(path, 0755)
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -71,7 +71,7 @@ version =
md5sum-x86_64 =
recipe = slapos.recipe.build
x86-64 = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
x86_64-linux-gnu = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
library =
${nss:location}/lib
${nspr:location}/lib
......@@ -79,6 +79,6 @@ library =
path =
install =
import os, shutil
extract_dir = self.extract(self.download(*options[guessPlatform()].split()))
extract_dir = self.extract(self.download(*options[multiarch()].split()))
os.mkdir(location)
shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
......@@ -116,7 +116,7 @@ md5sum-x86_64 =
recipe = slapos.recipe.build
x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
x86_64-linux-gnu = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
library =
......@@ -167,7 +167,7 @@ library =
path =
${fontconfig:location}/bin
install =
url, md5sum = self.options[guessPlatform()].split()
url, md5sum = self.options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
# XXX adjust some permissions
......
......@@ -14,8 +14,3 @@ md5sum = c0feb5855604f68b09bdb3acb623619e
environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
[cmake-3.18]
<= cmake
url = https://cmake.org/files/v3.18/cmake-3.18.4.tar.gz
md5sum = 0380beaee1c39a22455db02651abe7be
[buildout]
parts =
consul
[consul]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
url_x86-64 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_amd64.zip
url_x86 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_386.zip
# supported architectures md5sums
md5sum_x86 = dfdc0eedd79baab7e6bc56c1582fd02e
md5sum_x86-64 = d6bc0898ea37ae2198370a9e1978d1bb
install =
import shutil
platform = guessPlatform()
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
recipe = slapos.recipe.build:download-unpacked
version = 0.8.3
base = https://releases.hashicorp.com/consul/${:version}/consul_${:version}
i386-linux-gnu = ${:base}_linux_386.zip dfdc0eedd79baab7e6bc56c1582fd02e
x86_64-linux-gnu = ${:base}_linux_amd64.zip d6bc0898ea37ae2198370a9e1978d1bb
......@@ -10,6 +10,7 @@ extends =
../zstd/buildout.cfg
../zlib/buildout.cfg
../nghttp2/buildout.cfg
../ca-certificates/buildout.cfg
parts =
curl
......@@ -38,6 +39,7 @@ configure-options =
--disable-alt-svc
--with-zlib=${zlib:location}
--with-ssl=${openssl:location}
--with-ca-path=${openssl:location}/etc/ssl/certs
--without-gnutls
--without-polarssl
--without-mbedtls
......
......@@ -136,8 +136,8 @@ recipe = slapos.recipe.build
slapos_promise =
file:firefox
x86 = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 ${:i686-md5sum}
x86-64 = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ${:x86_64-md5sum}
i386-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 ${:i686-md5sum}
x86_64-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ${:x86_64-md5sum}
library =
${alsa:location}/lib
......@@ -185,7 +185,7 @@ path =
${fontconfig:location}/bin
install =
url, md5sum = options[guessPlatform()].split()
url, md5sum = options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
${:post-install}
......@@ -243,11 +243,11 @@ x86_64-md5sum = 4a185d3179862a35104603b9274452e7
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:_buildout_section_name_}
x86 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
x86-64 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
i386-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
x86_64-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
install =
import shutil
url, md5sum = options[guessPlatform()].split()
url, md5sum = options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
shutil.copy(extract_dir + '/geckodriver', location)
......@@ -103,10 +103,8 @@ environment =
GI_LIBS=-L${gobject-introspection:location}/lib -lgirepository-1.0
[trusted-config]
recipe = hexagonit.recipe.download
ignore-existing = true
url = https://raw.githubusercontent.com/t-woerner/firewalld/v0.3.14/config/zones/trusted.xml
download-only = true
recipe = slapos.recipe.build:download
url = https://raw.githubusercontent.com/t-woerner/firewalld/v0.3.14/config/zones/${:filename}
filename = trusted.xml
md5sum = 893752ba2e93a1e96334dfee19f884ad
......
......@@ -33,4 +33,3 @@ environment =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 640
......@@ -6,6 +6,7 @@ extends =
../git/buildout.cfg
../pkgconfig/buildout.cfg
../swig/buildout.cfg
../patch/buildout.cfg
./buildout.hash.cfg
parts = gowork go
......@@ -22,10 +23,19 @@ make-targets= cd src && unset GOBIN && ./all.bash && cp -alf .. ${:location}
# some testdata files have an issue with slapos.extension.strip.
post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
environment =
PATH=${swig:location}/bin:%(PATH)s
PATH=${swig:location}/bin:${patch:location}/bin:%(PATH)s
GOROOT_FINAL=${:location}
${:environment-extra}
# TestChown and TestSCMCredentials currently fail in a user-namespace
# https://github.com/golang/go/issues/42525
# the patches apply to go >= 1.12
patch-options = -p1
patches =
${:_profile_base_location_}/skip-chown-tests.patch#d4e3c8ef83788fb2a5d80dd75034786f
${:_profile_base_location_}/fix-TestSCMCredentials.patch#1d8dbc97cd579e03fafd8627d48f1c59
[golang14]
<= golang-common
# https://golang.org/doc/install/source#bootstrapFromSource
......@@ -34,6 +44,9 @@ md5sum = dbf727a4b0e365bf88d97cbfde590016
environment-extra =
make-targets= cd src && unset GOBIN && ./make.bash && cp -alf .. ${:location}
# skip-chown-tests.patch does not apply to go1.4, but we don't run go1.4 tests.
patches =
[golang1.12]
<= golang-common
......@@ -109,7 +122,6 @@ environment-extra =
recipe = slapos.recipe.template:jinja2
exe = ${buildout:bin-directory}/go
rendered= ${:exe}
mode = 755
template= inline:
#!/bin/sh -e
. ${gowork:env.sh}
......
From 385ca858ac89efccffd557eccc1113281306bd88 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Mon, 6 Dec 2021 22:50:27 +0300
Subject: [PATCH] syscall: tests: Fix TestSCMCredentials for `unshare -Umc`
---
src/syscall/creds_test.go | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/syscall/creds_test.go b/src/syscall/creds_test.go
index c1a8b516e8..ed6e80c0c3 100644
--- a/src/syscall/creds_test.go
+++ b/src/syscall/creds_test.go
@@ -78,8 +78,10 @@ func TestSCMCredentials(t *testing.T) {
if sys, ok := err.(*os.SyscallError); ok {
err = sys.Err
}
- if err != syscall.EPERM {
- t.Fatalf("WriteMsgUnix failed with %v, want EPERM", err)
+ // can get EINVAL instead of EPERM under `unshare -Umc` because uid0 is not mapped and maps to -1
+ // see also https://github.com/golang/go/issues/42525
+ if !(err == syscall.EPERM || err == syscall.EINVAL) {
+ t.Fatalf("WriteMsgUnix failed with %v, want EPERM/EINVAL", err)
}
}
--
2.30.2
From: regnat <rg@regnat.ovh>
Date: Wed, 3 Nov 2021 10:17:28 +0100
Subject: [PATCH] Disable the chown tests
See https://github.com/golang/go/issues/42525 and
https://github.com/NixOS/nix/issues/3245
---
os/os_unix_test.go | 3 +++
1 file changed, 3 insertions(+)
diff --git a/os/os_unix_test.go b/os/os_unix_test.go
index 51693fd..0936542 100644
--- a/src/os/os_unix_test.go
+++ b/src/os/os_unix_test.go
@@ -40,6 +40,7 @@ func checkUidGid(t *testing.T, path string, uid, gid int) {
}
func TestChown(t *testing.T) {
+ t.Skipf("https://github.com/golang/go/issues/42525")
// Use TempDir() to make sure we're on a local file system,
// so that the group ids returned by Getgroups will be allowed
// on the file. On NFS, the Getgroups groups are
@@ -83,6 +84,7 @@ func TestChown(t *testing.T) {
}
func TestFileChown(t *testing.T) {
+ t.Skipf("https://github.com/golang/go/issues/42525")
// Use TempDir() to make sure we're on a local file system,
// so that the group ids returned by Getgroups will be allowed
// on the file. On NFS, the Getgroups groups are
@@ -126,6 +128,7 @@ func TestFileChown(t *testing.T) {
}
func TestLchown(t *testing.T) {
+ t.Skipf("https://github.com/golang/go/issues/42525")
// Use TempDir() to make sure we're on a local file system,
// so that the group ids returned by Getgroups will be allowed
// on the file. On NFS, the Getgroups groups are
--
2.31.1
......@@ -34,11 +34,10 @@ install =
[helloweb-go]
# we already have gowork/bin/helloweb with helloweb Go build.
# Add bin/helloweb-go that runs go version of helloweb without any environment preset needed.
recipe = collective.recipe.template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
input = inline:
#!/bin/sh
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:#!/bin/sh -e
. ${gowork:env.sh}
exec helloweb "$@"
......@@ -71,7 +70,7 @@ scripts = helloweb=helloweb-python
# rubygemsrecipe with fixed url and this way pinned rubygems version
recipe = rubygemsrecipe
url = https://rubygems.org/rubygems/rubygems-2.5.2.zip
ruby-location = ${ruby2.1:location}
ruby-location = ${ruby:location}
ruby-executable = ${:ruby-location}/bin/ruby
gems = bundler==1.11.2
......@@ -93,15 +92,14 @@ path = ${helloweb:location}/ruby/
configure-command = :
make-binary =
make-targets= cd ${:path} && ${bundler:bundle} install
make-targets= ${bundler:bundle} install
[helloweb-ruby]
# NOTE slapos.cookbook:wrapper also works, but currently _only_ in instance
recipe = collective.recipe.template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
input = inline:
#!/bin/sh
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:#!/bin/sh -e
export BUNDLE_GEMFILE=${helloweb-ruby-bundle:path}/Gemfile
exec ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_} "$@"
......@@ -26,8 +26,7 @@ configure-command = :
make-binary = :
post-install =
mv * %(location)s
for file in %(location)s/bin/* %(location)s/lib/*.so ; do
echo appending rpath to $file
${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
done
cd %(location)s
set -x
${patchelf:location}/bin/patchelf --set-rpath %(rpath)s bin/* lib/*.so
rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server
From 3859e99f50abe11a8dade28efa9ea3d99dfaac11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petar=20Koreti=C4=87?= <petar.koretic@gmail.com>
Date: Fri, 11 Apr 2014 10:03:40 +0200
Subject: [RFC 1/2] Remove unused variable 'size'
---
json_tokener.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/json_tokener.c b/json_tokener.c
index 19de8ef..9a76293 100644
--- a/json_tokener.c
+++ b/json_tokener.c
@@ -352,12 +352,10 @@ struct json_object* json_tokener_parse_ex(struct json_tokener *tok,
case json_tokener_state_inf: /* aka starts with 'i' */
{
- int size;
int size_inf;
int is_negative = 0;
printbuf_memappend_fast(tok->pb, &c, 1);
- size = json_min(tok->st_pos+1, json_null_str_len);
size_inf = json_min(tok->st_pos+1, json_inf_str_len);
char *infbuf = tok->pb->buf;
if (*infbuf == '-')
--
2.5.0
From 89ae583a8624fad6be4a7d1da084b0c410e4fc63 Mon Sep 17 00:00:00 2001
From: Stuart Walsh <stu@ipng.org.uk>
Date: Tue, 31 Mar 2015 12:23:03 +0100
Subject: [RFC 2/2] Fix uninitialised variable compile warning, and also fix
unused-when-used warning
---
json_object.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/json_object.h b/json_object.h
index 200ac40..e6c6a4f 100644
--- a/json_object.h
+++ b/json_object.h
@@ -339,8 +339,8 @@ extern void json_object_object_del(struct json_object* obj, const char *key);
#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && __STDC_VERSION__ >= 199901L
# define json_object_object_foreach(obj,key,val) \
- char *key; \
- struct json_object *val __attribute__((__unused__)); \
+ char *key = NULL; \
+ struct json_object *val = NULL; \
for(struct lh_entry *entry ## key = json_object_get_object(obj)->head, *entry_next ## key = NULL; \
({ if(entry ## key) { \
key = (char*)entry ## key->k; \
--
2.5.0
[buildout]
extends =
../patch/buildout.cfg
parts =
json-c
[json-c-patch-base]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
download-only = true
[json-c-patch-3859e99f50abe11a8dade28efa9ea3d99dfaac11]
< = json-c-patch-base
filename = 0001-Remove-unused-variable-size.patch
md5sum = e8ebc602fbab128d22f1216cb15c4626
[json-c-patch-89ae583a8624fad6be4a7d1da084b0c410e4fc63]
< = json-c-patch-base
filename = 0002-Fix-uninitialised-variable-compile-warning-and-also-.patch
md5sum = 5525ab9ee78157ce6d6100e374ac7767
[json-c]
recipe = slapos.recipe.cmmi
url = https://s3.amazonaws.com/json-c_releases/releases/json-c-0.12.tar.gz
md5sum = 3ca4bbb881dfc4017e8021b5e0a8c491
patch-options = -p1
patches =
${json-c-patch-3859e99f50abe11a8dade28efa9ea3d99dfaac11:location}/${json-c-patch-3859e99f50abe11a8dade28efa9ea3d99dfaac11:filename}
${json-c-patch-89ae583a8624fad6be4a7d1da084b0c410e4fc63:location}/${json-c-patch-89ae583a8624fad6be4a7d1da084b0c410e4fc63:filename}
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -22,8 +22,6 @@ python_executable = ${buildout:bin-directory}/${:interpreter}
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[jupyter-notebook-config]
<= download-file-base
......@@ -44,7 +42,6 @@ mode = 0644
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
......
......@@ -15,7 +15,7 @@
[instance-jupyter-notebook]
filename = instance.cfg.in
md5sum = 1d5fe6cc4e48672ae7be1c223794a932
md5sum = 357f28614a13cdbf00a29a83cbfd2642
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
......
......@@ -37,10 +37,6 @@ key_file = ${generate-certificate:key_file}
logfile = ${directory:log}/jupyter_notebook.log
notebook_dir = ${directory:notebook_dir}
[dynamic-jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 0644
[generate-certificate]
; TODO: there is a slapos recipe to generate certificates. Use it instead
recipe = plone.recipe.command
......@@ -76,10 +72,9 @@ environment =
LANG=C.UTF-8
[jupyter-notebook-config]
<= dynamic-jinja2-template-base
recipe = slapos.recipe.template:jinja2
template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
mode = 0744
context =
raw config_cfg ${buildout:directory}/knowledge0.cfg
......@@ -134,7 +129,7 @@ link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
recipe = slapos.recipe.template:jinja2
template = {{ kernel_json_location }}/{{ kernel_json_filename }}
rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
......
[buildout]
extends =
../patch/buildout.cfg
parts =
libiconv
[libiconv.gets.patch]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
md5sum = 8a20d8afe0617fce56f77537d2b84621
download-only = true
filename = libiconv.gets.patch
[libiconv]
virtual-depends =
${libiconv.gets.patch:md5sum}
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patches =
${libiconv.gets.patch:location}/${libiconv.gets.patch:filename}
${:_profile_base_location_}/libiconv.gets.patch#8a20d8afe0617fce56f77537d2b84621
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.14.tar.gz
md5sum = e34509b1623cec449dfeb73d7ce9c6c6
......@@ -10,6 +10,7 @@ parts =
[libmicrohttpd]
recipe = slapos.recipe.cmmi
shared = true
url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz
md5sum = dcd6045ecb4ea18c120afedccbd1da74
configure-options =
......
......@@ -20,10 +20,12 @@ officedir = libreoffice5.2
install =
import os
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
platform = guessPlatform()
url = options['url'] % (ARCH_DIR_MAP[platform], platform)
md5sum = options['md5sum_' + platform]
arch = {
'i386-linux-gnu': ('x86', 'x86'),
'x86_64-linux-gnu': ('x86_64', 'x86-64'),
}[multiarch()]
url = options['url'] % arch
md5sum = options['md5sum_' + arch[1]]
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
storagedir = os.path.join(workdir, 'storage')
......
[buildout]
extends =
../xz-utils/buildout.cfg
../patch/buildout.cfg
parts =
m4
[m4]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.xz
md5sum = 730bb15d96fffe47e148d1e09235af82
url = http://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz
md5sum = 0d90823e1426f1da2fd872df0311298d
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patches =
https://sources.debian.org/data/main/m/m4/1.4.18-5/debian/patches/01-fix-ftbfs-with-glibc-2.28.patch#058a786425e507f911649205b61ffcac
......@@ -14,7 +14,6 @@ recipe = slapos.recipe.template:jinja2
location = ${buildout:parts-directory}/${:_buildout_section_name_}
template = ${:_profile_base_location_}/${:filename}
rendered = ${matplotlibrc:location}/matplotlibrc
mode = 0644
backend = agg
context =
key backend matplotlibrc:backend
......
[buildout]
parts = memcached
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../libevent/buildout.cfg
[memcached-fix-array-subscript-is-above-array-bounds]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
filename = memcached-1.4-fix-array-subscript-is-above-array-bounds.patch
download-only = true
md5sum = 472508b9a4b6c0b9f5d6f2abce3444e3
[memcached-gcc-4.4.patch]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
filename = memcached-gcc-4.4.patch
download-only = true
md5sum = fd98d0cbfc4d3a25ac9808472fbe62f8
[memcached]
recipe = slapos.recipe.cmmi
url = http://memcached.googlecode.com/files/memcached-1.4.8.tar.gz
md5sum = b7104e269511621c2777367d6d6315fe
patches =
${memcached-fix-array-subscript-is-above-array-bounds:location}/${memcached-fix-array-subscript-is-above-array-bounds:filename} ${memcached-gcc-4.4.patch:location}/${memcached-gcc-4.4.patch:filename}
patch-options = -p1
configure-command =
aclocal
autoheader
automake --foreign
autoconf
./configure
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-libevent=${libevent:location}
--disable-docs
environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:%(PATH)s
LDFLAGS =-Wl,-rpath=${libevent:location}/lib
Took originally from OpenSuse spec. Needed on opensuse to avoid "error: array subscript is above array bounds"
--- memcached-orig/memcached.c
+++ memcached-new/memcached.c 2010/05/06 11:40:56
@@ -2335,15 +2335,18 @@
inline static void process_stats_detail(conn *c, const char *command) {
assert(c != NULL);
- if (strcmp(command, "on") == 0) {
+ char on[] = "on";
+ char off[] = "off";
+ char dump[] = "dump";
+ if (strcmp(command, on) == 0) {
settings.detail_enabled = 1;
out_string(c, "OK");
}
- else if (strcmp(command, "off") == 0) {
+ else if (strcmp(command, off) == 0) {
settings.detail_enabled = 0;
out_string(c, "OK");
}
- else if (strcmp(command, "dump") == 0) {
+ else if (strcmp(command, dump) == 0) {
int len;
char *stats = stats_prefix_dump(&len);
write_and_free(c, stats, len);
# In OpenSuse 11.2, 'gcc -dumpversion' returns '4.4', not '4.4.*'.
--- memcached-1.4.8/configure.ac.orig
+++ memcached-1.4.8/configure.ac
@@ -502,7 +502,7 @@
GCC_VERSION=`$CC -dumpversion`
CFLAGS="$CFLAGS -Wall -Werror -pedantic -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls"
case $GCC_VERSION in
- 4.4.*)
+ 4.4 | 4.4.*)
CFLAGS="$CFLAGS -fno-strict-aliasing"
;;
esac
[buildout]
extends =
../patch/buildout.cfg
parts =
ncurses
......@@ -9,13 +7,10 @@ recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/ncurses/ncurses-6.2.tar.gz
md5sum = e812da327b1c2214ac1aed440ea3ae8d
patch-options = -p1
patches =
${:_profile_base_location_}/ncurses-5.9-gcc-5.patch#57f4cd0cc0c0a42a5ddb2167f9546d72
configure-options =
--prefix=@@LOCATION@@
--prefix=%(location)s
--enable-pc-files
--with-pkg-config-libdir=@@LOCATION@@/lib/pkgconfig
--with-pkg-config-libdir=%(location)s/lib/pkgconfig
--with-shared
--without-ada
--without-manpages
......@@ -36,6 +31,5 @@ post-install =
# pass dummy LDCONFIG to skip needless calling of ldconfig by non-root user
environment =
LDCONFIG=/bin/echo
PATH=${patch:location}/bin:%(PATH)s
make-options =
-j1
https://bugs.gentoo.org/545114
extracted from the upstream change (which had many unrelated commits in one)
From 97bb4678dc03e753290b39bbff30ba2825df9517 Mon Sep 17 00:00:00 2001
From: "Thomas E. Dickey" <dickey@invisible-island.net>
Date: Sun, 7 Dec 2014 03:10:09 +0000
Subject: [PATCH] ncurses 5.9 - patch 20141206
+ modify MKlib_gen.sh to work around change in development version of
gcc introduced here:
https://gcc.gnu.org/ml/gcc-patches/2014-06/msg02185.html
https://gcc.gnu.org/ml/gcc-patches/2014-07/msg00236.html
(reports by Marcus Shawcroft, Maohui Lei).
diff --git a/ncurses/base/MKlib_gen.sh b/ncurses/base/MKlib_gen.sh
index d8cc3c9..b91398c 100755
--- a/ncurses/base/MKlib_gen.sh
+++ b/ncurses/base/MKlib_gen.sh
@@ -474,11 +474,22 @@ sed -n -f $ED1 \
-e 's/gen_$//' \
-e 's/ / /g' >>$TMP
+cat >$ED1 <<EOF
+s/ / /g
+s/^ //
+s/ $//
+s/P_NCURSES_BOOL/NCURSES_BOOL/g
+EOF
+
+# A patch discussed here:
+# https://gcc.gnu.org/ml/gcc-patches/2014-06/msg02185.html
+# introduces spurious #line markers. Work around that by ignoring the system's
+# attempt to define "bool" and using our own symbol here.
+sed -e 's/bool/P_NCURSES_BOOL/g' $TMP > $ED2
+cat $ED2 >$TMP
+
$preprocessor $TMP 2>/dev/null \
-| sed \
- -e 's/ / /g' \
- -e 's/^ //' \
- -e 's/_Bool/NCURSES_BOOL/g' \
+| sed -f $ED1 \
| $AWK -f $AW2 \
| sed -f $ED3 \
| sed \
[buildout]
parts =
neon
extends =
../libxml2/buildout.cfg
../openssl/buildout.cfg
../pkgconfig/buildout.cfg
../zlib/buildout.cfg
[neon]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.webdav.org/neon/neon-0.29.5.tar.gz
md5sum = ff369e69ef0f0143beb5626164e87ae2
configure-options =
--disable-static
--enable-shared
--with-ssl=openssl
--without-expat
--without-gssapi
--with-libxml2
--enable-threadsafe-ssl=posix
--disable-nls
environment =
PATH=${libxml2:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
CPPFLAGS=-I${openssl:location}/include -I${zlib:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${libxml2:location}/lib
[buildout]
extends =
../defaults.cfg
parts =
ninja
......@@ -8,8 +10,8 @@ shared = true
version = 1.10.2
url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
configure-command = ./configure.py
configure-options = --bootstrap
configure-command = ${python:executable} configure.py
configure-options = --bootstrap --verbose
make-binary = true
post-install =
set %(location)s/bin
......
[buildout]
extends =
../pygolang/buildout.cfg
../util-linux/buildout.cfg
../python-prctl/buildout.cfg
../git/buildout.cfg
# nxdtest is bin/ program to run nxdtest.
# use ${nxdtest:exe} to run it.
[nxdtest]
recipe = zc.recipe.egg:scripts
eggs = ${pygolang:egg}
${nxdtest-egg:egg}
scripts = nxdtest
# convenience for nxdtest users
<= pyprog
exe = ${buildout:bin-directory}/nxdtest
entry = nxdtest:main
eggs = ${nxdtest-egg:egg}
initialization =
# $PATH for unshare and mount
import os
path = os.environ.get('PATH', '')
if path != '':
path = ':' + path
os.environ['PATH'] = '${util-linux:location}/bin' + path
[nxdtest-egg]
recipe = zc.recipe.egg:develop
setup = ${nxdtest-repository:location}
egg = nxdtest
depends = ${python-prctl:egg}
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = 9f413221
revision = 6f75fa90
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
......@@ -17,7 +17,7 @@ egg = nxdtest[test]
[python-interpreter]
eggs +=
${pytest:eggs}
${nxdtest:eggs}
${nxdtest-egg:egg}
# env.sh for interpreter to be on $PATH.
[nxdtest-env.sh]
......
[buildout]
extends =
../patch/buildout.cfg
parts = ocropus
[ocropus-patch-scons]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
filename = ocropus.SConstruct-local-installation.patch
url = ${:_profile_base_location_}/${:filename}
md5sum = 08710ec022f3ce13e5c0b584dfee2c1c
[ocropus]
recipe = slapos.recipe.cmmi
url = http://www.nexedi.org/static/packages/source/ocropus/ocropus-0.4.4.tar.gz
md5sum = 1485dbe9aab27574bfe3c8b4395cf3ce
configure-options =
${iulib:location}
patches = ${ocropus-patch-scons:location}/${ocropus-patch-scons:filename}
patch-options = -p1
environment =
PATH=${patch:destination_directory}:${scons-bin:destination_directory}:%(PATH)s
[scons-local]
recipe = hexagonit.recipe.download
ignore-existing = true
url = http://prdownloads.sourceforge.net/scons/scons-local-2.0.1.tar.gz
[scons-bin]
recipe = plone.recipe.command
destination_directory = ${scons-local:location}
command =
ln -sf ${scons-local:location}/scons.py ${:destination_directory}/scons
[iulib-patch-scons]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
filename = iulib.SConstruct-libtiff-detection.patch
url = ${:_profile_base_location_}/${:filename}
md5sum = dbbdf909814fb84ffe47e2ff59980db7
[iulib]
recipe = slapos.recipe.cmmi
url = http://www.nexedi.org/static/packages/source/ocropus/iulib-0.4.4.tar.gz
md5sum = 3d9754dbd64c56029ce1cd7c2f61894c
# workaround http://code.google.com/p/iulib/issues/detail?id=27
patches = ${iulib-patch-scons:location}/${iulib-patch-scons:filename}
patch-options = -p1
environment =
PATH=${patch:destination_directory}:${scons-bin:destination_directory}:%(PATH)s
......@@ -13,14 +13,14 @@ extends =
parts = ocropy
[ocropy-eng-traineddata]
recipe = hexagonit.recipe.download
recipe = slapos.recipe.build:download
shared = true
filename = en-default.pyrnn.gz
md5sum = cedd140c7d7650e910f0550ad0f04727
download-only = true
url = https://raw.githubusercontent.com/zuphilip/ocropy-models/master/en-default.pyrnn.gz
url = https://raw.githubusercontent.com/zuphilip/ocropy-models/master/${:filename}
[ocropy-env]
OCROPY_MODEL_PATH = ${ocropy-eng-traineddata:location}/${ocropy-eng-traineddata:filename}
OCROPY_MODEL_PATH = ${ocropy-eng-traineddata:target}
HOME = ${ocropy:egg}
[ocropy]
......
# OpenOffice.org daemon software buildout
[buildout]
extends = ../openoffice-bin/buildout.cfg
parts +=
oood
[oood]
recipe = hexagonit.recipe.download
ignore-existing = true
url = http://www.nexedi.org/static/tarballs/oood/oood-r36294.tar.gz
md5sum = 9e71251eea4b310fd6bd4ebf8abf890e
strip-top-level-dir = true
......@@ -24,8 +24,8 @@ patch-options = -p1
patches =
${:_profile_base_location_}/no_create_privsep_path.patch#6ab983d16c9b4caf111c737dcad6ec9b
environment =
CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
configure-options =
--prefix=${:location}
--exec-prefix=${:location}
......
[buildout]
parts =
packer
[packer]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
url_x86-64 = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_linux_amd64.zip
url_x86 = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_linux_386.zip
recipe = slapos.recipe.build:download-unpacked
version = 0.7.5
# supported architectures md5sums
md5sum_x86 = a545108a0ccfde7c1e74de6c4e6fdded
md5sum_x86-64 = f343d709b84db494e8d6ec38259aa4a6
install =
import shutil
platform = guessPlatform()
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
base = https://releases.hashicorp.com/packer/${:version}/packer_${:version}
i386-linux-gnu = ${:base}_linux_386.zip a545108a0ccfde7c1e74de6c4e6fdded
x86_64-linux-gnu = ${:base}_linux_amd64.zip f343d709b84db494e8d6ec38259aa4a6
......@@ -124,6 +124,5 @@ template = inline:
exec ${perl:location}/bin/perl \
{% for item in inc -%}{% if item %} -I "{{ item }}" \
{% endif %}{% endfor %} "$@"
mode = 0755
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
perl-bin = ${:rendered}
......@@ -10,13 +10,13 @@ recipe = slapos.recipe.build
slapos_promise =
file:phantomjs-slapos
x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
base = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux
i386-linux-gnu = ${:base}-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
x86_64-linux-gnu = ${:base}-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
install =
import os
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
extract_dir = self.extract(self.download(*options[multiarch()].split()))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, location)
with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
......
# http://www.proftpd.org/ - Highly configurable GPL-licensed FTP server software
#
# Because it uses collective.recipe.environment this components needs slapos.core >= 1.4.7
# ( grid: do not hide `$USER` when running buildout )
#
[buildout]
extends =
../openssl/buildout.cfg
......@@ -15,11 +12,12 @@ extends =
../zstd/buildout.cfg
# proftpd server
[proftpd-environment]
recipe = collective.recipe.environment
[proftpd-grp]
recipe = collective.recipe.grp
[proftpd-info]
recipe = slapos.recipe.build
init =
import grp, os, pwd
options['USER'] = pwd.getpwuid(os.getuid())[0]
options['GROUP'] = grp.getgrgid(os.getgid())[0]
[proftpd]
recipe = slapos.recipe.cmmi
......@@ -36,12 +34,16 @@ environment =
CFLAGS=-DPR_RUN_DIR=\"/proc/self/cwd/var\"
CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
install_user=${proftpd-environment:USER}
install_group=${proftpd-grp:GROUP}
install_user=${proftpd-info:USER}
install_group=${proftpd-info:GROUP}
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patches =
${:_profile_base_location_}/0001-mod_rlimit-don-t-change-PR_SET_DUMPABLE-flag.patch#6e58a7a429ff96a51812dc9835e4c227
post-install =
sed -i '1c\
#!${perl:location}/bin/perl
' %(prefix)s/bin/ftpasswd
# mod_auth_web: a proftpd module to authenticate users against an HTTP service
[proftpd-mod_auth_web-repository]
......@@ -63,16 +65,6 @@ command =
${proftpd-mod_auth_web-repository:location}/mod_auth_web.c
location=${proftpd:location}/libexec/mod_auth_web.so
# ftpasswd: a perl script to manage a proftpd AuthUserFile
[ftpasswd]
recipe = slapos.recipe.build:download
url = https://raw.githubusercontent.com/proftpd/proftpd/v1.3.6/contrib/ftpasswd
md5sum = 4a47df2cab86d8de7077a445bb416f31
download-only = true
mode = 0755
[proftpd-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
......@@ -82,8 +74,7 @@ command = ${coreutils-output:test} -x ${:proftpd} -a -x ${:ftpasswd}
modules-deps =
${proftpd-mod_auth_web:recipe}
perl = ${perl:location}/bin/perl
ftpasswd = ${ftpasswd:target}
ftpasswd = ${proftpd:location}/bin/ftpasswd
proftpd = ${proftpd:location}/sbin/proftpd
prxs = ${proftpd:location}/bin/prxs
ftpdctl = ${proftpd:location}/bin/ftpdctl
......@@ -2,54 +2,49 @@
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../libtool/buildout.cfg
../cmake/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../git/buildout.cfg
../openssl/buildout.cfg
../bzip2/buildout.cfg
../perl/buildout.cfg
../gnutls/buildout.cfg
../cmake/buildout.cfg
../curl/buildout.cfg
../git/buildout.cfg
../gnutls/buildout.cfg
../libzip/buildout.cfg
../m4/buildout.cfg
../pcre/buildout.cfg
../jemalloc/buildout.cfg
../libmicrohttpd/buildout.cfg
../libtool/buildout.cfg
../m4/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../pcre/buildout.cfg
../perl/buildout.cfg
# https://github.com/libinjection/libinjection/issues/4
../python-2.7/buildout.cfg
parts =
proxysql
[proxysql]
recipe = slapos.recipe.cmmi
version = v2.0.12
shared = true
version = v2.3.2
url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz
md5sum = 70ec17fe73703a25730fdd44b6bc3ef5
prefix = @@LOCATION@@
# Patch installation path for SlapOS
pre-configure =
mkdir -p ${:prefix}/bin ${:prefix}/etc/init.d ${:prefix}/lib/systemd/system
sed -ri "
s#(\s)/usr/bin#\1${:prefix}/bin#g
s#(\s)/etc#\1${:prefix}/etc#g
s#(\s)/usr/lib#\1${:prefix}/lib#
s#(\s)/var/lib#\1${:prefix}/lib#g
s#(\s)useradd#\1echo useradd#g
s#(\s)systemctl#\1echo systemctl#g
s#(\s)chkconfig#\1echo chkconfig#g
s#(\s)update-rc.d#\1echo update-rc.d#g" Makefile
md5sum = 969129ac43c9f64641509891a116e0e1
configure-command = true
make-options = GIT_VERSION=${:version}
# The install target does not honor DESTDIR:
# let's forget it and only install what we need.
make-targets = src/proxysql
post-install =
install -Dt %(location)s/bin %(make-targets)s
environment =
GIT_VERSION=${:version}
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake-3.18:location}/bin:%(PATH)s
PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake:location}/bin:%(PATH)s:${python2.7:location}/bin
CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include
CFLAGS=-I${gnutls:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib
CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include
CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib
CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib:${zlib:location}/lib
LIBTOOL=libtool
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
patch-options = -p1
patches =
# PR #3402 (mariadb_client: backport patch to fix syntax error in cmake 3.20)
https://github.com/sysown/proxysql/commit/a3cfa56d257219f7610cd5711045bb5d84485a91.patch
......@@ -3,8 +3,9 @@ parts = pure-ftpd
[pure-ftpd]
recipe = slapos.recipe.cmmi
url = https://download.pureftpd.org/pub/pure-ftpd/releases/pure-ftpd-1.0.49.tar.bz2
md5sum = b7025f469711d88bd84a3518f67c1470
url = https://github.com/jedisct1/pure-ftpd/releases/download/${:version}/pure-ftpd-${:version}.tar.bz2
version = 1.0.50
md5sum = f55af39fc85edceba4b02ee4fa116d00
# See https://download.pureftpd.org/pub/pure-ftpd/doc/README for more configurations
# We need the trick about UPLOAD_PIPE_FILE and UPLOAD_PIPE_LOCK so that the files are created inside the $CWD/var/run
......
......@@ -49,5 +49,64 @@ initialization =
scripts = ${:interpreter}
# pyprog provides macro recipe to build python programs.
#
# Contrary to zc.recipe.egg:scripts it generates scripts that are run with
# sys.executable being correctly set. In particular it is valid to spawn
# sys.executable from inside the program and assume that all specified eggs are
# still accessible and can be imported.
#
# Usage example:
#
# [myprog]
# <= pyprog
# exe = ${buildout:bin-directory}/myprog
# entry = my.py.mod:main
# eggs = ...
[pyprog]
recipe = slapos.recipe.build
initialization =
depends =
_name = ${:_buildout_section_name_}
init =
name = options['_name'] # options['_buildout_section_name_'] does not work
exe = options['exe']
entry = options['entry']
eggs = options['eggs']
pyinit = options['initialization']
options['depends'] += '$${.%s.pyprog:recipe}' % name
# mod:func -> 'from mod import func; sys.exit(func())'
mod, func = entry.split(':')
entry_run = 'from %s import %s; sys.exit(%s())' % (mod, func, func)
# indent pyinit with ' '
__pyinit = '\n'.join([' '+_ for _ in pyinit.splitlines()])
self.buildout.parse("""
# .X.pyprog is python program to start and run entry
# it uses .X.pyexe as underlying python interpreter
[.%(name)s.pyprog]
recipe = slapos.recipe.template:jinja2
exe = %(exe)s
rendered= $${:exe}
mode = 755
template=
inline:
#!$${.%(name)s.pyexe:exe}
import sys
%(__pyinit)s
%(entry_run)s
# .X.pyexe is python interpreter used by .X.pyprog
[.%(name)s.pyexe]
<= python-interpreter
eggs += %(eggs)s
interpreter = $${:_buildout_section_name_}
exe = $${buildout:bin-directory}/$${:interpreter}
""" % locals())
[versions]
pygolang = 0.0.8
pygolang = 0.0.9
......@@ -16,3 +16,4 @@ eggs = ${pytest:eggs}
[versions]
pytest = 4.6.11:whl
pytest-timeout = 1.4.2
pytest-mock = 2.0.0:whl
[buildout]
extends =
../libcap/buildout.cfg
[python-prctl]
recipe = zc.recipe.egg:custom
egg = python-prctl
include-dirs =
${libcap:location}/include
library-dirs =
${libcap:location}/lib
rpath =
${:library-dirs}
# setup.py also calls cpp directly to verify for sys/capabilities.h
environment = python-prctl-env
[python-prctl-env]
C_INCLUDE_PATH=${libcap:location}/include
[versions]
python-prctl = 1.8.1
......@@ -72,11 +72,6 @@ archive = ${:version}
<= debian-netinst-base
arch = amd64
[debian-amd64-wheezy-netinst.iso]
<= debian-amd64-netinst-base
version = 7.11.0
md5sum = 096c1c18b44c269808bd815d58c53c8f
[debian-amd64-jessie-netinst.iso]
<= debian-amd64-netinst-base
version = 8.11.1
......
......@@ -25,21 +25,10 @@ environment =
PKG_CONFIG_PATH=${libyaml:location}/lib/
[ruby2.1]
<= ruby-common
url = http://ftp.ruby-lang.org/pub/ruby/2.1/ruby-2.1.8.tar.xz
md5sum = f18ed96bd1d5890f97a17d0d17aaefdd
[ruby2.2]
<= ruby-common
url = http://ftp.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.xz
md5sum = dbce9b9d79d90f213ba8d448b0b6ed86
[ruby2.3]
<= ruby-common
url = http://ftp.ruby-lang.org/pub/ruby/2.3/ruby-2.3.8.tar.xz
md5sum = 927e1857f3dd5a1bdec26892dbae2a05
[ruby]
<= ruby2.2
<= ruby2.3
[buildout]
# You need to define rootdir and destdir on an upper level
#
# rootdir is where the parts will be installed by the debian package (e.g. /opt/slapos)
# destdir is where we can put the files during compilation (e.g. /tmp/build-slapos)
# buildout MUST BE launched in $destdir/$rootdir (this is enforced by an assert)
#
# in this file, we teach many packages to use prefix = $rootdir/parts/package and we
# set DESTDIR = $destdir so in reality the files land in $destdir/$rootdir/parts/package
# which is EXACTLY what cmmi recipe uses by default (buildout run dir/parts/package)
# that's why everything is working
#
# please be careful when touching this file
#rootdir = %TARGET_DIRECTORY%
#destdir = %BUILD_ROOT_DIRECTORY%
#builddir = %BUILD_DIRECTORY%
extends =
buildout.cfg
......@@ -13,34 +25,24 @@ extensions =
extends-cache = extends-cache
download-cache = download-cache
# Uguu, upstream buildout.cfg must be patched as it works the other way
# around from a packager point of view at least, thus at the end static
# path, such as Python HOME directory, are wrong...
#
# Currently:
# ./configure --prefix=BUILD_DIRECTORY && make install
# Instead of:
# ./configure --prefix=INSTALL_DIRECTORY && make install DESTDIR=BUILD_DIRECTORY
[python]
init +=
buildout = self.buildout['buildout']
assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
"Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)",
buildout['directory'], buildout['destdir'] + buildout['rootdir'])
[python2.7]
configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
environment +=
DESTDIR=${buildout:destdir}
[gettext]
# Add gettext library path to RPATH as its binaries are used to build
# glib for example
environment =
PATH=${perl:location}/bin:${lunzip:location}/bin:%(PATH)s
CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include
LDFLAGS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${buildout:builddir}/parts/${:_buildout_section_name_}/lib
[bison]
configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
make-options +=
DESTDIR=${buildout:destdir}
environment +=
PERL5LIB=${perl:location}/lib/5.28.1/
......@@ -62,7 +64,6 @@ environment +=
[dbus]
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
make-options +=
......@@ -74,7 +75,6 @@ post-install =
mkdir -p ${buildout:destdir}/parts/${:_buildout_section_name_}/var/run/dbus
[dbus-glib]
location = ${buildout:parts-directory}/${:_buildout_section_name_}
environment +=
LD_LIBRARY_PATH=${dbus:location}/lib
DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
......@@ -92,21 +92,13 @@ post-install =
sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
[openssl]
shared = false
prefix = ${buildout:rootdir}/parts/${:_buildout_section_name_}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
make-options +=
INSTALL_PREFIX=${buildout:destdir}
DESTDIR=${buildout:destdir}
environment =
environment +=
PERL5LIB=${perl:location}/lib/5.28.1/
PERL=${perl:location}/bin/perl
[bison-go]
<= bison
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
make-options =
[gobject-introspection]
pre-configure =
......@@ -115,10 +107,8 @@ configure-options +=
--enable-shared
environment +=
PERL5LIB=${perl:location}/lib/5.28.1/
post-install =
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
rm -rf ${bison-go:location}
[pygobject3]
pre-configure +=
......@@ -126,18 +116,6 @@ pre-configure +=
post-install =
sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
[ncurses]
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-shared
--without-ada
--without-manpages
--without-tests
--without-normal
--without-debug
--without-gpm
--enable-rpath
[flex]
environment +=
BISON_PKGDATADIR=${bison:location}/share/bison/
......@@ -154,7 +132,6 @@ post-install =
sed -i "s#'${buildout:destdir}/parts/site_perl#'/opt/slapos/parts/site_perl#" ${perl:location}/lib/5.28.1/*-linux-thread-multi/Config.pm
[perl]
location = ${buildout:destdir}/parts/perl
configure-command =
sh Configure -des \
-Dprefix=${buildout:rootdir}/parts/${:_buildout_section_name_} \
......
......@@ -6,7 +6,6 @@ extends =
../apache/buildout.cfg
../libexpat/buildout.cfg
../libuuid/buildout.cfg
../neon/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
......@@ -14,7 +13,6 @@ extends =
../serf/buildout.cfg
../sqlite3/buildout.cfg
../zlib/buildout.cfg
../swig/buildout.cfg
../lz4/buildout.cfg
parts =
subversion
......@@ -55,45 +53,3 @@ environment =
PKG_CONFIG_PATH=${apache:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${serf:location}/lib/pkgconfig
CPPFLAGS=-I${libexpat:location}/include -I${libuuid:location}/include
LDFLAGS=-L${libexpat:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${apache:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${serf:location}/lib -Wl,-rpath=${lz4:location}/lib
[subversion-1.9]
recipe = hexagonit.recipe.cmmi
shared = true
url = http://www.apache.org/dist/subversion/subversion-1.7.9.tar.gz
md5sum = dfb083e8bfac88aa28d606168b08e4ff
configure-options =
--disable-static
--with-apr=${apache:location}/bin/apr-1-config
--with-apr-util=${apache:location}/bin/apu-1-config
--with-apxs=${apache:location}/bin/apxs
--with-zlib=${zlib:location}
--with-sqlite=${sqlite3:location}
--with-neon=${neon:location}
--with-swig=${swig:location}/bin/swig
--without-berkeley-db
--without-sasl
--without-apr_memcache
--without-gnome-keyring
--without-kwallet
--without-jdk
--without-jikes
--without-junit
--without-ctypesgen
--without-ruby-sitedir
--without-ruby-test-verbose
--disable-nls
# it seems that parallel build sometimes fails.
make-options =
-j1
make-targets =
install
swig-py
install-swig-py
environment =
PATH=${pkgconfig:location}/bin:${neon:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${apache:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig:${openssl-1.0:location}/lib/pkgconfig:${neon:location}/lib/pkgconfig
CPPFLAGS=-I${libexpat:location}/include -I${libuuid:location}/include
LDFLAGS=-L${libexpat:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${neon:location}/lib -Wl,-rpath=${apache:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
[buildout]
extends =
../java-jdk/buildout.cfg
../libsecret/buildout.cfg
../pkgconfig/buildout.cfg
../patchelf/buildout.cfg
../yarn/buildout.cfg
download-plugins.cfg
buildout.hash.cfg
[preloadTemplate.html]
recipe = slapos.recipe.build:download
shared = true
url = ${:_profile_base_location_}/${:_buildout_section_name_}
[yarn.lock]
recipe = slapos.recipe.build:download
shared = true
url = ${:_profile_base_location_}/${:_buildout_section_name_}
[theia]
recipe = slapos.recipe.cmmi
shared = true
path = ${yarn.lock:location}
# To regenerate yarn.lock, use package.json instead
# path = ${package.json:location}
environment =
TMPDIR=@@LOCATION@@/tmp
PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libsecret:pkg-config-path}
LDFLAGS=-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib
pre-configure =
mkdir -p $TMPDIR
echo '${package.json:content}' > %(location)s/package.json
# To regenerate yarn.lock, comment the line below
cp yarn.lock %(location)s
configure-command = true
make-binary = cd %(location)s && ${yarn:location}/bin/yarn
make-targets = theia build
post-install =
# On Debian 9 the -rpath are not taken into account by yarn build for unknown reasons
cd %(location)s/node_modules/keytar/build/Release
${patchelf:location}/bin/patchelf --set-rpath \
${libsecret:location}/lib:${gettext:location}/lib:${glib:location}/lib \
keytar.node
[theia-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:
#!/bin/sh
cd ${theia:location}
exec ${yarn:location}/bin/yarn theia start "$@"
[theia-open]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:
#!/bin/sh
exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
[theia-plugins]
recipe = slapos.recipe.build
urls = ${theia-download-plugins:urls}
install =
import os
for line in options['urls'].splitlines():
extension_name, url, md5sum = line.split()
extract_dir = self.extract(self.download(url, md5sum))
destination_dir = os.path.join(options['location'], extension_name)
self.copyTree(guessworkdir(extract_dir), destination_dir)
os.chmod(destination_dir, 0o750)
[package.json]
content =
{
"private": true,
"theia": {
"backend": {
"config": {
"warnOnPotentiallyInsecureHostPattern": false
}
},
"frontend": {
"config": {
"applicationName": "Theia SlapOS",
"preferences": {
"application.confirmExit": "always",
"files.associations": {
"*.cfg": "zc-buildout"
},
"files.enableTrash": false,
"files.exclude": {
"**.pyc": true,
"**.egg-info": true,
"__pycache__": true,
".git": true,
".env": true,
"**/node_modules/**": true
},
"files.watcherExclude": {
"**/.eggs/**": true,
"**/.env/**": true,
"**/.git/**": true,
"**/node_modules/**": true
},
"editor.multiCursorModifier": "ctrlCmd",
"editor.tabSize": 2,
"plantuml.server": "https://plantuml.host.vifib.net/svg/",
"plantuml.render": "PlantUMLServer",
"gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }],
"java.home": "${java-jdk:location}"
},
"warnOnPotentiallyInsecureHostPattern": false
}
},
"generator": {
"config": {
"preloadTemplate": "${preloadTemplate.html:target}"
}
}
},
"dependencies": {
"@theia/bulk-edit": "latest",
"@theia/callhierarchy": "latest",
"@theia/console": "latest",
"@theia/core": "latest",
"@theia/debug": "latest",
"@theia/editor": "latest",
"@theia/editor-preview": "latest",
"@theia/file-search": "latest",
"@theia/filesystem": "latest",
"@theia/getting-started": "latest",
"@theia/keymaps": "latest",
"@theia/markers": "latest",
"@theia/messages": "latest",
"@theia/metrics": "latest",
"@theia/mini-browser": "latest",
"@theia/monaco": "latest",
"@theia/navigator": "latest",
"@theia/outline-view": "latest",
"@theia/output": "latest",
"@theia/plugin-dev": "latest",
"@theia/plugin-ext": "latest",
"@theia/plugin-ext-vscode": "latest",
"@theia/preferences": "latest",
"@theia/preview": "latest",
"@theia/process": "latest",
"@theia/property-view": "latest",
"@theia/scm": "latest",
"@theia/scm-extra": "latest",
"@theia/search-in-workspace": "latest",
"@theia/task": "latest",
"@theia/terminal": "latest",
"@theia/timeline": "latest",
"@theia/typehierarchy": "latest",
"@theia/userstorage": "latest",
"@theia/variable-resolver": "latest",
"@theia/vsx-registry": "latest",
"@theia/workspace": "latest",
"@perrinjerome/theia-open": "latest",
"@perrinjerome/theia-open-cli": "latest"
},
"devDependencies": {
"@theia/cli": "latest"
}
}
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[preloadTemplate.html]
_update_hash_filename_ = preloadTemplate.html
md5sum = 8157c22134200bd862a07c6521ebf799
[yarn.lock]
_update_hash_filename_ = yarn.lock
md5sum = 067d2db611b21f77885f3adfd7f81453
......@@ -13,7 +13,6 @@ filename = lapack-${:version}.tgz
url = http://www.netlib.org/lapack/${:filename}
md5sum = b1d3e3e425b2e44a06760ff173104bdf
download-only = true
mode = 0644
[libatlas]
recipe = slapos.recipe.cmmi
......
......@@ -24,7 +24,7 @@ configure-options =
--disable-makeinstall-chown
--disable-makeinstall-setuid
--disable-more
--disable-mount
--enable-mount
--disable-nls
--disable-pivot_root
--disable-pylibmount
......@@ -36,7 +36,7 @@ configure-options =
--without-tinfo
--disable-tls
--disable-ul
--disable-unshare
--enable-unshare
--disable-uuidd
--disable-wall
--without-libiconv-prefix
......
......@@ -32,6 +32,17 @@ late-command =
# a DNS proxy on both IPv4 and IPv6 without translating queries to what the
# host supports.
dpkg -P rdnssd
# Fix partially Let's Encrypt certificate on old OS due to expired root CA.
# This is enough for Python but not wget.
dpkg --compare-versions 20200601~ le `dpkg-query -f '$${Version}' -W ca-certificates 2>/dev/null ||echo 1:0` || (
set ca-certificates_20200601~deb9u2_all.deb
wget http://security.debian.org/debian-security/pool/updates/main/c/ca-certificates/$1
echo 6cb3ce4329229d71a6f06b9f13c710457c05a469012ea31853ac300873d5a3e1 $1 |sha256sum -c
dpkg -i $1
rm $1
cd /etc/ssl/certs
rm DST_Root_CA_X3.pem 2e5ac55d.0 12d55845.0
)
mount |grep -q 'on / .*\bdiscard\b' || ! type fstrim || {
apt-get clean
sync
......@@ -40,10 +51,6 @@ late-command =
debconf.debconf =
debconf/frontend noninteractive
debconf/priority critical
debian-wheezy/preseed.apt-setup/services-select = volatile
debian-wheezy/preseed.mirror/country = manual
debian-wheezy/preseed.mirror/http/hostname = archive.debian.org
debian-wheezy/preseed.mirror/http/directory = /debian
debian-jessie/preseed.mirror/country = manual
debian-jessie/preseed.mirror/http/hostname = archive.debian.org
debian-jessie/preseed.mirror/http/directory = /debian
......@@ -61,10 +68,6 @@ x86_64.initrd = install.amd/initrd.gz
<= debian-stable
x86_64.iso = debian-amd64-testing-netinst.iso
[debian-wheezy]
<= debian-stable
x86_64.iso = debian-amd64-wheezy-netinst.iso
[debian-jessie]
<= debian-stable
x86_64.iso = debian-amd64-jessie-netinst.iso
......
......@@ -6,11 +6,10 @@ parts =
vnu
[vnu]
recipe = hexagonit.recipe.download
ignore-existing = true
strip-top-level-dir = true
url = https://github.com/validator/validator/releases/download/17.11.1/vnu.war_17.11.1.zip
md5sum = 2af6dec153a5011cd6fcc85ce5fb599d
recipe = slapos.recipe.build:download-unpacked
shared = true
url = https://github.com/validator/validator/releases/download/20.6.30/vnu.war_20.6.30.zip
md5sum = af595613407034da0797e4d10c03b6a2
[vnu-output]
# Shared binary location to ease migration
......
......@@ -16,19 +16,13 @@ parts =
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
url_x86-64 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz
url_x86 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic-i386.tar.xz
# supported architectures md5sums
md5sum_x86 = ce1a2c0b2cf786ccc5d5828c42c99ddd
md5sum_x86-64 = 96b7306cebb9e65355f69f7ab63df68b
base = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic
i386-linux-gnu = ${:base}-i386.tar.xz ce1a2c0b2cf786ccc5d5828c42c99ddd
x86_64-linux-gnu = ${:base}-amd64.tar.xz 96b7306cebb9e65355f69f7ab63df68b
install =
import os,shutil, sys, tempfile
platform = guessPlatform()
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
path = self.download(url, md5sum)
path = self.download(*options[multiarch()].split())
extract_dir = tempfile.mkdtemp(self.name)
self.cleanup_dir_list.append(extract_dir)
self.logger.debug('Created working directory %s', extract_dir)
......
[buildout]
parts = xpdf
[xpdf]
<= xpdf-3.02
[xpdf-patch-download]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ftp://ftp.foolabs.com/pub/xpdf/${:filename}
download-only = true
[xpdf-3.02pl1.patch]
<= xpdf-patch-download
filename = xpdf-3.02pl1.patch
md5sum = 877118786dfe27d1b7aa5a6759cc6e45
[xpdf-3.02pl2.patch]
<= xpdf-patch-download
filename = xpdf-3.02pl2.patch
md5sum = 3a5cb165ae66781e0b21e6219ae06795
[xpdf-3.02pl3.patch]
<= xpdf-patch-download
filename = xpdf-3.02pl3.patch
md5sum = 581963ede0fb5715e1a69f01b5b8ce63
[xpdf-3.02pl4.patch]
<= xpdf-patch-download
filename = xpdf-3.02pl4.patch
md5sum = 70b752716798dd341a4bf890df5f6fdc
[xpdf-3.02pl5.patch]
<= xpdf-patch-download
filename = xpdf-3.02pl5.patch
md5sum = 504902ca5e9d66c67eed03636ec6b163
[xpdf-3.02]
recipe = slapos.recipe.cmmi
md5sum = 599dc4cc65a07ee868cf92a667a913d2
url = ftp://ftp.foolabs.com/pub/xpdf/xpdf-3.02.tar.gz
configure-options =
--without-x
patch-options = -p1
patches =
${xpdf-3.02pl1.patch:location}/${xpdf-3.02pl1.patch:filename}
${xpdf-3.02pl2.patch:location}/${xpdf-3.02pl2.patch:filename}
${xpdf-3.02pl3.patch:location}/${xpdf-3.02pl3.patch:filename}
${xpdf-3.02pl4.patch:location}/${xpdf-3.02pl4.patch:filename}
${xpdf-3.02pl5.patch:location}/${xpdf-3.02pl5.patch:filename}
......@@ -5,7 +5,7 @@ parts =
[xz-utils]
recipe = slapos.recipe.cmmi
shared = true
url = http://tukaani.org/xz/xz-5.2.5.tar.bz2
url = https://tukaani.org/xz/xz-5.2.5.tar.bz2
md5sum = 33ab3ef79aa1146b83b778210e7b0a54
configure-options =
--disable-static
......@@ -24,14 +24,19 @@ yarn-download = ${yarn-download-1.16.0:location}
yarn-download = ${yarn-download-1.3.2:location}
[yarn-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${:location}/bin/yarn
template = inline:
recipe = slapos.recipe.build
shared = true
content =
#!/bin/sh
PATH=${nodejs:location}/bin/:$PATH
exec ${:yarn-download}/bin/yarn $@
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin-yarn = ${:rendered}
install =
import os
bin = os.path.join(options['location'], 'bin')
os.makedirs(bin)
with open(os.path.join(bin, 'yarn'), 'w') as f:
os.fchmod(f.fileno(), 0o755)
f.write(options['content'])
[yarn-download]
recipe = slapos.recipe.build:download-unpacked
......
......@@ -42,5 +42,4 @@ template = inline:
[versions]
mock = 3.0.5
freezegun = 1.0.0
......@@ -10,7 +10,7 @@ Usage::
"""
import os
from __future__ import print_function
import sys
import json
import collections
......@@ -19,15 +19,15 @@ import collections
def main():
exit_code = 0
for f in sys.argv[1:]:
print 'Processing %s' % (f,)
with open(f, 'rb') as infile:
print('Processing', f,)
with open(f) as infile:
try:
obj = json.load(infile, object_pairs_hook=collections.OrderedDict)
except ValueError as e:
exit_code = 1
print e
print(e, file=sys.stderr)
else:
with open(f, 'wb') as outfile:
with open(f, 'w') as outfile:
json.dump(obj, outfile, sort_keys=False, indent=2, separators=(',', ': '))
outfile.write('\n')
sys.exit(exit_code)
......
This source diff could not be displayed because it is too large. You can view the blob instead.
{
"dependencies": {
"husky": "^1.1.2",
"lint-staged": "^7.3.0"
},
"lint-staged": {
"*.json": [
"python2 ./format-json",
"git add"
"./format-json"
],
"{component,software,stack}/**": [
"python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"python2\", \"./update-hash\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'",
"python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"./update-hash\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'",
"python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"git\", \"add\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'"
]
},
"husky": {
"hooks": {
"pre-commit": "lint-staged"
}
"devDependencies": {
"husky": "^7.0.0",
"lint-staged": "^12.1.2"
},
"scripts": {
"prepare": "husky install && git config merge.tool update-hash-mergetool && git config mergetool.update-hash-mergetool.cmd './update-hash-mergetool \"$BASE\" \"$LOCAL\" \"$REMOTE\" \"$MERGED\"' && git config mergetool.update-hash-mergetool.trustExitCode true"
}
}
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.217'
version = '1.0.220'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......@@ -64,7 +64,6 @@ setup(name=name,
install_requires=[
'enum34; python_version<"3.4"', # for inotify-simple
'jsonschema',
'hexagonit.recipe.download',
'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces
'inotify_simple',
......@@ -78,7 +77,6 @@ setup(name=name,
entry_points={
'zc.buildout': [
'addresiliency = slapos.recipe.addresiliency:Recipe',
'accords = slapos.recipe.accords:Recipe',
'apacheperl = slapos.recipe.apacheperl:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe',
'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
......@@ -91,8 +89,6 @@ setup(name=name,
'check_parameter = slapos.recipe.check_parameter:Recipe',
'cloud9 = slapos.recipe.cloud9:Recipe',
'cloudooo.test = slapos.recipe.erp5_test:CloudoooRecipe',
'condor = slapos.recipe.condor:Recipe',
'condor.submit = slapos.recipe.condor:AppSubmit',
'copyfilelist = slapos.recipe.copyfilelist:Recipe',
'cron = slapos.recipe.dcron:Recipe',
'cron.d = slapos.recipe.dcron:Part',
......@@ -126,7 +122,6 @@ setup(name=name,
'neoppod.admin = slapos.recipe.neoppod:Admin',
'neoppod.master = slapos.recipe.neoppod:Master',
'neoppod.storage = slapos.recipe.neoppod:Storage',
'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
'notifier = slapos.recipe.notifier:Recipe',
'notifier.callback = slapos.recipe.notifier:Callback',
'notifier.notify = slapos.recipe.notifier:Notify',
......@@ -161,8 +156,6 @@ setup(name=name,
'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
'slapconfiguration.jsondump = slapos.recipe.slapconfiguration:JsonDump',
'slapcontainer = slapos.recipe.container:Recipe',
'sphinx= slapos.recipe.sphinx:Recipe',
'squid = slapos.recipe.squid:Recipe',
'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
......@@ -181,9 +174,6 @@ setup(name=name,
'zero-knowledge.read = slapos.recipe.zero_knowledge:ReadRecipe',
'zero-knowledge.write = slapos.recipe.zero_knowledge:WriteRecipe'
],
'slapos.recipe.nosqltestbed.plugin': [
'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',
],
},
extras_require=extras_require,
test_suite='slapos.test',
......
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import shutil
from slapos.recipe.librecipe import GenericSlapRecipe
import shutil
import subprocess
import sys
class Recipe(GenericSlapRecipe):
def _install(self):
path_list = []
accords_location = self.buildout['accordsdirectory']['accords']
parameter_dict = dict(
userid=self.options['userid'],
tenantname=self.options['tenantname'],
password=self.options['password'],
domain=self.options['domain'],
openstack_url=self.options['openstack_url'],
python_location=sys.executable,
accords_location=accords_location,
manifest_name=self.options['manifest-name'],
# XXX this is workaround
accords_lib_directory=self.options['accords_lib_directory'],
computer_id = self.computer_id,
computer_partition_id = self.computer_partition_id,
server_url = self.server_url,
software_release_url = self.software_release_url,
key_file = self.key_file,
cert_file = self.cert_file,
path = '%s:%s' % (self.options['accords_bin_directory'],
os.environ.get('PATH', '')),
)
# Generate os-config.xml
os_config_file = self.createFile(self.options['os-config'],
self.substituteTemplate(self.getTemplateFilename('os_config.xml.in'),
parameter_dict))
path_list.append(os_config_file)
# Put modified accords configuration file
accords_configuration_parameter_dict = dict(
listen_ip = self.options['listen-ip']
)
accords_configuration_file_location = self.createFile(
self.options['accords-configuration-file'],
self.substituteTemplate(self.getTemplateFilename('accords.ini.in'),
accords_configuration_parameter_dict))
path_list.append(accords_configuration_file_location)
# XXX is it dangerous?
security_path = os.path.join(accords_location, 'security')
if os.path.exists(security_path):
shutil.rmtree(security_path)
# Initiate configuration
subprocess.check_call('./accords-config',
cwd=accords_location
)
# Generate manifest
manifest_origin_location = self.options['manifest-source']
manifest_location = self.options['manifest-destination']
shutil.copy(manifest_origin_location, manifest_location)
path_list.append(manifest_location)
# Generate wrapper
wrapper_location = self.createPythonScript(self.options['accords-wrapper'],
__name__ + '.accords.runAccords',
(parameter_dict,))
path_list.append(wrapper_location)
# Generate helper for debug
self.createExecutable(
self.options['testos-wrapper'],
self.substituteTemplate(self.getTemplateFilename('testos.in'),
parameter_dict)
)
return path_list
#!%(python_location)s
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
from slapos import slap
import signal
import subprocess
from subprocess import Popen
import sys
import time
def runAccords(accords_conf):
"""Launch ACCORDS, parse manifest, broker manifest, send connection
informations to SlapOS Master. Destroy instance and stops ACCORDS at
SIGTERM."""
computer_id = accords_conf['computer_id']
computer_partition_id = accords_conf['computer_partition_id']
server_url = accords_conf['server_url']
software_release_url = accords_conf['software_release_url']
key_file = accords_conf['key_file']
cert_file = accords_conf['cert_file']
accords_lib_directory = accords_conf['accords_lib_directory']
accords_location = accords_conf['accords_location']
manifest_name = accords_conf['manifest_name']
environment = dict(
LD_LIBRARY_PATH=accords_lib_directory,
PATH=accords_conf['path'],
HOME=accords_location,
)
# Set handler to stop ACCORDS when end of world comes
# XXX use subprocess.check_call and add exception handlers
def sigtermHandler(signum, frame):
Popen(['./co-command', 'stop', '/service/*'],
cwd=accords_location, env=environment).communicate()
Popen(['./co-stop'],
cwd=accords_location, env=environment).communicate()
sys.exit(0)
signal.signal(signal.SIGTERM, sigtermHandler)
# Launch ACCORDS, parse & broke manifest to deploy instance
print 'Starting ACCORDS and friends...'
subprocess.check_call(['./co-start'],cwd=accords_location, env=environment)
print 'Parsing manifest...'
subprocess.check_call(['./co-parser', manifest_name],
cwd=accords_location, env=environment)
print 'Brokering manifest...'
subprocess.check_call(['./co-broker', manifest_name],
cwd=accords_location, env=environment)
print 'Done.'
# Parse answer
# XXX
connection_dict = dict(connection='hardcoded')
# Send information about published service to SlapOS Master
slap_connection = slap.slap()
slap_connection.initializeConnection(server_url, key_file, cert_file)
computer_partition = slap_connection.registerComputerPartition(computer_id,
computer_partition_id)
computer_partition.setConnectionDict(connection_dict)
# Go to sleep, wait kill
while(True):
time.sleep(60)
# REST host (default: 127.0.0.1)
resthost=%(listen_ip)s
# REST port (default: 8086)
#restport=8086
# Target (default: ./accords.xml)
#target=accords.xml
# Acitvate TLS (default: 0)
#tls=0
# Activate monitoring (default: 1)
#monitor=1
# Trace (default: 1)
#trace=1
# Threads (default:1)
#threads=1
# Be verbose (default: 1)
#verbose=0
# Debug (default: 1)
#debug=1
# Domain (default: occi)
#domain=occi
# Operator (default: accords)
#operator=accords
# Password (default: co-system)
#password=co-system
<?xml version="1.0" encoding="UTF8"?>
<manifest name="coips:model" xmlns="http://www.compatibleone.fr/schemes/cords.xsd">
<description>Infrastructure profile used by production tool</description>
<node name="coips:model">
<infrastructure name="coips:model">
<compute name="coips:model" cores="1" speed="1GHz" architecture="x686" memory="1GB"/>
<storage name="coips:model" size="10GB" type="SATA"/>
<network name="coips:model" vlan="true" label="database"/>
</infrastructure>
</node>
</manifest>
\ No newline at end of file
<os_configs>
<os_config
id="e1f892e3-slap-slap-slap-9354b95d3b17"
name="slaposrecipe"
description="Configuration of Account used by slapos recipe"
user="%(userid)s"
password="%(password)s"
authenticate=""
agent="CompatibleOne/OpenStackClient/1.0a.0.01"
host="%(openstack_url)s"
version="v1.1"
namespace="%(domain)s"
base=""
tls="0"
current="0"
/>
</os_configs>
#!/bin/sh
export PATH=%(path)s
export ENO_HOST=%(openstack_url)s
export ENO_USER=%(userid)s
export ENO_PASS=%(password)s
export ENO_VERSION=v1.1
export ENO_TENANT=%(tenantname)s
testos --host $ENO_HOST --password $ENO_PASS --user $ENO_USER --version $ENO_VERSION --tenant $ENO_TENANT $1 $2 $3 $4 $5 $6 $7
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import zc.buildout
import filecmp
import shutil
import re
import json
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational condor architecture."""
def __init__(self, buildout, name, options):
self.environ = {}
self.role = ''
environment_section = options.get('environment-section', '').strip()
if environment_section and environment_section in buildout:
# Use environment variables from the designated config section.
self.environ.update(buildout[environment_section])
for variable in options.get('environment', '').splitlines():
if variable.strip():
try:
key, value = variable.split('=', 1)
self.environ[key.strip()] = value
except ValueError:
raise zc.buildout.UserError('Invalid environment variable definition: %s', variable)
# Extrapolate the environment variables using values from the current
# environment.
for key in self.environ:
self.environ[key] = self.environ[key] % os.environ
return GenericBaseRecipe.__init__(self, buildout, name, options)
def _options(self, options):
#Path of condor compiled package
self.package = options['package'].strip()
self.rootdir = options['rootdirectory'].strip()
#Other condor dependances
self.javabin = options['java-bin'].strip()
self.dash = options['dash'].strip()
#Directory to deploy condor
self.prefix = options['rootdirectory'].strip()
self.localdir = options['local-dir'].strip()
self.wrapperdir = options['wrapper-dir'].strip()
self.wrapper_bin = options['bin'].strip()
self.wrapper_sbin = options['sbin'].strip()
self.diskspace = options['disk-space'].strip()
self.ipv6 = options['ip'].strip()
self.condor_host = options['condor_host'].strip()
self.collector_name = options['collector_name'].strip()
self.host_list = self.options.get('allowed-write', '*')
self.email = self.options.get('admin-email', "root@$(FULL_HOSTNAME)")
def install(self):
path_list = []
#get UID and GID for current slapuser
stat_info = os.stat(self.rootdir)
slapuser = str(stat_info.st_uid)+"."+str(stat_info.st_gid)
domain_name = 'slapos%s.com' % stat_info.st_uid
#Configure condor
configure_script = os.path.join(self.package, 'condor_configure')
install_args = [configure_script, '--install='+self.package,
'--prefix='+self.prefix, '--overwrite', '--verbose',
'--local-dir='+self.localdir] #--ignore-missing-libs
if self.options['machine-role'].strip() == "manager":
self.role = "manager,submit"
elif self.options['machine-role'].strip() == "worker":
self.role = "execute"
install_args += ['--central-manager='+self.condor_host]
install_args += ['--type='+self.role]
configure = subprocess.Popen(install_args, env=self.environ,
stdout=subprocess.PIPE)
configure.communicate()[0]
if configure.returncode is None or configure.returncode != 0:
return path_list
#Generate condor_configure file
condor_config = os.path.join(self.rootdir, 'etc/condor_config')
config_local = os.path.join(self.localdir, 'condor_config.local')
condor_configure = dict(condor_host=self.condor_host, releasedir=self.prefix,
localdir=self.localdir, config_local=config_local,
slapuser=slapuser, ipv6=self.ipv6,
diskspace=self.diskspace, javabin=self.javabin,
host_list=self.host_list, collector_name=self.collector_name,
email=self.email, domain_name=domain_name)
destination = os.path.join(condor_config)
config = self.createFile(destination,
self.substituteTemplate(self.getTemplateFilename('condor_config.generic'),
condor_configure))
path_list.append(config)
#Search if is needed to update condor_config.local file
find = re.search('NETWORK_INTERFACE[\s]*=[\s]*(%s)' % self.ipv6,
open(config_local, 'r').read())
if not find:
#update condor_config.local
with open(config_local, 'a') as f:
if self.role == "execute":
f.write("\nSTART = TRUE")
f.write("\nCOLLECTOR_NAME = %s\n \nNETWORK_INTERFACE=%s" %
(self.collector_name, self.ipv6))
#create condor binary launcher for slapos
if not os.path.exists(self.wrapper_bin):
os.makedirs(self.wrapper_bin, int('0o744', 8))
if not os.path.exists(self.wrapper_sbin):
os.makedirs(self.wrapper_sbin, int('0o744', 8))
#generate script for each file in prefix/bin
for binary in os.listdir(self.prefix+'/bin'):
wrapper_location = os.path.join(self.wrapper_bin, binary)
current_exe = os.path.join(self.prefix, 'bin', binary)
wrapper = open(wrapper_location, 'w')
content = """#!%s
export LD_LIBRARY_PATH=%s
export PATH=%s
export CONDOR_CONFIG=%s
export CONDOR_LOCATION=%s
export CONDOR_IDS=%s
export HOME=%s
export HOSTNAME=%s
exec %s $*""" % (self.dash,
self.environ['LD_LIBRARY_PATH'], self.environ['PATH'],
condor_config, self.prefix, slapuser, self.environ['HOME'],
self.environ['HOSTNAME'], current_exe)
wrapper.write(content)
wrapper.close()
path_list.append(wrapper_location)
os.chmod(wrapper_location, 0o744)
#generate script for each file in prefix/sbin
for binary in os.listdir(self.prefix+'/sbin'):
wrapper_location = os.path.join(self.wrapper_sbin, binary)
current_exe = os.path.join(self.prefix, 'sbin', binary)
wrapper = open(wrapper_location, 'w')
content = """#!%s
export LD_LIBRARY_PATH=%s
export PATH=%s
export CONDOR_CONFIG=%s
export CONDOR_LOCATION=%s
export CONDOR_IDS=%s
export HOME=%s
export HOSTNAME=%s
exec %s $*""" % (self.dash,
self.environ['LD_LIBRARY_PATH'], self.environ['PATH'],
condor_config, self.prefix, slapuser, self.environ['HOME'],
self.environ['HOSTNAME'], current_exe)
wrapper.write(content)
wrapper.close()
path_list.append(wrapper_location)
os.chmod(wrapper_location, 0o744)
#generate script for start condor
wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_condor'),
__name__ + '.configure.condorStart',
(os.path.join(self.wrapper_sbin, 'condor_reconfig'),
os.path.join(self.wrapper_sbin, 'condor_master'))
)
path_list.append(wrapper)
return path_list
class AppSubmit(GenericBaseRecipe):
"""Submit a condor job into an existing Condor master instance"""
def download(self, url, filename=None, md5sum=None):
cache = os.path.join(self.options['rootdirectory'].strip(), 'tmp')
if not os.path.exists(cache):
os.mkdir(cache)
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(url, md5sum)
if filename:
name = os.path.join(cache, filename)
os.rename(path, name)
return name
return path
def copy_file(self, source, dest):
""""Copy file with source to dest with auto replace
return True if file has been copied and dest ha been replaced
"""
result = False
if source and os.path.exists(source):
if os.path.exists(dest):
if filecmp.cmp(dest, source):
return False
os.unlink(dest)
result = True
shutil.copy(source, dest)
return result
def getFiles(self):
"""This is used to download app files if necessary and update options values"""
app_list = json.loads(self.options['condor-app-list'])
if not app_list:
return None
for app in app_list:
if app_list[app].get('files', None):
file_list = app_list[app]['files']
for file in file_list:
if file and (file.startswith('http') or file.startswith('ftp')):
file_list[file] = self.download(file_list[file])
os.chmod(file_list[file], 0o600)
else:
app_list[app]['files'] = {}
executable = app_list[app].get('executable', '')
if executable and (executable.startswith('http') or executable.startswith('ftp')):
app_list[app]['executable'] = self.download(executable,
app_list[app]['executable-name'])
os.chmod(app_list[app]['executable-name'], 0o700)
submit_file = app_list[app].get('description-file', '')
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
app_list[app]['description-file'] = self.download(submit_file, 'submit')
os.chmod(app_list[app]['description-file'], 0o600)
return app_list
def install(self):
path_list = []
#check if curent condor instance is an condor master
if self.options['machine-role'].strip() != "manager":
raise Exception("Cannot submit a job to Condor worker instance")
#Setup directory
jobdir = self.options['job-dir'].strip()
if not os.path.exists(jobdir):
os.mkdir(jobdir)
app_list = self.getFiles()
for appname in app_list:
appdir = os.path.join(jobdir, appname)
if not os.path.exists(appdir):
os.mkdir(appdir)
submitfile = os.path.join(appdir, 'submit')
self.copy_file(app_list[appname]['executable'],
os.path.join(appdir, app_list[appname]['executable-name'])
)
install = self.copy_file(app_list[appname]['description-file'], submitfile)
sig_install = os.path.join(appdir, '.install')
if install:
with open(sig_install, 'w') as f:
f.write('to_install')
for file in app_list[appname]['files']:
destination = os.path.join(appdir, file)
if os.path.exists(destination):
os.unlink(destination)
os.symlink(app_list[appname]['files'][file], destination)
#generate wrapper for submitting job
submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname),
__name__ + '.configure.submitJob',
(os.path.join(self.options['bin'].strip(), 'condor_submit'),
'submit', appdir, appname, sig_install)
)
path_list.append(submit_job)
return path_list
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import subprocess
import time
def submitJob(submit, submit_file, appdir, appname, sig_install):
"""Run condor_submit (if needed) for job deployment"""
time.sleep(10)
print "Check if needed to submit %s job's" % appname
if not os.path.exists(sig_install):
print "Nothing for install or update...Exited"
return
# '-a', "log = out.log", '-a', "error = error.log",
launch_args = submit, '-verbose', submit_file
process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=appdir)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute condor_submit.\nThe error was: %s" % result
else:
os.unlink(sig_install)
def condorStart(condor_reconfig, start_bin):
"""Start Condor if deamons is currently stopped"""
if subprocess.call(condor_reconfig):
#process failled to reconfig condor that mean that condor deamons is not curently started
subprocess.call(start_bin)
This source diff could not be displayed because it is too large. You can view the blob instead.
## Pathnames:
## Where have you installed the bin, sbin and lib condor directories?
RELEASE_DIR = %(releasedir)s
## Where is the local condor directory for each host?
## This is where the local config file(s), logs and
## spool/execute directories are located
LOCAL_DIR = %(localdir)s
## Mail parameters:
## When something goes wrong with condor at your site, who should get
## the email?
CONDOR_ADMIN =
## The user/group ID <uid>.<gid> of the "Condor" user.
## (this can also be specified in the environment)
## Note: the CONDOR_IDS setting is ignored on Win32 platforms
CONDOR_IDS = %(slapuser)s
## Condor needs to create a few lock files to synchronize access to
## various log files. Because of problems we've had with network
## filesystems and file locking over the years, we HIGHLY recommend
## that you put these lock files on a local partition on each
## machine. If you don't have your LOCAL_DIR on a local partition,
## be sure to change this entry. Whatever user (or group) condor is
## running as needs to have write access to this directory. If
## you're not running as root, this is whatever user you started up
## the condor_master as. If you are running as root, and there's a
## condor account, it's probably condor. Otherwise, it's whatever
## you've set in the CONDOR_IDS environment variable. See the Admin
## manual for details on this.
LOCK = /tmp/condor-lock.$(HOSTNAME)0.829500835462571
DAEMON_LIST = MASTER, SCHEDD, STARTD
## Java parameters:
## If you would like this machine to be able to run Java jobs,
## then set JAVA to the path of your JVM binary. If you are not
## interested in Java, there is no harm in leaving this entry
## empty or incorrect.
JAVA = %(java-bin)s
JAVA_MAXHEAP_ARGUMENT = -Xmx1024m
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import ConfigParser
import uuid
import os
import subprocess
# XXX : This is in order to get the computer_partition object
# which exposes the state of the current partition.
#
# XXX : We could have modify slapgrid in order to put the
# state of the current partition offline. But this is
# written to have the most minimal impact.
from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericSlapRecipe):
def _options(self, options):
config_filename = self.options['config']
container_uuid = None
if os.path.exists(config_filename):
config = ConfigParser.ConfigParser()
config.read(config_filename)
if config.has_option('requested', 'name'):
container_uuid = uuid.UUID(hex=config.get('requested', 'name'))
if container_uuid is None:
# uuid wasn't generated at first in order to avoid
# wasting entropy
container_uuid = uuid.uuid4()
options['slapcontainer-name'] = container_uuid.hex
return options
def _install(self):
path_list = []
self.logger.info("Putting slapcontainer configuration file...")
config = ConfigParser.ConfigParser()
config.add_section('requested')
config.set('requested', 'status',
self.computer_partition.getState())
config.set('requested', 'name', self.options['slapcontainer-name'])
config.add_section('rootfs')
config.set('rootfs', 'image', self.options['image'])
config.set('rootfs', 'complete', self.options['image-complete'])
config.add_section('network')
config.set('network', 'interface', self.options['interface'])
config.add_section('config')
config.set('config', 'file', self.options['lxc-config'])
# Just a touch
open(self.options['lxc-config'], 'a').close()
config_filename = self.options['config']
with open(config_filename, 'w') as config_file:
config.write(config_file)
path_list.append(config_filename)
return path_list
##############################################################################
#
# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import shutil
import os
import zc.buildout
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
# XXX-Cedric: write docstring
def install(self):
path_list = []
# Download and unpack application if not already existing
htdocs_location = self.options['htdocs']
if not (os.path.exists(htdocs_location) and os.listdir(htdocs_location)):
try:
os.rmdir(htdocs_location)
except:
pass
self.download(htdocs_location)
# Install php.ini
php_ini = self.createFile(os.path.join(self.options['php-ini-dir'],
'php.ini'),
self.substituteTemplate(self.getTemplateFilename('php.ini.in'),
dict(tmp_directory=self.options['tmp-dir']))
)
path_list.append(php_ini)
# Install apache
apache_config = dict(
pid_file=self.options['pid-file'],
lock_file=self.options['lock-file'],
ip=self.options['ip'],
port=self.options['port'],
error_log=self.options['error-log'],
access_log=self.options['access-log'],
document_root=self.options['htdocs'],
php_ini_dir=self.options['php-ini-dir'],
)
httpd_conf = self.createFile(self.options['httpd-conf'],
self.substituteTemplate(self.getTemplateFilename('apache.in'),
apache_config)
)
path_list.append(httpd_conf)
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['httpd-binary'],
'-f',
self.options['httpd-conf'],
'-DFOREGROUND'
))
path_list.append(wrapper)
return path_list
# Apache static configuration
# Automatically generated
# Basic server configuration
PidFile "%(pid_file)s"
Listen %(ip)s:%(port)s
PHPINIDir %(php_ini_dir)s
ServerAdmin someone@email
DefaultType text/plain
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
AddType application/x-httpd-php .php .phtml .php5 .php4
AddType application/x-httpd-php-source .phps
# Log configuration
ErrorLog "%(error_log)s"
LogLevel warn
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined
LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common
CustomLog "%(access_log)s" common
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Require all denied
</Directory>
<Directory %(document_root)s>
Options FollowSymLinks
AllowOverride All
Require all granted
</Directory>
DocumentRoot %(document_root)s
DirectoryIndex index.html index.php
# List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule mime_module modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule dir_module modules/mod_dir.so
LoadModule php5_module modules/libphp5.so
LoadModule alias_module modules/mod_alias.so
[PHP]
engine = On
safe_mode = Off
expose_php = Off
error_reporting = E_ALL & ~(E_DEPRECATED|E_NOTICE|E_WARNING)
display_errors = On
display_startup_errors = Off
log_errors = On
log_errors_max_len = 1024
ignore_repeated_errors = Off
ignore_repeated_source = Off
session.save_path = "%(tmp_directory)s"
session.auto_start = 0
date.timezone = Europe/Paris
file_uploads = On
upload_max_filesize = 8M
post_max_size = 8M
magic_quotes_gpc=Off
......@@ -32,7 +32,6 @@ import os
import sys
import inspect
import re
import shutil
import stat
from six.moves.urllib.parse import quote
import itertools
......@@ -255,31 +254,3 @@ class GenericBaseRecipe(object):
url = urlunparse((scheme, netloc, path, params, query, fragment))
return url
def setLocationOption(self):
if not self.options.get('location'):
self.options['location'] = os.path.join(
self.buildout['buildout']['parts-directory'], self.name)
def download(self, destination=None):
""" A simple wrapper around h.r.download, downloading to self.location"""
self.setLocationOption()
import hexagonit.recipe.download
if not destination:
destination = self.location
if os.path.exists(destination):
# leftovers from a previous failed attempt, removing it.
self.logger.warning('Removing already existing directory %s',
destination)
shutil.rmtree(destination)
os.mkdir(destination)
try:
options = self.options.copy()
options['destination'] = destination
hexagonit.recipe.download.Recipe(
self.buildout, self.name, options).install()
except:
shutil.rmtree(destination)
raise
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import print_function
import sys
import pkg_resources
from logging import Formatter
from slapos.recipe.librecipe import BaseSlapRecipe
class NoSQLTestBed(BaseSlapRecipe):
def _install(self):
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
try:
entry_point = pkg_resources.iter_entry_points(group='slapos.recipe.nosqltestbed.plugin',
name=self.parameter_dict.get('plugin', 'kumo')).next()
plugin_class = entry_point.load()
testbed = plugin_class()
except:
print(Formatter().formatException(sys.exc_info()))
return None
software_type = self.parameter_dict.get('slap_software_type', 'default')
if software_type is None or software_type == 'RootSoftwareInstance':
software_type = 'default'
if "run_%s" % software_type in dir(testbed) and \
callable(getattr(testbed, "run_%s" % software_type)):
return getattr(testbed, "run_%s" % software_type)(self)
else:
raise NotImplementedError("Do not support %s" % software_type)
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import pkg_resources
from logging import Formatter
class KumoTestBed(object):
def run_default(self, recipe):
run_kumo_cloud(recipe)
def run_kumo_cloud(self, recipe):
""" Deploy kumofs system on a cloud. """
kumo_cloud_config = {}
kumo_cloud_config.update(recipe.options)
kumo_cloud_config.update(recipe.parameter_dict)
kumo_cloud_config['address'] = recipe.getGlobalIPv6Address()
kumo_cloud_config['report_path'] = recipe.log_directory
kumo_cloud_config.setdefault('max_server', 4)
kumo_cloud_config.setdefault('max_tester', 5)
kumo_cloud_config.setdefault('nb_thread', 32)
kumo_cloud_config.setdefault('nb_request', 1024000)
kumo_cloud_config.setdefault('erp5_publish_url', '')
kumo_cloud_config.setdefault('erp5_publish_project', '')
computer_guid_list = []
computer_guid_list.append("COMP-23") # manager
computer_guid_list.append("COMP-13") # server 1
computer_guid_list.append("COMP-14") # server 2
computer_guid_list.append("COMP-20") # server 3
computer_guid_list.append("COMP-19") # server 4
computer_guid_list.append("COMP-23") # tester 1
computer_guid_list.append("COMP-22") # tester 2
computer_guid_list.append("COMP-14") # tester 3
computer_guid_list.append("COMP-20") # tester 4
computer_guid_list.append("COMP-19") # tester 5
kumo_cloud_config.setdefault('computer_guid_list', ":".join(computer_guid_list))
kumo_cloud_config['software_release_url'] = recipe.software_release_url
kumo_cloud_config['server_url'] = recipe.server_url
kumo_cloud_config['key_file'] = recipe.key_file
kumo_cloud_config['cert_file'] = recipe.cert_file
kumo_cloud_config['computer_id'] = recipe.computer_id
kumo_cloud_config['computer_partition_id'] = recipe.computer_partition_id
kumo_cloud_config['plugin_name'] = 'kumo'
kumo_cloud_connection = {}
kumo_cloud_connection['url'] = "http://["+kumo_cloud_config['address']+"]:5000/"
kumo_cloud_connection['computer_guid_list'] = kumo_cloud_config['computer_guid_list']
recipe.computer_partition.setConnectionDict(kumo_cloud_connection)
nosqltester_manager_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumotester_manager_run.in'))
nosqltester_manager_runner_path = recipe.createRunningWrapper("kumotester_manager",
recipe.substituteTemplate(nosqltester_manager_wrapper_template_location, kumo_cloud_config))
return [nosqltester_manager_runner_path]
def run_all(self, recipe):
""" Run all services on one machine. """
all_config = {}
all_config.update(recipe.options)
ipaddress = "[%s]" % recipe.getGlobalIPv6Address()
all_config['manager_address'] = ipaddress
all_config['manager_port'] = 19700
all_config['server_address'] = ipaddress
all_config['server_port'] = 19800
all_config['server_listen_port'] = 19900
all_config['server_storage'] = os.path.join(recipe.data_root_directory, "kumodb.tch")
all_config['gateway_address'] = ipaddress
all_config['gateway_port'] = 11411
all_config['manager_log'] = os.path.join(recipe.log_directory, "kumo-manager.log")
all_config['server_log'] = os.path.join(recipe.log_directory, "kumo-server.log")
all_config['gateway_log'] = os.path.join(recipe.log_directory, "kumo-gateway.log")
manager_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_manager_run.in'))
manager_runner_path = recipe.createRunningWrapper("kumo-manager",
recipe.substituteTemplate(manager_wrapper_template_location, all_config))
server_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_server_run.in'))
server_runner_path = recipe.createRunningWrapper("kumo-server",
recipe.substituteTemplate(server_wrapper_template_location, all_config))
gateway_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_gateway_run.in'))
gateway_runner_path = recipe.createRunningWrapper("kumo-gateway",
recipe.substituteTemplate(gateway_wrapper_template_location, all_config))
return [manager_runner_path, server_runner_path, gateway_runner_path]
def run_kumo_manager(self, recipe):
""" Run the kumofs manager. """
manager_config = {}
manager_config.update(recipe.options)
manager_config['manager_address'] = "[%s]" % recipe.getGlobalIPv6Address()
manager_config['manager_port'] = 19700
manager_config['manager_log'] = os.path.join(recipe.log_directory, "kumo-manager.log")
manager_connection = {}
manager_connection['address'] = manager_config['manager_address']
manager_connection['port'] = manager_config['manager_port']
recipe.computer_partition.setConnectionDict(manager_connection)
manager_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_manager_run.in'))
manager_runner_path = recipe.createRunningWrapper("kumo-manager",
recipe.substituteTemplate(manager_wrapper_template_location, manager_config))
return [manager_runner_path]
def run_kumo_server(self, recipe):
""" Run the kumofs server. """
server_config = {}
server_config.update(recipe.options)
server_config.update(recipe.parameter_dict)
server_config['server_address'] = "[%s]" % recipe.getGlobalIPv6Address()
server_config['server_port'] = 19800
server_config['server_listen_port'] = 19900
server_config['server_storage'] = os.path.join(recipe.var_directory,"kumodb.tch")
server_config['server_log'] = os.path.join(recipe.log_directory, "kumo-server.log")
server_connection = {}
server_connection['address'] = server_config['server_address']
recipe.computer_partition.setConnectionDict(server_connection)
server_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_server_run.in'))
server_runner_path = recipe.createRunningWrapper("kumo-server",
recipe.substituteTemplate(server_wrapper_template_location, server_config))
return [server_runner_path]
def run_kumo_gateway(self, recipe):
""" Run the kumofs gateway. """
gateway_config = {}
gateway_config.update(recipe.options)
gateway_config.update(recipe.parameter_dict)
gateway_config['gateway_address'] = "[%s]" % recipe.getGlobalIPv6Address()
gateway_config['gateway_port'] = 11411
gateway_config['gateway_log'] = os.path.join(recipe.log_directory, "kumo-gateway.log")
gateway_connection = {}
gateway_connection['address'] = gateway_config['gateway_address']
gateway_connection['port'] = gateway_config['gateway_port']
recipe.computer_partition.setConnectionDict(gateway_connection)
gateway_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_gateway_run.in'))
gateway_runner_path = recipe.createRunningWrapper("kumo-gateway",
recipe.substituteTemplate(gateway_wrapper_template_location, gateway_config))
return [gateway_runner_path]
def run_kumo_tester(self, recipe):
""" Run the kumofs tester. """
tester_config = {}
tester_config.update(recipe.options)
tester_config.update(recipe.parameter_dict)
tester_config['tester_address'] = recipe.getGlobalIPv6Address()
# tester_config['url'] = "http://%s:5000/" % tester_config['tester_address']
# tester_config['start_url'] = "http://%s:5000/start" % tester_config['tester_address']
tester_config['report_path'] = recipe.log_directory
config_dict['binary'] = "%s -g -l %s -p %s -t %s %s" % (config_dict['memstrike_binary'],
config_dict['gateway_address'].strip("[]"),
str(config_dict['gateway_port']),
str(config_dict['nb_thread']),
str(config_dict['nb_request']))
tester_config['log_directory'] = recipe.log_directory
tester_config['compress_method'] = "bz2"
tester_connection = {}
tester_connection['url'] = "http://%s:5000/" % tester_config['tester_address']
recipe.computer_partition.setConnectionDict(tester_connection)
tester_wrapper_template_location = pkg_resources.resource_filename(
'slapos.recipe.nosqltestbed', os.path.join(
'template', 'nosqltester_run.in'))
tester_runner_path = recipe.createRunningWrapper("nosqltester",
recipe.substituteTemplate(tester_wrapper_template_location, tester_config))
return [tester_runner_path]
def run_kumo_tester_and_gateway(self, recipe):
""" Run the kumofs tester and gateway on the same partition. """
address = recipe.getGlobalIPv6Address()
config_dict = {}
config_dict.update(recipe.options)
config_dict.update(recipe.parameter_dict)
# Gateway part
config_dict['gateway_address'] = "[%s]" % address
config_dict['gateway_port'] = 11411
config_dict['gateway_log'] = os.path.join(recipe.log_directory, "kumo-gateway.log")
# Tester part
config_dict['tester_address'] = address
config_dict['report_path'] = recipe.log_directory
config_dict['binary'] = "%s -g -l %s -p %s -t %s %s" % (config_dict['memstrike_binary'],
config_dict['gateway_address'].strip("[]"),
str(config_dict['gateway_port']),
str(config_dict['nb_thread']),
str(config_dict['nb_request']))
config_dict['log_directory'] = recipe.log_directory
config_dict['compress_method'] = "bz2"
connection_dict = {}
# connection_dict['address'] = config_dict['gateway_address']
# connection_dict['port'] = config_dict['gateway_port']
connection_dict['url'] = "http://%s:5000/" % config_dict['tester_address']
recipe.computer_partition.setConnectionDict(connection_dict)
gateway_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'kumo_gateway_run.in'))
gateway_runner_path = recipe.createRunningWrapper("kumo-gateway",
recipe.substituteTemplate(gateway_wrapper_template_location, config_dict))
tester_wrapper_template_location = pkg_resources.resource_filename(
'slapos.recipe.nosqltestbed', os.path.join(
'template', 'nosqltester_run.in'))
tester_runner_path = recipe.createRunningWrapper("nosqltester",
recipe.substituteTemplate(tester_wrapper_template_location, config_dict))
return [gateway_runner_path, tester_runner_path]
def run_memstrike_set(self, recipe):
""" Run memstrike in set mode. """
memstrike_config = {}
memstrike_config.update(recipe.options)
memstrike_config.update(recipe.parameter_dict)
memstrike_config['gateway_address'] = memstrike_config['gateway_address'].strip("[]")
memstrike_connection = {}
memstrike_connection['status'] = "OK"
recipe.computer_partition.setConnectionDict(memstrike_connection)
memstrike_wrapper_template_location = pkg_resources.resource_filename(
__name__, os.path.join(
'template', 'memstrike_run.in'))
memstrike_runner_path = recipe.createRunningWrapper("memstrike_set",
recipe.substituteTemplate(memstrike_wrapper_template_location, memstrike_config))
return [memstrike_runner_path]
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(gateway_binary)s -F -E -m %(manager_address)s:%(manager_port)s \
-t %(gateway_address)s:%(gateway_port)s --verbose -o %(gateway_log)s
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(manager_binary)s -a -l %(manager_address)s:%(manager_port)s \
--verbose -o %(manager_log)s
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(server_binary)s -l %(server_address)s:%(server_port)s \
-L %(server_listen_port)s -m %(manager_address)s:%(manager_port)s \
-s %(server_storage)s --verbose -o %(server_log)s
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(nosqltester_manager_binary)s %(plugin_name)s -a %(address)s \
-r %(report_path)s -s %(max_server)s -t %(max_tester)s \
--erp5-publish-url "%(erp5_publish_url)s" --erp5-publish-project "%(erp5_publish_project)s" \
%(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s \
%(nb_thread)s %(nb_request)s
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(memstrike_binary)s -s -l %(gateway_address)s -p %(gateway_port)s -t %(nb_thread)s %(nb_request)s
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(nosqltester_manager_binary)s %(plugin_name)s -a %(address)s \
-r %(report_path)s -s %(max_server)s -t %(max_tester)s \
--erp5-publish-url "%(erp5_publish_url)s" --erp5-publish-project "%(erp5_publish_project)s" \
%(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s
#!/bin/sh
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
exec %(nosqltester_binary)s -m %(host_address)s -a %(tester_address)s \
-r %(report_path)s -b "%(binary)s" -l %(log_directory)s \
-c "%(compress_method)s"
......@@ -147,7 +147,7 @@ class Recipe(GenericBaseRecipe):
with open(postgres_conf, 'w') as cfg:
cfg.write(textwrap.dedent("""\
listen_addresses = '%s'
port = %s
%s
logging_collector = on
log_rotation_size = 50MB
max_connections = 100
......@@ -163,7 +163,7 @@ class Recipe(GenericBaseRecipe):
unix_socket_permissions = 0700
""" % (
','.join(set(ipv4).union(ipv6)),
self.options['port'],
'port = %s' % self.options['port'] if self.options['port'] else '',
pgdata,
)))
......
......@@ -88,7 +88,9 @@ class Recipe(object):
Possible names depend on requested partition's software type.
state (optional)
Requested state, default value is the state of the requester.
Requested state, default value is "started", except the state of
the requester is "stopped" (which changes the default value to
"stopped").
Output:
See "return" input key.
......@@ -120,8 +122,13 @@ class Recipe(object):
slave = options.get('slave', 'false').lower() in \
librecipe.GenericBaseRecipe.TRUE_VALUES
# By default XXXX Way of doing it is ugly and dangerous
requested_state = options.get('state', buildout['slap-connection'].get('requested','started'))
# By default, propagate the state of the parent instance
# Except if parent is destroyed, as it may lead to the unexpected
# destruction of the full instance tree
default_state = buildout['slap-connection'].get('requested', 'started')
if default_state not in ('started', 'stopped'):
default_state = 'started'
requested_state = options.get('state', default_state)
options['requested-state'] = requested_state
slap = slapmodule.slap()
......
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
# Configuration file
config = dict(
data_directory=self.options['data-directory'],
ip_address=self.options['ip'],
port=self.options['sphinx-port'],
sql_port=self.options['sql-port'],
searchd_log=self.options['searchd-log'],
query_log=self.options['query-log'],
pid=self.options['pid'],
)
sphinx_conf_path = self.createFile(
self.options['configuration-file'],
self.substituteTemplate(self.getTemplateFilename('sphinx.conf.in'),
config)
)
# Create init script
wrapper = self.createWrapper(
self.options['wrapper'],
(self.options['sphinx-searchd-binary'].strip(), '-c',
sphinx_conf_path, '--nodetach'),
)
return [wrapper, sphinx_conf_path]
#############################################################################
## index definition
#############################################################################
# realtime index
#
# you can run INSERT, REPLACE, and DELETE on this index on the fly
# using MySQL protocol (see 'listen' directive below)
index erp5
{
# 'rt' index type must be specified to use RT index
type = rt
# index files path and file name, without extension
# mandatory, path must be writable, extensions will be auto-appended
path = %(data_directory)s/erp5
# RAM chunk size limit
# RT index will keep at most this much data in RAM, then flush to disk
# optional, default is 32M
#
# rt_mem_limit = 512M
# full-text field declaration
# multi-value, mandatory
rt_field = SearchableText
# unsigned integer attribute declaration
# multi-value (an arbitrary number of attributes is allowed), optional
# declares an unsigned 32-bit attribute
rt_attr_uint = uid
# RT indexes currently support the following attribute types:
# uint, bigint, float, timestamp, string
#
# rt_attr_bigint = guid
# rt_attr_float = gpa
# rt_attr_timestamp = ts_added
# rt_attr_string = author
# document attribute values (docinfo) storage mode
# optional, default is 'extern'
# known values are 'none', 'extern' and 'inline'
# docinfo = extern
# memory locking for cached data (.spa and .spi), to prevent swapping
# optional, default is 0 (do not mlock)
# requires searchd to be run from root
# mlock = 0
# a list of morphology preprocessors to apply
# optional, default is empty
#
# builtin preprocessors are 'none', 'stem_en', 'stem_ru', 'stem_enru',
# 'soundex', and 'metaphone'; additional preprocessors available from
# libstemmer are 'libstemmer_XXX', where XXX is algorithm code
# (see libstemmer_c/libstemmer/modules.txt)
#
# morphology = stem_en, stem_ru, soundex
# morphology = libstemmer_german
# morphology = libstemmer_sv
morphology = stem_en
# minimum word length at which to enable stemming
# optional, default is 1 (stem everything)
#
# min_stemming_len = 1
# stopword files list (space separated)
# optional, default is empty
# contents are plain text, charset_table and stemming are both applied
#
# stopwords = %(data_directory)s/erp5/stopwords.txt
# wordforms file, in "mapfrom > mapto" plain text format
# optional, default is empty
#
# wordforms = %(data_directory)s/erp5/wordforms.txt
# tokenizing exceptions file
# optional, default is empty
#
# plain text, case sensitive, space insensitive in map-from part
# one "Map Several Words => ToASingleOne" entry per line
#
# exceptions = %(data_directory)s/erp5/exceptions.txt
# minimum indexed word length
# default is 1 (index everything)
min_word_len = 1
# charset encoding type
# optional, default is 'sbcs'
# known types are 'sbcs' (Single Byte CharSet) and 'utf-8'
charset_type = utf-8
# charset definition and case folding rules "table"
# optional, default value depends on charset_type
#
# defaults are configured to include English and Russian characters only
# you need to change the table to include additional ones
# this behavior MAY change in future versions
#
# 'sbcs' default value is
# charset_table = 0..9, A..Z->a..z, _, a..z, U+A8->U+B8, U+B8, U+C0..U+DF->U+E0..U+FF, U+E0..U+FF
#
# 'utf-8' default value is
# charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F
charset_table = \
U+00C0->a, U+00C1->a, U+00C2->a, U+00C3->a, U+00C4->a, U+00C5->a, U+00E0->a, U+00E1->a, U+00E2->a, U+00E3->a, U+00E4->a, U+00E5->a, U+0100->a, U+0101->a, U+0102->a, U+0103->a, U+010300->a, U+0104->a, U+0105->a, U+01CD->a, U+01CE->a, U+01DE->a, U+01DF->a, \
U+01E0->a, U+01E1->a, U+01FA->a, U+01FB->a, U+0200->a, U+0201->a, U+0202->a, U+0203->a, U+0226->a, U+0227->a, U+023A->a, U+0250->a, U+04D0->a, U+04D1->a, U+1D2C->a, U+1D43->a, U+1D44->a, U+1D8F->a, U+1E00->a, U+1E01->a, U+1E9A->a, U+1EA0->a, U+1EA1->a, \
U+1EA2->a, U+1EA3->a, U+1EA4->a, U+1EA5->a, U+1EA6->a, U+1EA7->a, U+1EA8->a, U+1EA9->a, U+1EAA->a, U+1EAB->a, U+1EAC->a, U+1EAD->a, U+1EAE->a, U+1EAF->a, U+1EB0->a, U+1EB1->a, U+1EB2->a, U+1EB3->a, U+1EB4->a, U+1EB5->a, U+1EB6->a, U+1EB7->a, U+2090->a, \
U+2C65->a, U+0180->b, U+0181->b, U+0182->b, U+0183->b, U+0243->b, U+0253->b, U+0299->b, U+16D2->b, U+1D03->b, U+1D2E->b, U+1D2F->b, U+1D47->b, U+1D6C->b, U+1D80->b, U+1E02->b, U+1E03->b, U+1E04->b, U+1E05->b, U+1E06->b, U+1E07->b, U+00C7->c, U+00E7->c, \
U+0106->c, U+0107->c, U+0108->c, U+0109->c, U+010A->c, U+010B->c, U+010C->c, U+010D->c, U+0187->c, U+0188->c, U+023B->c, U+023C->c, U+0255->c, U+0297->c, U+1D9C->c, U+1D9D->c, U+1E08->c, U+1E09->c, U+212D->c, U+2184->c, U+010E->d, U+010F->d, U+0110->d, \
U+0111->d, U+0189->d, U+018A->d, U+018B->d, U+018C->d, U+01C5->d, U+01F2->d, U+0221->d, U+0256->d, U+0257->d, U+1D05->d, U+1D30->d, U+1D48->d, U+1D6D->d, U+1D81->d, U+1D91->d, U+1E0A->d, U+1E0B->d, U+1E0C->d, U+1E0D->d, U+1E0E->d, U+1E0F->d, U+1E10->d, \
U+1E11->d, U+1E12->d, U+1E13->d, U+00C8->e, U+00C9->e, U+00CA->e, U+00CB->e, U+00E8->e, U+00E9->e, U+00EA->e, U+00EB->e, U+0112->e, U+0113->e, U+0114->e, U+0115->e, U+0116->e, U+0117->e, U+0118->e, U+0119->e, U+011A->e, U+011B->e, U+018E->e, U+0190->e, \
U+01DD->e, U+0204->e, U+0205->e, U+0206->e, U+0207->e, U+0228->e, U+0229->e, U+0246->e, U+0247->e, U+0258->e, U+025B->e, U+025C->e, U+025D->e, U+025E->e, U+029A->e, U+1D07->e, U+1D08->e, U+1D31->e, U+1D32->e, U+1D49->e, U+1D4B->e, U+1D4C->e, U+1D92->e, \
U+1D93->e, U+1D94->e, U+1D9F->e, U+1E14->e, U+1E15->e, U+1E16->e, U+1E17->e, U+1E18->e, U+1E19->e, U+1E1A->e, U+1E1B->e, U+1E1C->e, U+1E1D->e, U+1EB8->e, U+1EB9->e, U+1EBA->e, U+1EBB->e, U+1EBC->e, U+1EBD->e, U+1EBE->e, U+1EBF->e, U+1EC0->e, U+1EC1->e, \
U+1EC2->e, U+1EC3->e, U+1EC4->e, U+1EC5->e, U+1EC6->e, U+1EC7->e, U+2091->e, U+0191->f, U+0192->f, U+1D6E->f, U+1D82->f, U+1DA0->f, U+1E1E->f, U+1E1F->f, U+011C->g, U+011D->g, U+011E->g, U+011F->g, U+0120->g, U+0121->g, U+0122->g, U+0123->g, U+0193->g, \
U+01E4->g, U+01E5->g, U+01E6->g, U+01E7->g, U+01F4->g, U+01F5->g, U+0260->g, U+0261->g, U+0262->g, U+029B->g, U+1D33->g, U+1D4D->g, U+1D77->g, U+1D79->g, U+1D83->g, U+1DA2->g, U+1E20->g, U+1E21->g, U+0124->h, U+0125->h, U+0126->h, U+0127->h, U+021E->h, \
U+021F->h, U+0265->h, U+0266->h, U+029C->h, U+02AE->h, U+02AF->h, U+02B0->h, U+02B1->h, U+1D34->h, U+1DA3->h, U+1E22->h, U+1E23->h, U+1E24->h, U+1E25->h, U+1E26->h, U+1E27->h, U+1E28->h, U+1E29->h, U+1E2A->h, U+1E2B->h, U+1E96->h, U+210C->h, U+2C67->h, \
U+2C68->h, U+2C75->h, U+2C76->h, U+00CC->i, U+00CD->i, U+00CE->i, U+00CF->i, U+00EC->i, U+00ED->i, U+00EE->i, U+00EF->i, U+010309->i, U+0128->i, U+0129->i, U+012A->i, U+012B->i, U+012C->i, U+012D->i, U+012E->i, U+012F->i, U+0130->i, U+0131->i, U+0197->i, \
U+01CF->i, U+01D0->i, U+0208->i, U+0209->i, U+020A->i, U+020B->i, U+0268->i, U+026A->i, U+040D->i, U+0418->i, U+0419->i, U+0438->i, U+0439->i, U+0456->i, U+1D09->i, U+1D35->i, U+1D4E->i, U+1D62->i, U+1D7B->i, U+1D96->i, U+1DA4->i, U+1DA6->i, U+1DA7->i, \
U+1E2C->i, U+1E2D->i, U+1E2E->i, U+1E2F->i, U+1EC8->i, U+1EC9->i, U+1ECA->i, U+1ECB->i, U+2071->i, U+2111->i, U+0134->j, U+0135->j, U+01C8->j, U+01CB->j, U+01F0->j, U+0237->j, U+0248->j, U+0249->j, U+025F->j, U+0284->j, U+029D->j, U+02B2->j, U+1D0A->j, \
U+1D36->j, U+1DA1->j, U+1DA8->j, U+0136->k, U+0137->k, U+0198->k, U+0199->k, U+01E8->k, U+01E9->k, U+029E->k, U+1D0B->k, U+1D37->k, U+1D4F->k, U+1D84->k, U+1E30->k, U+1E31->k, U+1E32->k, U+1E33->k, U+1E34->k, U+1E35->k, U+2C69->k, U+2C6A->k, U+0139->l, \
U+013A->l, U+013B->l, U+013C->l, U+013D->l, U+013E->l, U+013F->l, U+0140->l, U+0141->l, U+0142->l, U+019A->l, U+01C8->l, U+0234->l, U+023D->l, U+026B->l, U+026C->l, U+026D->l, U+029F->l, U+02E1->l, U+1D0C->l, U+1D38->l, U+1D85->l, U+1DA9->l, U+1DAA->l, \
U+1DAB->l, U+1E36->l, U+1E37->l, U+1E38->l, U+1E39->l, U+1E3A->l, U+1E3B->l, U+1E3C->l, U+1E3D->l, U+2C60->l, U+2C61->l, U+2C62->l, U+019C->m, U+026F->m, U+0270->m, U+0271->m, U+1D0D->m, U+1D1F->m, U+1D39->m, U+1D50->m, U+1D5A->m, U+1D6F->m, U+1D86->m, \
U+1DAC->m, U+1DAD->m, U+1E3E->m, U+1E3F->m, U+1E40->m, U+1E41->m, U+1E42->m, U+1E43->m, U+00D1->n, U+00F1->n, U+0143->n, U+0144->n, U+0145->n, U+0146->n, U+0147->n, U+0148->n, U+0149->n, U+019D->n, U+019E->n, U+01CB->n, U+01F8->n, U+01F9->n, U+0220->n, \
U+0235->n, U+0272->n, U+0273->n, U+0274->n, U+1D0E->n, U+1D3A->n, U+1D3B->n, U+1D70->n, U+1D87->n, U+1DAE->n, U+1DAF->n, U+1DB0->n, U+1E44->n, U+1E45->n, U+1E46->n, U+1E47->n, U+1E48->n, U+1E49->n, U+1E4A->n, U+1E4B->n, U+207F->n, U+00D2->o, U+00D3->o, \
U+00D4->o, U+00D5->o, U+00D6->o, U+00D8->o, U+00F2->o, U+00F3->o, U+00F4->o, U+00F5->o, U+00F6->o, U+00F8->o, U+01030F->o, U+014C->o, U+014D->o, U+014E->o, U+014F->o, U+0150->o, U+0151->o, U+0186->o, U+019F->o, U+01A0->o, U+01A1->o, U+01D1->o, U+01D2->o, \
U+01EA->o, U+01EB->o, U+01EC->o, U+01ED->o, U+01FE->o, U+01FF->o, U+020C->o, U+020D->o, U+020E->o, U+020F->o, U+022A->o, U+022B->o, U+022C->o, U+022D->o, U+022E->o, U+022F->o, U+0230->o, U+0231->o, U+0254->o, U+0275->o, U+043E->o, U+04E6->o, U+04E7->o, \
U+04E8->o, U+04E9->o, U+04EA->o, U+04EB->o, U+1D0F->o, U+1D10->o, U+1D11->o, U+1D12->o, U+1D13->o, U+1D16->o, U+1D17->o, U+1D3C->o, U+1D52->o, U+1D53->o, U+1D54->o, U+1D55->o, U+1D97->o, U+1DB1->o, U+1E4C->o, U+1E4D->o, U+1E4E->o, U+1E4F->o, U+1E50->o, \
U+1E51->o, U+1E52->o, U+1E53->o, U+1ECC->o, U+1ECD->o, U+1ECE->o, U+1ECF->o, U+1ED0->o, U+1ED1->o, U+1ED2->o, U+1ED3->o, U+1ED4->o, U+1ED5->o, U+1ED6->o, U+1ED7->o, U+1ED8->o, U+1ED9->o, U+1EDA->o, U+1EDB->o, U+1EDC->o, U+1EDD->o, U+1EDE->o, U+1EDF->o, \
U+1EE0->o, U+1EE1->o, U+1EE2->o, U+1EE3->o, U+2092->o, U+2C9E->o, U+2C9F->o, U+01A4->p, U+01A5->p, U+1D18->p, U+1D3E->p, U+1D56->p, U+1D71->p, U+1D7D->p, U+1D88->p, U+1E54->p, U+1E55->p, U+1E56->p, U+1E57->p, U+2C63->p, U+024A->q, U+024B->q, U+02A0->q, \
U+0154->r, U+0155->r, U+0156->r, U+0157->r, U+0158->r, U+0159->r, U+0210->r, U+0211->r, U+0212->r, U+0213->r, U+024C->r, U+024D->r, U+0279->r, U+027A->r, U+027B->r, U+027C->r, U+027D->r, U+027E->r, U+027F->r, U+0280->r, U+0281->r, U+02B3->r, U+02B4->r, \
U+02B5->r, U+02B6->r, U+1D19->r, U+1D1A->r, U+1D3F->r, U+1D63->r, U+1D72->r, U+1D73->r, U+1D89->r, U+1DCA->r, U+1E58->r, U+1E59->r, U+1E5A->r, U+1E5B->r, U+1E5C->r, U+1E5D->r, U+1E5E->r, U+1E5F->r, U+211C->r, U+2C64->r, U+00DF->s, U+015A->s, U+015B->s, \
U+015C->s, U+015D->s, U+015E->s, U+015F->s, U+0160->s, U+0161->s, U+017F->s, U+0218->s, U+0219->s, U+023F->s, U+0282->s, U+02E2->s, U+1D74->s, U+1D8A->s, U+1DB3->s, U+1E60->s, U+1E61->s, U+1E62->s, U+1E63->s, U+1E64->s, U+1E65->s, U+1E66->s, U+1E67->s, \
U+1E68->s, U+1E69->s, U+1E9B->s, U+0162->t, U+0163->t, U+0164->t, U+0165->t, U+0166->t, U+0167->t, U+01AB->t, U+01AC->t, U+01AD->t, U+01AE->t, U+021A->t, U+021B->t, U+0236->t, U+023E->t, U+0287->t, U+0288->t, U+1D1B->t, U+1D40->t, U+1D57->t, U+1D75->t, \
U+1DB5->t, U+1E6A->t, U+1E6B->t, U+1E6C->t, U+1E6D->t, U+1E6E->t, U+1E6F->t, U+1E70->t, U+1E71->t, U+1E97->t, U+2C66->t, U+00D9->u, U+00DA->u, U+00DB->u, U+00DC->u, U+00F9->u, U+00FA->u, U+00FB->u, U+00FC->u, U+010316->u, U+0168->u, U+0169->u, U+016A->u, \
U+016B->u, U+016C->u, U+016D->u, U+016E->u, U+016F->u, U+0170->u, U+0171->u, U+0172->u, U+0173->u, U+01AF->u, U+01B0->u, U+01D3->u, U+01D4->u, U+01D5->u, U+01D6->u, U+01D7->u, U+01D8->u, U+01D9->u, U+01DA->u, U+01DB->u, U+01DC->u, U+0214->u, U+0215->u, \
U+0216->u, U+0217->u, U+0244->u, U+0289->u, U+1D1C->u, U+1D1D->u, U+1D1E->u, U+1D41->u, U+1D58->u, U+1D59->u, U+1D64->u, U+1D7E->u, U+1D99->u, U+1DB6->u, U+1DB8->u, U+1E72->u, U+1E73->u, U+1E74->u, U+1E75->u, U+1E76->u, U+1E77->u, U+1E78->u, U+1E79->u, \
U+1E7A->u, U+1E7B->u, U+1EE4->u, U+1EE5->u, U+1EE6->u, U+1EE7->u, U+1EE8->u, U+1EE9->u, U+1EEA->u, U+1EEB->u, U+1EEC->u, U+1EED->u, U+1EEE->u, U+1EEF->u, U+1EF0->u, U+1EF1->u, U+01B2->v, U+0245->v, U+028B->v, U+028C->v, U+1D20->v, U+1D5B->v, U+1D65->v, \
U+1D8C->v, U+1DB9->v, U+1DBA->v, U+1E7C->v, U+1E7D->v, U+1E7E->v, U+1E7F->v, U+2C74->v, U+0174->w, U+0175->w, U+028D->w, U+02B7->w, U+1D21->w, U+1D42->w, U+1E80->w, U+1E81->w, U+1E82->w, U+1E83->w, U+1E84->w, U+1E85->w, U+1E86->w, U+1E87->w, U+1E88->w, \
U+1E89->w, U+1E98->w, U+02E3->x, U+1D8D->x, U+1E8A->x, U+1E8B->x, U+1E8C->x, U+1E8D->x, U+2093->x, U+00DD->y, U+00FD->y, U+00FF->y, U+0176->y, U+0177->y, U+0178->y, U+01B3->y, U+01B4->y, U+0232->y, U+0233->y, U+024E->y, U+024F->y, U+028E->y, U+028F->y, \
U+02B8->y, U+1E8E->y, U+1E8F->y, U+1E99->y, U+1EF2->y, U+1EF3->y, U+1EF4->y, U+1EF5->y, U+1EF6->y, U+1EF7->y, U+1EF8->y, U+1EF9->y, U+0179->z, U+017A->z, U+017B->z, U+017C->z, U+017D->z, U+017E->z, U+01B5->z, U+01B6->z, U+0224->z, U+0225->z, U+0240->z, \
U+0290->z, U+0291->z, U+1D22->z, U+1D76->z, U+1D8E->z, U+1DBB->z, U+1DBC->z, U+1DBD->z, U+1E90->z, U+1E91->z, U+1E92->z, U+1E93->z, U+1E94->z, U+1E95->z, U+2128->z, U+2C6B->z, U+2C6C->z, U+00C6->U+00E6, U+01E2->U+00E6, U+01E3->U+00E6, U+01FC->U+00E6, \
U+01FD->U+00E6, U+1D01->U+00E6, U+1D02->U+00E6, U+1D2D->U+00E6, U+1D46->U+00E6, U+00E6, U+0622->U+0627, U+0623->U+0627, U+0624->U+0648, U+0625->U+0627, U+0626->U+064A, U+06C0->U+06D5, U+06C2->U+06C1, U+06D3->U+06D2, U+FB50->U+0671, U+FB51->U+0671, U+FB52->U+067B, \
U+FB53->U+067B, U+FB54->U+067B, U+FB56->U+067E, U+FB57->U+067E, U+FB58->U+067E, U+FB5A->U+0680, U+FB5B->U+0680, U+FB5C->U+0680, U+FB5E->U+067A, U+FB5F->U+067A, U+FB60->U+067A, U+FB62->U+067F, U+FB63->U+067F, U+FB64->U+067F, U+FB66->U+0679, U+FB67->U+0679, \
U+FB68->U+0679, U+FB6A->U+06A4, U+FB6B->U+06A4, U+FB6C->U+06A4, U+FB6E->U+06A6, U+FB6F->U+06A6, U+FB70->U+06A6, U+FB72->U+0684, U+FB73->U+0684, U+FB74->U+0684, U+FB76->U+0683, U+FB77->U+0683, U+FB78->U+0683, U+FB7A->U+0686, U+FB7B->U+0686, U+FB7C->U+0686, \
U+FB7E->U+0687, U+FB7F->U+0687, U+FB80->U+0687, U+FB82->U+068D, U+FB83->U+068D, U+FB84->U+068C, U+FB85->U+068C, U+FB86->U+068E, U+FB87->U+068E, U+FB88->U+0688, U+FB89->U+0688, U+FB8A->U+0698, U+FB8B->U+0698, U+FB8C->U+0691, U+FB8D->U+0691, U+FB8E->U+06A9, \
U+FB8F->U+06A9, U+FB90->U+06A9, U+FB92->U+06AF, U+FB93->U+06AF, U+FB94->U+06AF, U+FB96->U+06B3, U+FB97->U+06B3, U+FB98->U+06B3, U+FB9A->U+06B1, U+FB9B->U+06B1, U+FB9C->U+06B1, U+FB9E->U+06BA, U+FB9F->U+06BA, U+FBA0->U+06BB, U+FBA1->U+06BB, U+FBA2->U+06BB, \
U+FBA4->U+06C0, U+FBA5->U+06C0, U+FBA6->U+06C1, U+FBA7->U+06C1, U+FBA8->U+06C1, U+FBAA->U+06BE, U+FBAB->U+06BE, U+FBAC->U+06BE, U+FBAE->U+06D2, U+FBAF->U+06D2, U+FBB0->U+06D3, U+FBB1->U+06D3, U+FBD3->U+06AD, U+FBD4->U+06AD, U+FBD5->U+06AD, U+FBD7->U+06C7, \
U+FBD8->U+06C7, U+FBD9->U+06C6, U+FBDA->U+06C6, U+FBDB->U+06C8, U+FBDC->U+06C8, U+FBDD->U+0677, U+FBDE->U+06CB, U+FBDF->U+06CB, U+FBE0->U+06C5, U+FBE1->U+06C5, U+FBE2->U+06C9, U+FBE3->U+06C9, U+FBE4->U+06D0, U+FBE5->U+06D0, U+FBE6->U+06D0, U+FBE8->U+0649, \
U+FBFC->U+06CC, U+FBFD->U+06CC, U+FBFE->U+06CC, U+0621, U+0627..U+063A, U+0641..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06BF, U+06C1, U+06C3..U+06D2, U+06D5, U+06EE..U+06FC, U+06FF, U+0750..U+076D, U+FB55, U+FB59, U+FB5D, U+FB61, U+FB65, U+FB69, \
U+FB6D, U+FB71, U+FB75, U+FB79, U+FB7D, U+FB81, U+FB91, U+FB95, U+FB99, U+FB9D, U+FBA3, U+FBA9, U+FBAD, U+FBD6, U+FBE7, U+FBE9, U+FBFF, U+0531..U+0556->U+0561..U+0586, U+0561..U+0586, U+0587, U+09DC->U+09A1, U+09DD->U+09A2, U+09DF->U+09AF, U+09F0->U+09AC, \
U+09F1->U+09AC, U+0985..U+0990, U+0993..U+09B0, U+09B2, U+09B6..U+09B9, U+09CE, U+09E0, U+09E1, U+09E6..U+09EF, U+F900->U+8C48, U+F901->U+66F4, U+F902->U+8ECA, U+F903->U+8CC8, U+F904->U+6ED1, U+F905->U+4E32, U+F906->U+53E5, U+F907->U+9F9C, U+F908->U+9F9C, \
U+F909->U+5951, U+F90A->U+91D1, U+F90B->U+5587, U+F90C->U+5948, U+F90D->U+61F6, U+F90E->U+7669, U+F90F->U+7F85, U+F910->U+863F, U+F911->U+87BA, U+F912->U+88F8, U+F913->U+908F, U+F914->U+6A02, U+F915->U+6D1B, U+F916->U+70D9, U+F917->U+73DE, U+F918->U+843D, \
U+F919->U+916A, U+F91A->U+99F1, U+F91B->U+4E82, U+F91C->U+5375, U+F91D->U+6B04, U+F91E->U+721B, U+F91F->U+862D, U+F920->U+9E1E, U+F921->U+5D50, U+F922->U+6FEB, U+F923->U+85CD, U+F924->U+8964, U+F925->U+62C9, U+F926->U+81D8, U+F927->U+881F, U+F928->U+5ECA, \
U+F929->U+6717, U+F92A->U+6D6A, U+F92B->U+72FC, U+F92C->U+90CE, U+F92D->U+4F86, U+F92E->U+51B7, U+F92F->U+52DE, U+F930->U+64C4, U+F931->U+6AD3, U+F932->U+7210, U+F933->U+76E7, U+F934->U+8001, U+F935->U+8606, U+F936->U+865C, U+F937->U+8DEF, U+F938->U+9732, \
U+F939->U+9B6F, U+F93A->U+9DFA, U+F93B->U+788C, U+F93C->U+797F, U+F93D->U+7DA0, U+F93E->U+83C9, U+F93F->U+9304, U+F940->U+9E7F, U+F941->U+8AD6, U+F942->U+58DF, U+F943->U+5F04, U+F944->U+7C60, U+F945->U+807E, U+F946->U+7262, U+F947->U+78CA, U+F948->U+8CC2, \
U+F949->U+96F7, U+F94A->U+58D8, U+F94B->U+5C62, U+F94C->U+6A13, U+F94D->U+6DDA, U+F94E->U+6F0F, U+F94F->U+7D2F, U+F950->U+7E37, U+F951->U+964B, U+F952->U+52D2, U+F953->U+808B, U+F954->U+51DC, U+F955->U+51CC, U+F956->U+7A1C, U+F957->U+7DBE, U+F958->U+83F1, \
U+F959->U+9675, U+F95A->U+8B80, U+F95B->U+62CF, U+F95C->U+6A02, U+F95D->U+8AFE, U+F95E->U+4E39, U+F95F->U+5BE7, U+F960->U+6012, U+F961->U+7387, U+F962->U+7570, U+F963->U+5317, U+F964->U+78FB, U+F965->U+4FBF, U+F966->U+5FA9, U+F967->U+4E0D, U+F968->U+6CCC, \
U+F969->U+6578, U+F96A->U+7D22, U+F96B->U+53C3, U+F96C->U+585E, U+F96D->U+7701, U+F96E->U+8449, U+F96F->U+8AAA, U+F970->U+6BBA, U+F971->U+8FB0, U+F972->U+6C88, U+F973->U+62FE, U+F974->U+82E5, U+F975->U+63A0, U+F976->U+7565, U+F977->U+4EAE, U+F978->U+5169, \
U+F979->U+51C9, U+F97A->U+6881, U+F97B->U+7CE7, U+F97C->U+826F, U+F97D->U+8AD2, U+F97E->U+91CF, U+F97F->U+52F5, U+F980->U+5442, U+F981->U+5973, U+F982->U+5EEC, U+F983->U+65C5, U+F984->U+6FFE, U+F985->U+792A, U+F986->U+95AD, U+F987->U+9A6A, U+F988->U+9E97, \
U+F989->U+9ECE, U+F98A->U+529B, U+F98B->U+66C6, U+F98C->U+6B77, U+F98D->U+8F62, U+F98E->U+5E74, U+F98F->U+6190, U+F990->U+6200, U+F991->U+649A, U+F992->U+6F23, U+F993->U+7149, U+F994->U+7489, U+F995->U+79CA, U+F996->U+7DF4, U+F997->U+806F, U+F998->U+8F26, \
U+F999->U+84EE, U+F99A->U+9023, U+F99B->U+934A, U+F99C->U+5217, U+F99D->U+52A3, U+F99E->U+54BD, U+F99F->U+70C8, U+F9A0->U+88C2, U+F9A1->U+8AAA, U+F9A2->U+5EC9, U+F9A3->U+5FF5, U+F9A4->U+637B, U+F9A5->U+6BAE, U+F9A6->U+7C3E, U+F9A7->U+7375, U+F9A8->U+4EE4, \
U+F9A9->U+56F9, U+F9AA->U+5BE7, U+F9AB->U+5DBA, U+F9AC->U+601C, U+F9AD->U+73B2, U+F9AE->U+7469, U+F9AF->U+7F9A, U+F9B0->U+8046, U+F9B1->U+9234, U+F9B2->U+96F6, U+F9B3->U+9748, U+F9B4->U+9818, U+F9B5->U+4F8B, U+F9B6->U+79AE, U+F9B7->U+91B4, U+F9B8->U+96B8, \
U+F9B9->U+60E1, U+F9BA->U+4E86, U+F9BB->U+50DA, U+F9BC->U+5BEE, U+F9BD->U+5C3F, U+F9BE->U+6599, U+F9BF->U+6A02, U+F9C0->U+71CE, U+F9C1->U+7642, U+F9C2->U+84FC, U+F9C3->U+907C, U+F9C4->U+9F8D, U+F9C5->U+6688, U+F9C6->U+962E, U+F9C7->U+5289, U+F9C8->U+677B, \
U+F9C9->U+67F3, U+F9CA->U+6D41, U+F9CB->U+6E9C, U+F9CC->U+7409, U+F9CD->U+7559, U+F9CE->U+786B, U+F9CF->U+7D10, U+F9D0->U+985E, U+F9D1->U+516D, U+F9D2->U+622E, U+F9D3->U+9678, U+F9D4->U+502B, U+F9D5->U+5D19, U+F9D6->U+6DEA, U+F9D7->U+8F2A, U+F9D8->U+5F8B, \
U+F9D9->U+6144, U+F9DA->U+6817, U+F9DB->U+7387, U+F9DC->U+9686, U+F9DD->U+5229, U+F9DE->U+540F, U+F9DF->U+5C65, U+F9E0->U+6613, U+F9E1->U+674E, U+F9E2->U+68A8, U+F9E3->U+6CE5, U+F9E4->U+7406, U+F9E5->U+75E2, U+F9E6->U+7F79, U+F9E7->U+88CF, U+F9E8->U+88E1, \
U+F9E9->U+91CC, U+F9EA->U+96E2, U+F9EB->U+533F, U+F9EC->U+6EBA, U+F9ED->U+541D, U+F9EE->U+71D0, U+F9EF->U+7498, U+F9F0->U+85FA, U+F9F1->U+96A3, U+F9F2->U+9C57, U+F9F3->U+9E9F, U+F9F4->U+6797, U+F9F5->U+6DCB, U+F9F6->U+81E8, U+F9F7->U+7ACB, U+F9F8->U+7B20, \
U+F9F9->U+7C92, U+F9FA->U+72C0, U+F9FB->U+7099, U+F9FC->U+8B58, U+F9FD->U+4EC0, U+F9FE->U+8336, U+F9FF->U+523A, U+FA00->U+5207, U+FA01->U+5EA6, U+FA02->U+62D3, U+FA03->U+7CD6, U+FA04->U+5B85, U+FA05->U+6D1E, U+FA06->U+66B4, U+FA07->U+8F3B, U+FA08->U+884C, \
U+FA09->U+964D, U+FA0A->U+898B, U+FA0B->U+5ED3, U+FA0C->U+5140, U+FA0D->U+55C0, U+FA10->U+585A, U+FA12->U+6674, U+FA15->U+51DE, U+FA16->U+732A, U+FA17->U+76CA, U+FA18->U+793C, U+FA19->U+795E, U+FA1A->U+7965, U+FA1B->U+798F, U+FA1C->U+9756, U+FA1D->U+7CBE, \
U+FA1E->U+7FBD, U+FA20->U+8612, U+FA22->U+8AF8, U+FA25->U+9038, U+FA26->U+90FD, U+FA2A->U+98EF, U+FA2B->U+98FC, U+FA2C->U+9928, U+FA2D->U+9DB4, U+FA30->U+4FAE, U+FA31->U+50E7, U+FA32->U+514D, U+FA33->U+52C9, U+FA34->U+52E4, U+FA35->U+5351, U+FA36->U+559D, \
U+FA37->U+5606, U+FA38->U+5668, U+FA39->U+5840, U+FA3A->U+58A8, U+FA3B->U+5C64, U+FA3C->U+5C6E, U+FA3D->U+6094, U+FA3E->U+6168, U+FA3F->U+618E, U+FA40->U+61F2, U+FA41->U+654F, U+FA42->U+65E2, U+FA43->U+6691, U+FA44->U+6885, U+FA45->U+6D77, U+FA46->U+6E1A, \
U+FA47->U+6F22, U+FA48->U+716E, U+FA49->U+722B, U+FA4A->U+7422, U+FA4B->U+7891, U+FA4C->U+793E, U+FA4D->U+7949, U+FA4E->U+7948, U+FA4F->U+7950, U+FA50->U+7956, U+FA51->U+795D, U+FA52->U+798D, U+FA53->U+798E, U+FA54->U+7A40, U+FA55->U+7A81, U+FA56->U+7BC0, \
U+FA57->U+7DF4, U+FA58->U+7E09, U+FA59->U+7E41, U+FA5A->U+7F72, U+FA5B->U+8005, U+FA5C->U+81ED, U+FA5D->U+8279, U+FA5E->U+8279, U+FA5F->U+8457, U+FA60->U+8910, U+FA61->U+8996, U+FA62->U+8B01, U+FA63->U+8B39, U+FA64->U+8CD3, U+FA65->U+8D08, U+FA66->U+8FB6, \
U+FA67->U+9038, U+FA68->U+96E3, U+FA69->U+97FF, U+FA6A->U+983B, U+FA70->U+4E26, U+FA71->U+51B5, U+FA72->U+5168, U+FA73->U+4F80, U+FA74->U+5145, U+FA75->U+5180, U+FA76->U+52C7, U+FA77->U+52FA, U+FA78->U+559D, U+FA79->U+5555, U+FA7A->U+5599, U+FA7B->U+55E2, \
U+FA7C->U+585A, U+FA7D->U+58B3, U+FA7E->U+5944, U+FA7F->U+5954, U+FA80->U+5A62, U+FA81->U+5B28, U+FA82->U+5ED2, U+FA83->U+5ED9, U+FA84->U+5F69, U+FA85->U+5FAD, U+FA86->U+60D8, U+FA87->U+614E, U+FA88->U+6108, U+FA89->U+618E, U+FA8A->U+6160, U+FA8B->U+61F2, \
U+FA8C->U+6234, U+FA8D->U+63C4, U+FA8E->U+641C, U+FA8F->U+6452, U+FA90->U+6556, U+FA91->U+6674, U+FA92->U+6717, U+FA93->U+671B, U+FA94->U+6756, U+FA95->U+6B79, U+FA96->U+6BBA, U+FA97->U+6D41, U+FA98->U+6EDB, U+FA99->U+6ECB, U+FA9A->U+6F22, U+FA9B->U+701E, \
U+FA9C->U+716E, U+FA9D->U+77A7, U+FA9E->U+7235, U+FA9F->U+72AF, U+FAA0->U+732A, U+FAA1->U+7471, U+FAA2->U+7506, U+FAA3->U+753B, U+FAA4->U+761D, U+FAA5->U+761F, U+FAA6->U+76CA, U+FAA7->U+76DB, U+FAA8->U+76F4, U+FAA9->U+774A, U+FAAA->U+7740, U+FAAB->U+78CC, \
U+FAAC->U+7AB1, U+FAAD->U+7BC0, U+FAAE->U+7C7B, U+FAAF->U+7D5B, U+FAB0->U+7DF4, U+FAB1->U+7F3E, U+FAB2->U+8005, U+FAB3->U+8352, U+FAB4->U+83EF, U+FAB5->U+8779, U+FAB6->U+8941, U+FAB7->U+8986, U+FAB8->U+8996, U+FAB9->U+8ABF, U+FABA->U+8AF8, U+FABB->U+8ACB, \
U+FABC->U+8B01, U+FABD->U+8AFE, U+FABE->U+8AED, U+FABF->U+8B39, U+FAC0->U+8B8A, U+FAC1->U+8D08, U+FAC2->U+8F38, U+FAC3->U+9072, U+FAC4->U+9199, U+FAC5->U+9276, U+FAC6->U+967C, U+FAC7->U+96E3, U+FAC8->U+9756, U+FAC9->U+97DB, U+FACA->U+97FF, U+FACB->U+980B, \
U+FACC->U+983B, U+FACD->U+9B12, U+FACE->U+9F9C, U+FACF->U+2284A, U+FAD0->U+22844, U+FAD1->U+233D5, U+FAD2->U+3B9D, U+FAD3->U+4018, U+FAD4->U+4039, U+FAD5->U+25249, U+FAD6->U+25CD0, U+FAD7->U+27ED3, U+FAD8->U+9F43, U+FAD9->U+9F8E, U+2F800->U+4E3D, U+2F801->U+4E38, \
U+2F802->U+4E41, U+2F803->U+20122, U+2F804->U+4F60, U+2F805->U+4FAE, U+2F806->U+4FBB, U+2F807->U+5002, U+2F808->U+507A, U+2F809->U+5099, U+2F80A->U+50E7, U+2F80B->U+50CF, U+2F80C->U+349E, U+2F80D->U+2063A, U+2F80E->U+514D, U+2F80F->U+5154, U+2F810->U+5164, \
U+2F811->U+5177, U+2F812->U+2051C, U+2F813->U+34B9, U+2F814->U+5167, U+2F815->U+518D, U+2F816->U+2054B, U+2F817->U+5197, U+2F818->U+51A4, U+2F819->U+4ECC, U+2F81A->U+51AC, U+2F81B->U+51B5, U+2F81C->U+291DF, U+2F81D->U+51F5, U+2F81E->U+5203, U+2F81F->U+34DF, \
U+2F820->U+523B, U+2F821->U+5246, U+2F822->U+5272, U+2F823->U+5277, U+2F824->U+3515, U+2F825->U+52C7, U+2F826->U+52C9, U+2F827->U+52E4, U+2F828->U+52FA, U+2F829->U+5305, U+2F82A->U+5306, U+2F82B->U+5317, U+2F82C->U+5349, U+2F82D->U+5351, U+2F82E->U+535A, \
U+2F82F->U+5373, U+2F830->U+537D, U+2F831->U+537F, U+2F832->U+537F, U+2F833->U+537F, U+2F834->U+20A2C, U+2F835->U+7070, U+2F836->U+53CA, U+2F837->U+53DF, U+2F838->U+20B63, U+2F839->U+53EB, U+2F83A->U+53F1, U+2F83B->U+5406, U+2F83C->U+549E, U+2F83D->U+5438, \
U+2F83E->U+5448, U+2F83F->U+5468, U+2F840->U+54A2, U+2F841->U+54F6, U+2F842->U+5510, U+2F843->U+5553, U+2F844->U+5563, U+2F845->U+5584, U+2F846->U+5584, U+2F847->U+5599, U+2F848->U+55AB, U+2F849->U+55B3, U+2F84A->U+55C2, U+2F84B->U+5716, U+2F84C->U+5606, \
U+2F84D->U+5717, U+2F84E->U+5651, U+2F84F->U+5674, U+2F850->U+5207, U+2F851->U+58EE, U+2F852->U+57CE, U+2F853->U+57F4, U+2F854->U+580D, U+2F855->U+578B, U+2F856->U+5832, U+2F857->U+5831, U+2F858->U+58AC, U+2F859->U+214E4, U+2F85A->U+58F2, U+2F85B->U+58F7, \
U+2F85C->U+5906, U+2F85D->U+591A, U+2F85E->U+5922, U+2F85F->U+5962, U+2F860->U+216A8, U+2F861->U+216EA, U+2F862->U+59EC, U+2F863->U+5A1B, U+2F864->U+5A27, U+2F865->U+59D8, U+2F866->U+5A66, U+2F867->U+36EE, U+2F868->U+36FC, U+2F869->U+5B08, U+2F86A->U+5B3E, \
U+2F86B->U+5B3E, U+2F86C->U+219C8, U+2F86D->U+5BC3, U+2F86E->U+5BD8, U+2F86F->U+5BE7, U+2F870->U+5BF3, U+2F871->U+21B18, U+2F872->U+5BFF, U+2F873->U+5C06, U+2F874->U+5F53, U+2F875->U+5C22, U+2F876->U+3781, U+2F877->U+5C60, U+2F878->U+5C6E, U+2F879->U+5CC0, \
U+2F87A->U+5C8D, U+2F87B->U+21DE4, U+2F87C->U+5D43, U+2F87D->U+21DE6, U+2F87E->U+5D6E, U+2F87F->U+5D6B, U+2F880->U+5D7C, U+2F881->U+5DE1, U+2F882->U+5DE2, U+2F883->U+382F, U+2F884->U+5DFD, U+2F885->U+5E28, U+2F886->U+5E3D, U+2F887->U+5E69, U+2F888->U+3862, \
U+2F889->U+22183, U+2F88A->U+387C, U+2F88B->U+5EB0, U+2F88C->U+5EB3, U+2F88D->U+5EB6, U+2F88E->U+5ECA, U+2F88F->U+2A392, U+2F890->U+5EFE, U+2F891->U+22331, U+2F892->U+22331, U+2F893->U+8201, U+2F894->U+5F22, U+2F895->U+5F22, U+2F896->U+38C7, U+2F897->U+232B8, \
U+2F898->U+261DA, U+2F899->U+5F62, U+2F89A->U+5F6B, U+2F89B->U+38E3, U+2F89C->U+5F9A, U+2F89D->U+5FCD, U+2F89E->U+5FD7, U+2F89F->U+5FF9, U+2F8A0->U+6081, U+2F8A1->U+393A, U+2F8A2->U+391C, U+2F8A3->U+6094, U+2F8A4->U+226D4, U+2F8A5->U+60C7, U+2F8A6->U+6148, \
U+2F8A7->U+614C, U+2F8A8->U+614E, U+2F8A9->U+614C, U+2F8AA->U+617A, U+2F8AB->U+618E, U+2F8AC->U+61B2, U+2F8AD->U+61A4, U+2F8AE->U+61AF, U+2F8AF->U+61DE, U+2F8B0->U+61F2, U+2F8B1->U+61F6, U+2F8B2->U+6210, U+2F8B3->U+621B, U+2F8B4->U+625D, U+2F8B5->U+62B1, \
U+2F8B6->U+62D4, U+2F8B7->U+6350, U+2F8B8->U+22B0C, U+2F8B9->U+633D, U+2F8BA->U+62FC, U+2F8BB->U+6368, U+2F8BC->U+6383, U+2F8BD->U+63E4, U+2F8BE->U+22BF1, U+2F8BF->U+6422, U+2F8C0->U+63C5, U+2F8C1->U+63A9, U+2F8C2->U+3A2E, U+2F8C3->U+6469, U+2F8C4->U+647E, \
U+2F8C5->U+649D, U+2F8C6->U+6477, U+2F8C7->U+3A6C, U+2F8C8->U+654F, U+2F8C9->U+656C, U+2F8CA->U+2300A, U+2F8CB->U+65E3, U+2F8CC->U+66F8, U+2F8CD->U+6649, U+2F8CE->U+3B19, U+2F8CF->U+6691, U+2F8D0->U+3B08, U+2F8D1->U+3AE4, U+2F8D2->U+5192, U+2F8D3->U+5195, \
U+2F8D4->U+6700, U+2F8D5->U+669C, U+2F8D6->U+80AD, U+2F8D7->U+43D9, U+2F8D8->U+6717, U+2F8D9->U+671B, U+2F8DA->U+6721, U+2F8DB->U+675E, U+2F8DC->U+6753, U+2F8DD->U+233C3, U+2F8DE->U+3B49, U+2F8DF->U+67FA, U+2F8E0->U+6785, U+2F8E1->U+6852, U+2F8E2->U+6885, \
U+2F8E3->U+2346D, U+2F8E4->U+688E, U+2F8E5->U+681F, U+2F8E6->U+6914, U+2F8E7->U+3B9D, U+2F8E8->U+6942, U+2F8E9->U+69A3, U+2F8EA->U+69EA, U+2F8EB->U+6AA8, U+2F8EC->U+236A3, U+2F8ED->U+6ADB, U+2F8EE->U+3C18, U+2F8EF->U+6B21, U+2F8F0->U+238A7, U+2F8F1->U+6B54, \
U+2F8F2->U+3C4E, U+2F8F3->U+6B72, U+2F8F4->U+6B9F, U+2F8F5->U+6BBA, U+2F8F6->U+6BBB, U+2F8F7->U+23A8D, U+2F8F8->U+21D0B, U+2F8F9->U+23AFA, U+2F8FA->U+6C4E, U+2F8FB->U+23CBC, U+2F8FC->U+6CBF, U+2F8FD->U+6CCD, U+2F8FE->U+6C67, U+2F8FF->U+6D16, U+2F900->U+6D3E, \
U+2F901->U+6D77, U+2F902->U+6D41, U+2F903->U+6D69, U+2F904->U+6D78, U+2F905->U+6D85, U+2F906->U+23D1E, U+2F907->U+6D34, U+2F908->U+6E2F, U+2F909->U+6E6E, U+2F90A->U+3D33, U+2F90B->U+6ECB, U+2F90C->U+6EC7, U+2F90D->U+23ED1, U+2F90E->U+6DF9, U+2F90F->U+6F6E, \
U+2F910->U+23F5E, U+2F911->U+23F8E, U+2F912->U+6FC6, U+2F913->U+7039, U+2F914->U+701E, U+2F915->U+701B, U+2F916->U+3D96, U+2F917->U+704A, U+2F918->U+707D, U+2F919->U+7077, U+2F91A->U+70AD, U+2F91B->U+20525, U+2F91C->U+7145, U+2F91D->U+24263, U+2F91E->U+719C, \
U+2F91F->U+243AB, U+2F920->U+7228, U+2F921->U+7235, U+2F922->U+7250, U+2F923->U+24608, U+2F924->U+7280, U+2F925->U+7295, U+2F926->U+24735, U+2F927->U+24814, U+2F928->U+737A, U+2F929->U+738B, U+2F92A->U+3EAC, U+2F92B->U+73A5, U+2F92C->U+3EB8, U+2F92D->U+3EB8, \
U+2F92E->U+7447, U+2F92F->U+745C, U+2F930->U+7471, U+2F931->U+7485, U+2F932->U+74CA, U+2F933->U+3F1B, U+2F934->U+7524, U+2F935->U+24C36, U+2F936->U+753E, U+2F937->U+24C92, U+2F938->U+7570, U+2F939->U+2219F, U+2F93A->U+7610, U+2F93B->U+24FA1, U+2F93C->U+24FB8, \
U+2F93D->U+25044, U+2F93E->U+3FFC, U+2F93F->U+4008, U+2F940->U+76F4, U+2F941->U+250F3, U+2F942->U+250F2, U+2F943->U+25119, U+2F944->U+25133, U+2F945->U+771E, U+2F946->U+771F, U+2F947->U+771F, U+2F948->U+774A, U+2F949->U+4039, U+2F94A->U+778B, U+2F94B->U+4046, \
U+2F94C->U+4096, U+2F94D->U+2541D, U+2F94E->U+784E, U+2F94F->U+788C, U+2F950->U+78CC, U+2F951->U+40E3, U+2F952->U+25626, U+2F953->U+7956, U+2F954->U+2569A, U+2F955->U+256C5, U+2F956->U+798F, U+2F957->U+79EB, U+2F958->U+412F, U+2F959->U+7A40, U+2F95A->U+7A4A, \
U+2F95B->U+7A4F, U+2F95C->U+2597C, U+2F95D->U+25AA7, U+2F95E->U+25AA7, U+2F95F->U+7AEE, U+2F960->U+4202, U+2F961->U+25BAB, U+2F962->U+7BC6, U+2F963->U+7BC9, U+2F964->U+4227, U+2F965->U+25C80, U+2F966->U+7CD2, U+2F967->U+42A0, U+2F968->U+7CE8, U+2F969->U+7CE3, \
U+2F96A->U+7D00, U+2F96B->U+25F86, U+2F96C->U+7D63, U+2F96D->U+4301, U+2F96E->U+7DC7, U+2F96F->U+7E02, U+2F970->U+7E45, U+2F971->U+4334, U+2F972->U+26228, U+2F973->U+26247, U+2F974->U+4359, U+2F975->U+262D9, U+2F976->U+7F7A, U+2F977->U+2633E, U+2F978->U+7F95, \
U+2F979->U+7FFA, U+2F97A->U+8005, U+2F97B->U+264DA, U+2F97C->U+26523, U+2F97D->U+8060, U+2F97E->U+265A8, U+2F97F->U+8070, U+2F980->U+2335F, U+2F981->U+43D5, U+2F982->U+80B2, U+2F983->U+8103, U+2F984->U+440B, U+2F985->U+813E, U+2F986->U+5AB5, U+2F987->U+267A7, \
U+2F988->U+267B5, U+2F989->U+23393, U+2F98A->U+2339C, U+2F98B->U+8201, U+2F98C->U+8204, U+2F98D->U+8F9E, U+2F98E->U+446B, U+2F98F->U+8291, U+2F990->U+828B, U+2F991->U+829D, U+2F992->U+52B3, U+2F993->U+82B1, U+2F994->U+82B3, U+2F995->U+82BD, U+2F996->U+82E6, \
U+2F997->U+26B3C, U+2F998->U+82E5, U+2F999->U+831D, U+2F99A->U+8363, U+2F99B->U+83AD, U+2F99C->U+8323, U+2F99D->U+83BD, U+2F99E->U+83E7, U+2F99F->U+8457, U+2F9A0->U+8353, U+2F9A1->U+83CA, U+2F9A2->U+83CC, U+2F9A3->U+83DC, U+2F9A4->U+26C36, U+2F9A5->U+26D6B, \
U+2F9A6->U+26CD5, U+2F9A7->U+452B, U+2F9A8->U+84F1, U+2F9A9->U+84F3, U+2F9AA->U+8516, U+2F9AB->U+273CA, U+2F9AC->U+8564, U+2F9AD->U+26F2C, U+2F9AE->U+455D, U+2F9AF->U+4561, U+2F9B0->U+26FB1, U+2F9B1->U+270D2, U+2F9B2->U+456B, U+2F9B3->U+8650, U+2F9B4->U+865C, \
U+2F9B5->U+8667, U+2F9B6->U+8669, U+2F9B7->U+86A9, U+2F9B8->U+8688, U+2F9B9->U+870E, U+2F9BA->U+86E2, U+2F9BB->U+8779, U+2F9BC->U+8728, U+2F9BD->U+876B, U+2F9BE->U+8786, U+2F9BF->U+45D7, U+2F9C0->U+87E1, U+2F9C1->U+8801, U+2F9C2->U+45F9, U+2F9C3->U+8860, \
U+2F9C4->U+8863, U+2F9C5->U+27667, U+2F9C6->U+88D7, U+2F9C7->U+88DE, U+2F9C8->U+4635, U+2F9C9->U+88FA, U+2F9CA->U+34BB, U+2F9CB->U+278AE, U+2F9CC->U+27966, U+2F9CD->U+46BE, U+2F9CE->U+46C7, U+2F9CF->U+8AA0, U+2F9D0->U+8AED, U+2F9D1->U+8B8A, U+2F9D2->U+8C55, \
U+2F9D3->U+27CA8, U+2F9D4->U+8CAB, U+2F9D5->U+8CC1, U+2F9D6->U+8D1B, U+2F9D7->U+8D77, U+2F9D8->U+27F2F, U+2F9D9->U+20804, U+2F9DA->U+8DCB, U+2F9DB->U+8DBC, U+2F9DC->U+8DF0, U+2F9DD->U+208DE, U+2F9DE->U+8ED4, U+2F9DF->U+8F38, U+2F9E0->U+285D2, U+2F9E1->U+285ED, \
U+2F9E2->U+9094, U+2F9E3->U+90F1, U+2F9E4->U+9111, U+2F9E5->U+2872E, U+2F9E6->U+911B, U+2F9E7->U+9238, U+2F9E8->U+92D7, U+2F9E9->U+92D8, U+2F9EA->U+927C, U+2F9EB->U+93F9, U+2F9EC->U+9415, U+2F9ED->U+28BFA, U+2F9EE->U+958B, U+2F9EF->U+4995, U+2F9F0->U+95B7, \
U+2F9F1->U+28D77, U+2F9F2->U+49E6, U+2F9F3->U+96C3, U+2F9F4->U+5DB2, U+2F9F5->U+9723, U+2F9F6->U+29145, U+2F9F7->U+2921A, U+2F9F8->U+4A6E, U+2F9F9->U+4A76, U+2F9FA->U+97E0, U+2F9FB->U+2940A, U+2F9FC->U+4AB2, U+2F9FD->U+29496, U+2F9FE->U+980B, U+2F9FF->U+980B, \
U+2FA00->U+9829, U+2FA01->U+295B6, U+2FA02->U+98E2, U+2FA03->U+4B33, U+2FA04->U+9929, U+2FA05->U+99A7, U+2FA06->U+99C2, U+2FA07->U+99FE, U+2FA08->U+4BCE, U+2FA09->U+29B30, U+2FA0A->U+9B12, U+2FA0B->U+9C40, U+2FA0C->U+9CFD, U+2FA0D->U+4CCE, U+2FA0E->U+4CED, \
U+2FA0F->U+9D67, U+2FA10->U+2A0CE, U+2FA11->U+4CF8, U+2FA12->U+2A105, U+2FA13->U+2A20E, U+2FA14->U+2A291, U+2FA15->U+9EBB, U+2FA16->U+4D56, U+2FA17->U+9EF9, U+2FA18->U+9EFE, U+2FA19->U+9F05, U+2FA1A->U+9F0F, U+2FA1B->U+9F16, U+2FA1C->U+9F3B, U+2FA1D->U+2A600, \
U+2F00->U+4E00, U+2F01->U+4E28, U+2F02->U+4E36, U+2F03->U+4E3F, U+2F04->U+4E59, U+2F05->U+4E85, U+2F06->U+4E8C, U+2F07->U+4EA0, U+2F08->U+4EBA, U+2F09->U+513F, U+2F0A->U+5165, U+2F0B->U+516B, U+2F0C->U+5182, U+2F0D->U+5196, U+2F0E->U+51AB, U+2F0F->U+51E0, \
U+2F10->U+51F5, U+2F11->U+5200, U+2F12->U+529B, U+2F13->U+52F9, U+2F14->U+5315, U+2F15->U+531A, U+2F16->U+5338, U+2F17->U+5341, U+2F18->U+535C, U+2F19->U+5369, U+2F1A->U+5382, U+2F1B->U+53B6, U+2F1C->U+53C8, U+2F1D->U+53E3, U+2F1E->U+56D7, U+2F1F->U+571F, \
U+2F20->U+58EB, U+2F21->U+5902, U+2F22->U+590A, U+2F23->U+5915, U+2F24->U+5927, U+2F25->U+5973, U+2F26->U+5B50, U+2F27->U+5B80, U+2F28->U+5BF8, U+2F29->U+5C0F, U+2F2A->U+5C22, U+2F2B->U+5C38, U+2F2C->U+5C6E, U+2F2D->U+5C71, U+2F2E->U+5DDB, U+2F2F->U+5DE5, \
U+2F30->U+5DF1, U+2F31->U+5DFE, U+2F32->U+5E72, U+2F33->U+5E7A, U+2F34->U+5E7F, U+2F35->U+5EF4, U+2F36->U+5EFE, U+2F37->U+5F0B, U+2F38->U+5F13, U+2F39->U+5F50, U+2F3A->U+5F61, U+2F3B->U+5F73, U+2F3C->U+5FC3, U+2F3D->U+6208, U+2F3E->U+6236, U+2F3F->U+624B, \
U+2F40->U+652F, U+2F41->U+6534, U+2F42->U+6587, U+2F43->U+6597, U+2F44->U+65A4, U+2F45->U+65B9, U+2F46->U+65E0, U+2F47->U+65E5, U+2F48->U+66F0, U+2F49->U+6708, U+2F4A->U+6728, U+2F4B->U+6B20, U+2F4C->U+6B62, U+2F4D->U+6B79, U+2F4E->U+6BB3, U+2F4F->U+6BCB, \
U+2F50->U+6BD4, U+2F51->U+6BDB, U+2F52->U+6C0F, U+2F53->U+6C14, U+2F54->U+6C34, U+2F55->U+706B, U+2F56->U+722A, U+2F57->U+7236, U+2F58->U+723B, U+2F59->U+723F, U+2F5A->U+7247, U+2F5B->U+7259, U+2F5C->U+725B, U+2F5D->U+72AC, U+2F5E->U+7384, U+2F5F->U+7389, \
U+2F60->U+74DC, U+2F61->U+74E6, U+2F62->U+7518, U+2F63->U+751F, U+2F64->U+7528, U+2F65->U+7530, U+2F66->U+758B, U+2F67->U+7592, U+2F68->U+7676, U+2F69->U+767D, U+2F6A->U+76AE, U+2F6B->U+76BF, U+2F6C->U+76EE, U+2F6D->U+77DB, U+2F6E->U+77E2, U+2F6F->U+77F3, \
U+2F70->U+793A, U+2F71->U+79B8, U+2F72->U+79BE, U+2F73->U+7A74, U+2F74->U+7ACB, U+2F75->U+7AF9, U+2F76->U+7C73, U+2F77->U+7CF8, U+2F78->U+7F36, U+2F79->U+7F51, U+2F7A->U+7F8A, U+2F7B->U+7FBD, U+2F7C->U+8001, U+2F7D->U+800C, U+2F7E->U+8012, U+2F7F->U+8033, \
U+2F80->U+807F, U+2F81->U+8089, U+2F82->U+81E3, U+2F83->U+81EA, U+2F84->U+81F3, U+2F85->U+81FC, U+2F86->U+820C, U+2F87->U+821B, U+2F88->U+821F, U+2F89->U+826E, U+2F8A->U+8272, U+2F8B->U+8278, U+2F8C->U+864D, U+2F8D->U+866B, U+2F8E->U+8840, U+2F8F->U+884C, \
U+2F90->U+8863, U+2F91->U+897E, U+2F92->U+898B, U+2F93->U+89D2, U+2F94->U+8A00, U+2F95->U+8C37, U+2F96->U+8C46, U+2F97->U+8C55, U+2F98->U+8C78, U+2F99->U+8C9D, U+2F9A->U+8D64, U+2F9B->U+8D70, U+2F9C->U+8DB3, U+2F9D->U+8EAB, U+2F9E->U+8ECA, U+2F9F->U+8F9B, \
U+2FA0->U+8FB0, U+2FA1->U+8FB5, U+2FA2->U+9091, U+2FA3->U+9149, U+2FA4->U+91C6, U+2FA5->U+91CC, U+2FA6->U+91D1, U+2FA7->U+9577, U+2FA8->U+9580, U+2FA9->U+961C, U+2FAA->U+96B6, U+2FAB->U+96B9, U+2FAC->U+96E8, U+2FAD->U+9751, U+2FAE->U+975E, U+2FAF->U+9762, \
U+2FB0->U+9769, U+2FB1->U+97CB, U+2FB2->U+97ED, U+2FB3->U+97F3, U+2FB4->U+9801, U+2FB5->U+98A8, U+2FB6->U+98DB, U+2FB7->U+98DF, U+2FB8->U+9996, U+2FB9->U+9999, U+2FBA->U+99AC, U+2FBB->U+9AA8, U+2FBC->U+9AD8, U+2FBD->U+9ADF, U+2FBE->U+9B25, U+2FBF->U+9B2F, \
U+2FC0->U+9B32, U+2FC1->U+9B3C, U+2FC2->U+9B5A, U+2FC3->U+9CE5, U+2FC4->U+9E75, U+2FC5->U+9E7F, U+2FC6->U+9EA5, U+2FC7->U+9EBB, U+2FC8->U+9EC3, U+2FC9->U+9ECD, U+2FCA->U+9ED1, U+2FCB->U+9EF9, U+2FCC->U+9EFD, U+2FCD->U+9F0E, U+2FCE->U+9F13, U+2FCF->U+9F20, \
U+2FD0->U+9F3B, U+2FD1->U+9F4A, U+2FD2->U+9F52, U+2FD3->U+9F8D, U+2FD4->U+9F9C, U+2FD5->U+9FA0, U+3042->U+3041, U+3044->U+3043, U+3046->U+3045, U+3048->U+3047, U+304A->U+3049, U+304C->U+304B, U+304E->U+304D, U+3050->U+304F, U+3052->U+3051, U+3054->U+3053, \
U+3056->U+3055, U+3058->U+3057, U+305A->U+3059, U+305C->U+305B, U+305E->U+305D, U+3060->U+305F, U+3062->U+3061, U+3064->U+3063, U+3065->U+3063, U+3067->U+3066, U+3069->U+3068, U+3070->U+306F, U+3071->U+306F, U+3073->U+3072, U+3074->U+3072, U+3076->U+3075, \
U+3077->U+3075, U+3079->U+3078, U+307A->U+3078, U+307C->U+307B, U+307D->U+307B, U+3084->U+3083, U+3086->U+3085, U+3088->U+3087, U+308F->U+308E, U+3094->U+3046, U+3095->U+304B, U+3096->U+3051, U+30A2->U+30A1, U+30A4->U+30A3, U+30A6->U+30A5, U+30A8->U+30A7, \
U+30AA->U+30A9, U+30AC->U+30AB, U+30AE->U+30AD, U+30B0->U+30AF, U+30B2->U+30B1, U+30B4->U+30B3, U+30B6->U+30B5, U+30B8->U+30B7, U+30BA->U+30B9, U+30BC->U+30BB, U+30BE->U+30BD, U+30C0->U+30BF, U+30C2->U+30C1, U+30C5->U+30C4, U+30C7->U+30C6, U+30C9->U+30C8, \
U+30D0->U+30CF, U+30D1->U+30CF, U+30D3->U+30D2, U+30D4->U+30D2, U+30D6->U+30D5, U+30D7->U+30D5, U+30D9->U+30D8, U+30DA->U+30D8, U+30DC->U+30DB, U+30DD->U+30DB, U+30E4->U+30E3, U+30E6->U+30E5, U+30E8->U+30E7, U+30EF->U+30EE, U+30F4->U+30A6, U+30AB->U+30F5, \
U+30B1->U+30F6, U+30F7->U+30EF, U+30F8->U+30F0, U+30F9->U+30F1, U+30FA->U+30F2, U+30AF->U+31F0, U+30B7->U+31F1, U+30B9->U+31F2, U+30C8->U+31F3, U+30CC->U+31F4, U+30CF->U+31F5, U+30D2->U+31F6, U+30D5->U+31F7, U+30D8->U+31F8, U+30DB->U+31F9, U+30E0->U+31FA, \
U+30E9->U+31FB, U+30EA->U+31FC, U+30EB->U+31FD, U+30EC->U+31FE, U+30ED->U+31FF, U+FF66->U+30F2, U+FF67->U+30A1, U+FF68->U+30A3, U+FF69->U+30A5, U+FF6A->U+30A7, U+FF6B->U+30A9, U+FF6C->U+30E3, U+FF6D->U+30E5, U+FF6E->U+30E7, U+FF6F->U+30C3, U+FF71->U+30A1, \
U+FF72->U+30A3, U+FF73->U+30A5, U+FF74->U+30A7, U+FF75->U+30A9, U+FF76->U+30AB, U+FF77->U+30AD, U+FF78->U+30AF, U+FF79->U+30B1, U+FF7A->U+30B3, U+FF7B->U+30B5, U+FF7C->U+30B7, U+FF7D->U+30B9, U+FF7E->U+30BB, U+FF7F->U+30BD, U+FF80->U+30BF, U+FF81->U+30C1, \
U+FF82->U+30C3, U+FF83->U+30C6, U+FF84->U+30C8, U+FF85->U+30CA, U+FF86->U+30CB, U+FF87->U+30CC, U+FF88->U+30CD, U+FF89->U+30CE, U+FF8A->U+30CF, U+FF8B->U+30D2, U+FF8C->U+30D5, U+FF8D->U+30D8, U+FF8E->U+30DB, U+FF8F->U+30DE, U+FF90->U+30DF, U+FF91->U+30E0, \
U+FF92->U+30E1, U+FF93->U+30E2, U+FF94->U+30E3, U+FF95->U+30E5, U+FF96->U+30E7, U+FF97->U+30E9, U+FF98->U+30EA, U+FF99->U+30EB, U+FF9A->U+30EC, U+FF9B->U+30ED, U+FF9C->U+30EF, U+FF9D->U+30F3, U+FFA0->U+3164, U+FFA1->U+3131, U+FFA2->U+3132, U+FFA3->U+3133, \
U+FFA4->U+3134, U+FFA5->U+3135, U+FFA6->U+3136, U+FFA7->U+3137, U+FFA8->U+3138, U+FFA9->U+3139, U+FFAA->U+313A, U+FFAB->U+313B, U+FFAC->U+313C, U+FFAD->U+313D, U+FFAE->U+313E, U+FFAF->U+313F, U+FFB0->U+3140, U+FFB1->U+3141, U+FFB2->U+3142, U+FFB3->U+3143, \
U+FFB4->U+3144, U+FFB5->U+3145, U+FFB6->U+3146, U+FFB7->U+3147, U+FFB8->U+3148, U+FFB9->U+3149, U+FFBA->U+314A, U+FFBB->U+314B, U+FFBC->U+314C, U+FFBD->U+314D, U+FFBE->U+314E, U+FFC2->U+314F, U+FFC3->U+3150, U+FFC4->U+3151, U+FFC5->U+3152, U+FFC6->U+3153, \
U+FFC7->U+3154, U+FFCA->U+3155, U+FFCB->U+3156, U+FFCC->U+3157, U+FFCD->U+3158, U+FFCE->U+3159, U+FFCF->U+315A, U+FFD2->U+315B, U+FFD3->U+315C, U+FFD4->U+315D, U+FFD5->U+315E, U+FFD6->U+315F, U+FFD7->U+3160, U+FFDA->U+3161, U+FFDB->U+3162, U+FFDC->U+3163, \
U+3131->U+1100, U+3132->U+1101, U+3133->U+11AA, U+3134->U+1102, U+3135->U+11AC, U+3136->U+11AD, U+3137->U+1103, U+3138->U+1104, U+3139->U+1105, U+313A->U+11B0, U+313B->U+11B1, U+313C->U+11B2, U+313D->U+11B3, U+313E->U+11B4, U+313F->U+11B5, U+3140->U+111A, \
U+3141->U+1106, U+3142->U+1107, U+3143->U+1108, U+3144->U+1121, U+3145->U+1109, U+3146->U+110A, U+3147->U+110B, U+3148->U+110C, U+3149->U+110D, U+314A->U+110E, U+314B->U+110F, U+314C->U+1110, U+314D->U+1111, U+314E->U+1112, U+314F->U+1161, U+3150->U+1162, \
U+3151->U+1163, U+3152->U+1164, U+3153->U+1165, U+3154->U+1166, U+3155->U+1167, U+3156->U+1168, U+3157->U+1169, U+3158->U+116A, U+3159->U+116B, U+315A->U+116C, U+315B->U+116D, U+315C->U+116E, U+315D->U+116F, U+315E->U+1170, U+315F->U+1171, U+3160->U+1172, \
U+3161->U+1173, U+3162->U+1174, U+3163->U+1175, U+3165->U+1114, U+3166->U+1115, U+3167->U+11C7, U+3168->U+11C8, U+3169->U+11CC, U+316A->U+11CE, U+316B->U+11D3, U+316C->U+11D7, U+316D->U+11D9, U+316E->U+111C, U+316F->U+11DD, U+3170->U+11DF, U+3171->U+111D, \
U+3172->U+111E, U+3173->U+1120, U+3174->U+1122, U+3175->U+1123, U+3176->U+1127, U+3177->U+1129, U+3178->U+112B, U+3179->U+112C, U+317A->U+112D, U+317B->U+112E, U+317C->U+112F, U+317D->U+1132, U+317E->U+1136, U+317F->U+1140, U+3180->U+1147, U+3181->U+114C, \
U+3182->U+11F1, U+3183->U+11F2, U+3184->U+1157, U+3185->U+1158, U+3186->U+1159, U+3187->U+1184, U+3188->U+1185, U+3189->U+1188, U+318A->U+1191, U+318B->U+1192, U+318C->U+1194, U+318D->U+119E, U+318E->U+11A1, U+A490->U+A408, U+A491->U+A1B9, U+4E00..U+9FBB, \
U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, \
U+3059, U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, \
U+30BD, U+30BF, U+30C1, U+30C3, U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, U+31F9, \
U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, U+A492..U+A4C6, U+2C80->U+2C81, U+2C81, U+2C82->U+2C83, U+2C83, U+2C84->U+2C85, U+2C85, U+2C86->U+2C87, U+2C87, U+2C88->U+2C89, \
U+2C89, U+2C8A->U+2C8B, U+2C8B, U+2C8C->U+2C8D, U+2C8D, U+2C8E->U+2C8F, U+2C8F, U+2C90->U+2C91, U+2C91, U+2C92->U+2C93, U+2C93, U+2C94->U+2C95, U+2C95, U+2C96->U+2C97, U+2C97, U+2C98->U+2C99, U+2C99, U+2C9A->U+2C9B, U+2C9B, U+2C9C->U+2C9D, U+2C9D, U+2C9E->U+2C9F, \
U+2C9F, U+2CA0->U+2CA1, U+2CA1, U+2CA2->U+2CA3, U+2CA3, U+2CA4->U+2CA5, U+2CA5, U+2CA6->U+2CA7, U+2CA7, U+2CA8->U+2CA9, U+2CA9, U+2CAA->U+2CAB, U+2CAB, U+2CAC->U+2CAD, U+2CAD, U+2CAE->U+2CAF, U+2CAF, U+2CB0->U+2CB1, U+2CB1, U+2CB2->U+2CB3, U+2CB3, U+2CB4->U+2CB5, \
U+2CB5, U+2CB6->U+2CB7, U+2CB7, U+2CB8->U+2CB9, U+2CB9, U+2CBA->U+2CBB, U+2CBB, U+2CBC->U+2CBD, U+2CBD, U+2CBE->U+2CBF, U+2CBF, U+2CC0->U+2CC1, U+2CC1, U+2CC2->U+2CC3, U+2CC3, U+2CC4->U+2CC5, U+2CC5, U+2CC6->U+2CC7, U+2CC7, U+2CC8->U+2CC9, U+2CC9, U+2CCA->U+2CCB, \
U+2CCB, U+2CCC->U+2CCD, U+2CCD, U+2CCE->U+2CCF, U+2CCF, U+2CD0->U+2CD1, U+2CD1, U+2CD2->U+2CD3, U+2CD3, U+2CD4->U+2CD5, U+2CD5, U+2CD6->U+2CD7, U+2CD7, U+2CD8->U+2CD9, U+2CD9, U+2CDA->U+2CDB, U+2CDB, U+2CDC->U+2CDD, U+2CDD, U+2CDE->U+2CDF, U+2CDF, U+2CE0->U+2CE1, \
U+2CE1, U+2CE2->U+2CE3, U+2CE3, U+0400->U+0435, U+0401->U+0435, U+0402->U+0452, U+0452, U+0403->U+0433, U+0404->U+0454, U+0454, U+0405->U+0455, U+0455, U+0406->U+0456, U+0407->U+0456, U+0457->U+0456, U+0456, U+0408..U+040B->U+0458..U+045B, U+0458..U+045B, \
U+040C->U+043A, U+040D->U+0438, U+040E->U+0443, U+040F->U+045F, U+045F, U+0450->U+0435, U+0451->U+0435, U+0453->U+0433, U+045C->U+043A, U+045D->U+0438, U+045E->U+0443, U+0460->U+0461, U+0461, U+0462->U+0463, U+0463, U+0464->U+0465, U+0465, U+0466->U+0467, \
U+0467, U+0468->U+0469, U+0469, U+046A->U+046B, U+046B, U+046C->U+046D, U+046D, U+046E->U+046F, U+046F, U+0470->U+0471, U+0471, U+0472->U+0473, U+0473, U+0474->U+0475, U+0476->U+0475, U+0477->U+0475, U+0475, U+0478->U+0479, U+0479, U+047A->U+047B, U+047B, \
U+047C->U+047D, U+047D, U+047E->U+047F, U+047F, U+0480->U+0481, U+0481, U+048A->U+0438, U+048B->U+0438, U+048C->U+044C, U+048D->U+044C, U+048E->U+0440, U+048F->U+0440, U+0490->U+0433, U+0491->U+0433, U+0490->U+0433, U+0491->U+0433, U+0492->U+0433, U+0493->U+0433, \
U+0494->U+0433, U+0495->U+0433, U+0496->U+0436, U+0497->U+0436, U+0498->U+0437, U+0499->U+0437, U+049A->U+043A, U+049B->U+043A, U+049C->U+043A, U+049D->U+043A, U+049E->U+043A, U+049F->U+043A, U+04A0->U+043A, U+04A1->U+043A, U+04A2->U+043D, U+04A3->U+043D, \
U+04A4->U+043D, U+04A5->U+043D, U+04A6->U+043F, U+04A7->U+043F, U+04A8->U+04A9, U+04A9, U+04AA->U+0441, U+04AB->U+0441, U+04AC->U+0442, U+04AD->U+0442, U+04AE->U+0443, U+04AF->U+0443, U+04B0->U+0443, U+04B1->U+0443, U+04B2->U+0445, U+04B3->U+0445, U+04B4->U+04B5, \
U+04B5, U+04B6->U+0447, U+04B7->U+0447, U+04B8->U+0447, U+04B9->U+0447, U+04BA->U+04BB, U+04BB, U+04BC->U+04BD, U+04BE->U+04BD, U+04BF->U+04BD, U+04BD, U+04C0->U+04CF, U+04CF, U+04C1->U+0436, U+04C2->U+0436, U+04C3->U+043A, U+04C4->U+043A, U+04C5->U+043B, \
U+04C6->U+043B, U+04C7->U+043D, U+04C8->U+043D, U+04C9->U+043D, U+04CA->U+043D, U+04CB->U+0447, U+04CC->U+0447, U+04CD->U+043C, U+04CE->U+043C, U+04D0->U+0430, U+04D1->U+0430, U+04D2->U+0430, U+04D3->U+0430, U+04D4->U+00E6, U+04D5->U+00E6, U+04D6->U+0435, \
U+04D7->U+0435, U+04D8->U+04D9, U+04DA->U+04D9, U+04DB->U+04D9, U+04D9, U+04DC->U+0436, U+04DD->U+0436, U+04DE->U+0437, U+04DF->U+0437, U+04E0->U+04E1, U+04E1, U+04E2->U+0438, U+04E3->U+0438, U+04E4->U+0438, U+04E5->U+0438, U+04E6->U+043E, U+04E7->U+043E, \
U+04E8->U+043E, U+04E9->U+043E, U+04EA->U+043E, U+04EB->U+043E, U+04EC->U+044D, U+04ED->U+044D, U+04EE->U+0443, U+04EF->U+0443, U+04F0->U+0443, U+04F1->U+0443, U+04F2->U+0443, U+04F3->U+0443, U+04F4->U+0447, U+04F5->U+0447, U+04F6->U+0433, U+04F7->U+0433, \
U+04F8->U+044B, U+04F9->U+044B, U+04FA->U+0433, U+04FB->U+0433, U+04FC->U+0445, U+04FD->U+0445, U+04FE->U+0445, U+04FF->U+0445, U+0410..U+0418->U+0430..U+0438, U+0419->U+0438, U+0430..U+0438, U+041A..U+042F->U+043A..U+044F, U+043A..U+044F, U+0929->U+0928, \
U+0931->U+0930, U+0934->U+0933, U+0958->U+0915, U+0959->U+0916, U+095A->U+0917, U+095B->U+091C, U+095C->U+0921, U+095D->U+0922, U+095E->U+092B, U+095F->U+092F, U+0904..U+0928, U+092A..U+0930, U+0932, U+0933, U+0935..U+0939, U+0960, U+0961, U+0966..U+096F, \
U+097B..U+097F, U+10FC->U+10DC, U+10D0..U+10FA, U+10A0..U+10C5->U+2D00..U+2D25, U+2D00..U+2D25, U+0386->U+03B1, U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF, U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9, U+03AB->U+03C5, \
U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7, U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5, U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2, U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5, U+03D5->U+03C6, \
U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB, U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3, U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB, U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1, U+03F2->U+03C3, \
U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5, U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1, U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1, U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1, U+1F09->U+03B1, \
U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1, U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5, U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5, U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5, U+1F1B->U+03B5, \
U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7, U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7, U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7, U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7, U+1F2D->U+03B7, \
U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9, U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9, U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9, U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9, U+1F3D->U+03B9, \
U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF, U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF, U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF, U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5, U+1F51->U+03C5, \
U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5, U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5, U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9, U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9, U+1F65->U+03C9, \
U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9, U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9, U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1, U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7, U+1F75->U+03B7, \
U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF, U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9, U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1, U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1, U+1F87->U+03B1, \
U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1, U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1, U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7, U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7, U+1F97->U+03B7, \
U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7, U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7, U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9, U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9, U+1FA7->U+03C9, \
U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9, U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9, U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1, U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1, U+1FB8->U+03B1, \
U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1, U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7, U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5, U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9, U+1FD1->U+03B9, \
U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9, U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9, U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5, U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5, U+1FE7->U+03C5, \
U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5, U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9, U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF, U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9, U+0391..U+03A1->U+03B1..U+03C1, \
U+03B1..U+03C1, U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB, U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED, U+03EF, U+03F3, U+03F8, U+03FB, U+0A85..U+0A8C, U+0A8F, U+0A90, U+0A93..U+0AB0, U+0AB2, U+0AB3, \
U+0AB5..U+0AB9, U+0AE0, U+0AE1, U+0AE6..U+0AEF, U+0A33->U+0A32, U+0A36->U+0A38, U+0A59->U+0A16, U+0A5A->U+0A17, U+0A5B->U+0A1C, U+0A5E->U+0A2B, U+0A05..U+0A0A, U+0A0F, U+0A10, U+0A13..U+0A28, U+0A2A..U+0A30, U+0A32, U+0A35, U+0A38, U+0A39, U+0A5C, U+0A66..U+0A6F, \
U+FB1D->U+05D9, U+FB1F->U+05F2, U+FB20->U+05E2, U+FB21->U+05D0, U+FB22->U+05D3, U+FB23->U+05D4, U+FB24->U+05DB, U+FB25->U+05DC, U+FB26->U+05DD, U+FB27->U+05E8, U+FB28->U+05EA, U+FB2A->U+05E9, U+FB2B->U+05E9, U+FB2C->U+05E9, U+FB2D->U+05E9, U+FB2E->U+05D0, \
U+FB2F->U+05D0, U+FB30->U+05D0, U+FB31->U+05D1, U+FB32->U+05D2, U+FB33->U+05D3, U+FB34->U+05D4, U+FB35->U+05D5, U+FB36->U+05D6, U+FB38->U+05D8, U+FB39->U+05D9, U+FB3A->U+05DA, U+FB3B->U+05DB, U+FB3C->U+05DC, U+FB3E->U+05DE, U+FB40->U+05E0, U+FB41->U+05E1, \
U+FB43->U+05E3, U+FB44->U+05E4, U+FB46->U+05E6, U+FB47->U+05E7, U+FB48->U+05E8, U+FB49->U+05E9, U+FB4A->U+05EA, U+FB4B->U+05D5, U+FB4C->U+05D1, U+FB4D->U+05DB, U+FB4E->U+05E4, U+FB4F->U+05D0, U+05D0..U+05F2, U+0C85..U+0C8C, U+0C8E..U+0C90, U+0C92..U+0CA8, \
U+0CAA..U+0CB3, U+0CB5..U+0CB9, U+0CE0, U+0CE1, U+0CE6..U+0CEF, U+1900..U+191C, U+1930..U+1938, U+1946..U+194F, U+0D05..U+0D0C, U+0D0E..U+0D10, U+0D12..U+0D28, U+0D2A..U+0D39, U+0D60, U+0D61, U+0D66..U+0D6F, U+0B94->U+0B92, U+0B85..U+0B8A, U+0B8E..U+0B90, \
U+0B92, U+0B93, U+0B95, U+0B99, U+0B9A, U+0B9C, U+0B9E, U+0B9F, U+0BA3, U+0BA4, U+0BA8..U+0BAA, U+0BAE..U+0BB9, U+0BE6..U+0BEF, U+0E01..U+0E30, U+0E32, U+0E33, U+0E40..U+0E46, U+0E50..U+0E5B, U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z, U+FF41..U+FF5A->a..z, \
0..9, A..Z->a..z, a..z
# ignored characters list
# optional, default value is empty
#
# ignore_chars = U+00AD
# minimum word prefix length to index
# optional, default is 0 (do not index prefixes)
#
# min_prefix_len = 0
# minimum word infix length to index
# optional, default is 0 (do not index infixes)
#
# min_infix_len = 0
# list of fields to limit prefix/infix indexing to
# optional, default value is empty (index all fields in prefix/infix mode)
#
# prefix_fields = filename
# infix_fields = url, domain
# enable star-syntax (wildcards) when searching prefix/infix indexes
# search-time only, does not affect indexing, can be 0 or 1
# optional, default is 0 (do not use wildcard syntax)
#
# enable_star = 1
# expand keywords with exact forms and/or stars when searching fit indexes
# search-time only, does not affect indexing, can be 0 or 1
# optional, default is 0 (do not expand keywords)
#
# expand_keywords = 1
# n-gram length to index, for CJK indexing
# only supports 0 and 1 for now, other lengths to be implemented
# optional, default is 0 (disable n-grams)
#
ngram_len = 1
# n-gram characters list, for CJK indexing
# optional, default is empty
#
ngram_chars = U+4E00..U+9FBB, U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, U+3059, U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, U+30BD, U+30BF, U+30C1, U+30C3, U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, U+31F9, U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, U+A492..U+A4C6
# phrase boundary characters list
# optional, default is empty
#
# phrase_boundary = ., ?, !, U+2026 # horizontal ellipsis
# phrase boundary word position increment
# optional, default is 0
#
# phrase_boundary_step = 100
# blended characters list
# blended chars are indexed both as separators and valid characters
# for instance, AT&T will results in 3 tokens ("at", "t", and "at&t")
# optional, default is empty
#
# blend_chars = +, &, U+23
# blended token indexing mode
# a comma separated list of blended token indexing variants
# known variants are trim_none, trim_head, trim_tail, trim_both, skip_pure
# optional, default is trim_none
#
# blend_mode = trim_tail, skip_pure
# whether to strip HTML tags from incoming documents
# known values are 0 (do not strip) and 1 (do strip)
# optional, default is 0
html_strip = 0
# what HTML attributes to index if stripping HTML
# optional, default is empty (do not index anything)
#
# html_index_attrs = img=alt,title; a=title;
# what HTML elements contents to strip
# optional, default is empty (do not strip element contents)
#
# html_remove_elements = style, script
# whether to preopen index data files on startup
# optional, default is 0 (do not preopen), searchd-only
#
# preopen = 1
# whether to keep dictionary (.spi) on disk, or cache it in RAM
# optional, default is 0 (cache in RAM), searchd-only
#
# ondisk_dict = 1
# whether to enable in-place inversion (2x less disk, 90-95%% speed)
# optional, default is 0 (use separate temporary files), indexer-only
#
# inplace_enable = 1
# in-place fine-tuning options
# optional, defaults are listed below
#
# inplace_hit_gap = 0 # preallocated hitlist gap size
# inplace_docinfo_gap = 0 # preallocated docinfo gap size
# inplace_reloc_factor = 0.1 # relocation buffer size within arena
# inplace_write_factor = 0.1 # write buffer size within arena
# whether to index original keywords along with stemmed versions
# enables "=exactform" operator to work
# optional, default is 0
#
# index_exact_words = 1
# position increment on overshort (less that min_word_len) words
# optional, allowed values are 0 and 1, default is 1
#
# overshort_step = 1
# position increment on stopword
# optional, allowed values are 0 and 1, default is 1
#
# stopword_step = 1
# hitless words list
# positions for these keywords will not be stored in the index
# optional, allowed values are 'all', or a list file name
#
# hitless_words = all
# hitless_words = hitless.txt
# detect and index sentence and paragraph boundaries
# required for the SENTENCE and PARAGRAPH operators to work
# optional, allowed values are 0 and 1, default is 0
#
# index_sp = 1
# index zones, delimited by HTML/XML tags
# a comma separated list of tags and wildcards
# required for the ZONE operator to work
# optional, default is empty string (do not index zones)
#
# index_zones = title, h*, th
}
#############################################################################
## searchd settings
#############################################################################
searchd
{
# [hostname:]port[:protocol], or /unix/socket/path to listen on
# known protocols are 'sphinx' (SphinxAPI) and 'mysql41' (SphinxQL)
#
# multi-value, multiple listen points are allowed
# optional, defaults are 9312:sphinx and 9306:mysql41, as below
#
# listen = 127.0.0.1
# listen = 192.168.0.1:9312
# listen = 9312
# listen = /var/run/searchd.sock
listen = %(ip_address)s:%(port)s:sphinx
listen = %(ip_address)s:%(sql_port)s:mysql41
# log file, searchd run info is logged here
# optional, default is 'searchd.log'
log = %(searchd_log)s
# query log file, all search queries are logged here
# optional, default is empty (do not log queries)
query_log = %(query_log)s
# client read timeout, seconds
# optional, default is 5
read_timeout = 5
# request timeout, seconds
# optional, default is 5 minutes
client_timeout = 300
# maximum amount of children to fork (concurrent searches to run)
# optional, default is 0 (unlimited)
max_children = 30
# PID file, searchd process ID file name
# mandatory
pid_file = %(pid)s
# max amount of matches the daemon ever keeps in RAM, per-index
# WARNING, THERE'S ALSO PER-QUERY LIMIT, SEE SetLimits() API CALL
# default is 1000 (just like Google)
max_matches = 1000
# seamless rotate, prevents rotate stalls if precaching huge datasets
# optional, default is 1
seamless_rotate = 1
# whether to forcibly preopen all indexes on startup
# optional, default is 0 (do not preopen)
preopen_indexes = 0
# whether to unlink .old index copies on succesful rotation.
# optional, default is 1 (do unlink)
unlink_old = 1
# attribute updates periodic flush timeout, seconds
# updates will be automatically dumped to disk this frequently
# optional, default is 0 (disable periodic flush)
#
# attr_flush_period = 900
# instance-wide ondisk_dict defaults (per-index value take precedence)
# optional, default is 0 (precache all dictionaries in RAM)
#
# ondisk_dict_default = 1
# MVA updates pool size
# shared between all instances of searchd, disables attr flushes!
# optional, default size is 1M
mva_updates_pool = 1M
# max allowed network packet size
# limits both query packets from clients, and responses from agents
# optional, default size is 8M
max_packet_size = 8M
# crash log path
# searchd will (try to) log crashed query to 'crash_log_path.PID' file
# optional, default is empty (do not create crash logs)
#
# crash_log_path = (log_directory)
# max allowed per-query filter count
# optional, default is 256
max_filters = 256
# max allowed per-filter values count
# optional, default is 4096
max_filter_values = 4096
# socket listen queue length
# optional, default is 5
#
# listen_backlog = 5
# per-keyword read buffer size
# optional, default is 256K
#
# read_buffer = 256K
# unhinted read size (currently used when reading hits)
# optional, default is 32K
#
# read_unhinted = 32K
# max allowed per-batch query count (aka multi-query count)
# optional, default is 32
max_batch_queries = 32
# max common subtree document cache size, per-query
# optional, default is 0 (disable subtree optimization)
#
# subtree_docs_cache = 4M
# max common subtree hit cache size, per-query
# optional, default is 0 (disable subtree optimization)
#
# subtree_hits_cache = 8M
# multi-processing mode (MPM)
# known values are none, fork, prefork, and threads
# optional, default is fork
#
workers = threads # for RT to work
# max threads to create for searching local parts of a distributed index
# optional, default is 0, which means disable multi-threaded searching
# should work with all MPMs (ie. does NOT require workers=threads)
#
# dist_threads = 4
# binlog files path; use empty string to disable binlog
# optional, default is build-time configured data directory
#
binlog_path = # disable logging
# binlog_path = %(data_directory)s # binlog.001 etc will be created there
# binlog flush/sync mode
# 0 means flush and sync every second
# 1 means flush and sync every transaction
# 2 means flush every transaction, sync every second
# optional, default is 2
#
# binlog_flush = 2
# binlog per-file size limit
# optional, default is 128M, 0 means no limit
#
# binlog_max_log_size = 256M
}
import os
import shutil
import tempfile
import textwrap
import time
import unittest
......@@ -54,7 +53,7 @@ class PostgresTest(unittest.TestCase):
self.addCleanup(server_process.terminate)
# wait for server to accept connections
for i in range(60):
for i in range(10):
time.sleep(i)
try:
psycopg2.connect(self.buildout['postgres']['url']).close()
......@@ -114,3 +113,7 @@ class PostgresTest(unittest.TestCase):
class PostgresTestNonStandardPort(PostgresTest):
port = 5433
class PostgresTestEmptyPort(PostgresTest):
port = ''
......@@ -89,6 +89,35 @@ class RecipeTestMixin(object):
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='started')
def test_requester_stopped_state_propagated(self):
options = defaultdict(str)
options['return'] = 'anything'
self.buildout['slap-connection']['requested'] = 'stopped'
self.instance_getConnectionParameter.return_value = self.return_value_empty
with LogCapture() as log:
self.recipe(self.buildout, "request", options)
log.check()
self.request_instance.assert_called_with(
'', 'RootSoftwareInstance', '', filter_kw={},
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='stopped')
def test_requester_destroyed_state_not_propagated(self):
options = defaultdict(str)
options['return'] = 'anything'
self.buildout['slap-connection']['requested'] = 'destroyed'
self.instance_getConnectionParameter.return_value = self.return_value_empty
with LogCapture() as log:
self.recipe(self.buildout, "request", options)
log.check()
self.request_instance.assert_called_with(
'', 'RootSoftwareInstance', '', filter_kw={},
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='started')
class RecipeTest(RecipeTestMixin, unittest.TestCase):
recipe = request.Recipe
......
......@@ -63,7 +63,6 @@ eggs =
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-common.cfg.in
rendered = ${buildout:directory}/instance-common.cfg
mode = 0644
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
......@@ -129,29 +128,24 @@ htpasswd = ${:bin_directory}/htpasswd
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
section software_parameter_dict software-parameter-section
[profile-caddy-frontend]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
mode = 0644
[profile-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
mode = 0644
[profile-kedifa]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-kedifa.cfg.in
mode = 0644
[download-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
[profile-slave-list]
<=download-template
......@@ -181,7 +175,6 @@ mode = 640
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[template-trafficserver-records-config]
<=download-template
......
......@@ -10,8 +10,6 @@ parts +=
[instance-caucased]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
# XXX: following mode should be the default
mode = 644
[instance]
recipe = slapos.recipe.template:jinja2
......
......@@ -97,7 +97,6 @@ template-monitor = ${monitor2-template:rendered}
[template-cloudooo-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 640
[versions]
argparse = 1.4.0
......
......@@ -217,7 +217,7 @@
},
"timerserver-interval": {
"description": "Timerserver tick period, in seconds, or 0 to disable",
"default": 5,
"default": 1,
"type": "number"
},
"private-dev-shm": {
......
......@@ -5,6 +5,7 @@
"software-type": {
"default": {
"title": "Default",
"software-type": "default",
"description": "No automated database modification (ERP5Site is not automatically created).",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
......
......@@ -5,9 +5,11 @@
"software-type": {
"default": {
"title": "Default",
"software-type": "default",
"description": "Default setup for ERP5TestNode Instance.",
"request": "instance-erp5testnode-input-schema.json",
"response": "instance-output-schema.json"
"response": "instance-output-schema.json",
"index": 0
}
}
}
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-erp5testnode]
filename = instance.cfg.in
md5sum = bbfe2f0e83df4d6cd2120c0ef3c483cd
......@@ -14,7 +14,7 @@
# not need these here).
[deploy-script-controller-script]
filename = deploy-script-controller
md5sum = 8288e59eb442c662544daffbf446a033
md5sum = 8357771b70efd0740561b1cb46f6955e
[template-deploy-test]
filename = instance.cfg.in
......
......@@ -83,8 +83,6 @@ function upload ()
add_checks $LOG_FILE
t=`date '+%Y%m%d%H%S'`
mv $LOG_FILE ${LOG_FILE}.$t
# just to be sure flush all disk operations before uploading
flush
curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
}
......
......@@ -10,7 +10,7 @@ test = dynamic-template-resilient-test:rendered
[dynamic-template-resilient-test]
recipe = slapos.recipe.template:jinja2
template = ${template-resilient-test:location}/${template-resilient-test:filename}
template = ${template-resilient-test:target}
rendered = $${buildout:directory}/template-resilient-test.cfg
bin-directory = ${buildout:bin-directory}
context =
......@@ -19,4 +19,3 @@ context =
key slapparameter_dict slap-configuration:configuration
raw bin_directory ${buildout:bin-directory}
${template-resilient-test:extra-context}
mode = 0644
......@@ -6,14 +6,9 @@ extends =
parts += template-erp5testnode
[default-test-image]
recipe = hexagonit.recipe.download
ignore-existing = true
filename = ${:_buildout_section_name_}
recipe = slapos.recipe.build:download
url = http://www.nexedi.org/static/slapos/kvm_resiliency_test/virtual.qcow.gz
md5sum = dd82c771f6f7738fb4b0fc1330ed8236
download-only = true
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-resilient-test]
filename = instance-kvm-resilient-test.cfg.jinja2
......@@ -21,7 +16,7 @@ md5sum = 71ddbdeb8769bcb0ebb3c9407ef7e36c
# Ingest extra-context, on a the final template-resilient-test rendering
# always ingest raw values.
extra-context =
raw default_test_image_url file://${default-test-image:location}/${default-test-image:filename}
raw default_test_image_url file://${default-test-image:target}
raw default_test_image_md5sum ${default-test-image:md5sum}
[buildout]
extends = buildout.hash.cfg
[template-erp5testnode]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = d864a32edab3c4f7207a7d4fe6bb7e30
output = ${buildout:directory}/template.cfg
mode = 0644
[template]
output = ${buildout:directory}/template-original.cfg
[template-resilient-test]
recipe = hexagonit.recipe.download
ignore-existing = true
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
download-only = true
on-update = true
extra-context =
......@@ -397,10 +397,9 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
<= download-file
[gitlab-demo-backup.git]
recipe = hexagonit.recipe.download
recipe = slapos.recipe.build:download-unpacked
url = https://lab.nexedi.com/alain.takoudjou/labdemo.backup/repository/archive.tar.gz?ref=master
md5sum = d40e5e211dc9a4e5ada9c0250377c639
strip-top-level-dir = true
[versions]
cns.recipe.symlink = 0.2.3
......
......@@ -99,8 +99,6 @@ stop-on-error = true
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[influxdb-config-file]
<= download-file-base
......
......@@ -38,7 +38,6 @@ context =
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[instance-headless-chromium]
<= download-base
......
......@@ -32,7 +32,7 @@ return =
[proxy]
recipe = slapos.cookbook:wrapper
command-line = {{ python_executable }} {{ rina_proxy }} ${server:instance-guid} ${:ipv6} ${:port}
command-line = {{ rina_proxy }} ${server:instance-guid} ${:ipv6} ${:port}
wrapper-path = ${directory:service}/proxy
environment =
PATH={{ rina_tools_location }}/bin:%(PATH)s
......
......@@ -28,7 +28,6 @@ template = {{ instance_root }}
extra-context =
import urlparse urlparse
key ipv6 slap-configuration:ipv6-random
raw python_executable {{ python_executable }}
raw rina_proxy {{ rina_proxy }}
[server]
......
......@@ -6,37 +6,50 @@ parts =
slapos-cookbook
template
[file]
# For old GCC like 4.9.2 on Debian 8.
# XXX: This should be moved to component/rina-tools/buildout.cfg, next to where
# we force use of system GCC. However, our buildout patches are still not
# perfect concerning the processing of +=
environment +=
CFLAGS=-std=c99 -g -O2
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = f6c61225990986d94d0017b07b873aa7
md5sum = d7506e861ef87977eaa554b8928d2c99
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key python_executable python:executable
key rina_tools_location rina-tools:location
key instance_root instance-root:target
key instance_server instance-server:target
key rina_proxy proxy:target
key rina_proxy proxy:location
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
# XXX: following mode should be the default
mode = 644
[instance-root]
<= download-base
md5sum = f647054be67998f9eece174f106c4464
md5sum = 1c0e222aab51dfc598094e972f1d1482
[instance-server]
<= download-base
md5sum = 88a451b0f7f8def12713b92b91659b98
[proxy]
recipe = slapos.recipe.build:download
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:_buildout_section_name_}
url = ${:_profile_base_location_}/${:_buildout_section_name_}
mode = 755
md5sum = 78b77a6bda9958f547f7d89b747731e3
install =
import os, sys
with open(self.download(options['url'], options['md5sum'])) as src, \
open(options['location'], 'w') as dst:
os.fchmod(dst.fileno(), 0o755)
src.readline()
dst.write('#!%s\n' % sys.executable)
dst.write(src.read())
......@@ -28,7 +28,6 @@ rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
filename = instance.cfg.in
md5sum = 310aab063704794065ee3bc8f81fdc70
mode = 0644
context =
section buildout buildout
key nginx_location nginx:location
......@@ -44,28 +43,24 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = 9b7ed68551cac5967915979383238669
mode = 0644
[template_nginx_conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = templates/nginx_conf.in
md5sum = 61dc4c82bf48563228ce4dea6c5c6319
mode = 0644
[template_launcher]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = templates/launcher.in
md5sum = 6cb0d64905ae7fc67277c1bf76b86875
mode = 0644
[template_mime_types]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = templates/mime_types.in
md5sum = 4ef94a7b458d885cd79ba0b930a5727e
mode = 0644
[extra-eggs]
recipe = zc.recipe.egg
......
......@@ -29,7 +29,6 @@ parts =
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key nginx_location nginx:location
......@@ -50,7 +49,6 @@ context =
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
# Download instance_html5as.cfg.in
[instance_html5as]
......
......@@ -26,4 +26,4 @@ md5sum = dc8b8d03b0af9cd32398d1fe86267bb7
[template]
filename = instance.cfg.in
md5sum = 94fc13254c819cba33b03f30251bc469
md5sum = c5b18ea5d353f96c41294541186f2528
......@@ -18,6 +18,7 @@ context =
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:validator}
default = $${:validator}
validator = dynamic-template-validator:rendered
[slap-configuration]
......
......@@ -39,7 +39,6 @@ setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
class TestHtmlValidatorServer(InstanceTestCase):
def test(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
......@@ -68,3 +67,8 @@ class TestHtmlValidatorServer(InstanceTestCase):
[httplib.UNAUTHORIZED, False],
[result.status_code, result.is_redirect]
)
class TestHtmlValidatorServerInsideWebRunner(TestHtmlValidatorServer):
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
......@@ -29,7 +29,6 @@ template_monitor = ${monitor-template:rendered}
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
section parameter_list profile-common
......@@ -37,7 +36,6 @@ context =
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template_nginx_conf]
<= download-base
......
......@@ -36,13 +36,11 @@ update-command = ${:command}
[download-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[instance]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
key bash_location bash:location
key bin_directory buildout:bin-directory
......
......@@ -19,7 +19,7 @@ md5sum = de37ec3d4adb0be4c67bcc7397f27c91
[instance-jupyter]
filename = instance-jupyter.cfg.in
md5sum = 95e3da48abdd257fb9d5dbdf14ea87b9
md5sum = 1136bb108aa8d703e0ecf2005b4a1f28
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
......
......@@ -55,10 +55,6 @@ frontend-additional-instance-name = Jupyter Frontend Additional
{{ k }} = {{ v }}
{% endfor -%}
[dynamic-jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 0644
[generate-certificate]
; TODO: there is a slapos recipe to generate certificates. Use it instead
recipe = plone.recipe.command
......@@ -98,10 +94,9 @@ recipe = slapos.cookbook:generate.password
bytes = 10
[jupyter-notebook-config]
<= dynamic-jinja2-template-base
recipe = slapos.recipe.template:jinja2
template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
mode = 0744
context =
key password jupyter-password:passwd
raw gcc_location {{ gcc_location }}
......@@ -192,7 +187,7 @@ link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
recipe = slapos.recipe.template:jinja2
template = {{ kernel_json_location }}/{{ kernel_json_filename }}
rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
......
......@@ -25,8 +25,6 @@ python_executable = ${buildout:bin-directory}/${:interpreter}
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[jupyter-notebook-config]
<= download-file-base
......
......@@ -15,11 +15,11 @@
[template]
filename = instance.cfg.in
md5sum = f2b0f1ed27148504f220e06eaceff935
md5sum = 087f226ba90928dcc5a722d7008c867a
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 93cbee3403e7e23b4278143c32209ddc
md5sum = b260fce887535fc69a259d1fd31af1b2
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -51,7 +51,7 @@ md5sum = 4bcb07c1a9223e2d956651aa25d23654
[template-ansible-promise]
filename = template/ansible-promise.in
md5sum = b7e87479a289f472b634a046b44b5257
md5sum = a8cf453d20f01c707f02c4b4014580d8
[template-kvm-run]
filename = template/template-kvm-run.in
......@@ -78,17 +78,17 @@ filename = instance-kvm-http.cfg.in
md5sum = d57764bb7135037b4d21543b2f56ce1d
[image-download-controller]
_update_hash_filename_ = template/image-download-controller.py
filename = template/image-download-controller.py
md5sum = 9c67058edcc4edae0b57956c0932a9fc
[image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py
filename = template/image-download-config-creator.py
md5sum = 54261e418ab9860efe73efd514c4d47f
[whitelist-firewall-download-controller]
_update_hash_filename_ = template/whitelist-firewall-download-controller.py
filename = template/whitelist-firewall-download-controller.py
md5sum = bc64e29546833817636261d1b28aa6dc
[whitelist-domains-default]
_update_hash_filename_ = template/whitelist-domains-default
filename = template/whitelist-domains-default
md5sum = e9d40162ba77472775256637a2617d14
......@@ -84,7 +84,8 @@ storage-path = ${directory:srv}/tap_mac
[gen-passwd]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:srv}/passwd
storage-path = ${directory:srv}/.passwd
# VNC protocol supports passwords of 8 characters max
bytes = 8
{% if boot_image_url_select_enabled %}
......@@ -94,10 +95,7 @@ bytes = 8
promise = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename}
[boot-image-url-select-source-config]
recipe = slapos.recipe.template:jinja2
......@@ -193,10 +191,7 @@ config-filename = ${boot-image-url-select-download-wrapper:error-state-file}
promise = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename}
[boot-image-url-list-source-config]
recipe = slapos.recipe.template:jinja2
......@@ -292,10 +287,7 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
promise = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
......@@ -768,7 +760,7 @@ sla-instance_guid = ${slap-parameter:frontend-instance-guid}
<= monitor-promise-base
promise = check_url_available
name = frontend_promise.py
config-url = ${publish-connection-information:url}
config-url = ${request-slave-frontend:connection-secure_access}
{% if additional_frontend %}
[request-slave-frontend-additional]
......@@ -909,9 +901,9 @@ command =
ip a | grep eth0: && [ \$IFACE = ens3 ] && IFACE=eth0
ip a | grep eth1: && [ \$IFACE = ens4 ] && IFACE=eth1
${:ipv4-add-address}
${:ipv4-set-link-up}
${:ipv4-add-gateway-route}
${:ipv4-add-default-route}
${:ipv4-set-link-up}
EOF
update-command = ${:command}
{% endif -%}
......@@ -930,8 +922,8 @@ command =
#!/bin/sh
IFACE={{ iface }}
${:ipv6-add-address}
${:ipv6-add-default-route}
${:ipv6-set-link-up}
${:ipv6-add-default-route}
EOF
update-command = ${:command}
{% endif -%}
......
......@@ -67,7 +67,7 @@ filename = template-kvm-cluster.cfg
extra-context =
section parameter_dict dynamic-template-kvm-cluster-parameters
raw logrotate_cfg ${template-logrotate-base:rendered}
raw template_content ${template-content:location}/${template-content:filename}
raw template_content ${template-content:target}
raw template_httpd_cfg ${template-httpd:rendered}
raw template_monitor ${monitor2-template:rendered}
......@@ -77,13 +77,13 @@ template = ${template-kvm:location}/instance-kvm.cfg.jinja2
filename = template-kvm.cfg
extra-context =
section slap_configuration slap-configuration
raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
raw ansible_promise_tpl ${template-ansible-promise:target}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-bullseye-netinst.iso:target}
raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
raw whitelist_domains_default ${whitelist-domains-default:target}
raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
raw image_download_controller ${image-download-controller:target}
raw image_download_config_creator ${image-download-config-creator:target}
......@@ -94,12 +94,12 @@ extra-context =
raw python_eggs_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
raw qemu_executable_location ${qemu:location}/bin/qemu-system-x86_64
raw qemu_img_executable_location ${qemu:location}/bin/qemu-img
raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}
raw qemu_start_promise_tpl ${template-qemu-ready:target}
raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
raw template_httpd_cfg ${template-httpd:rendered}
raw template_content ${template-content:location}/${template-content:filename}
raw template_kvm_controller_run ${template-kvm-controller:location}/${template-kvm-controller:filename}
raw template_kvm_run ${template-kvm-run:location}/${template-kvm-run:filename}
raw template_content ${template-content:target}
raw template_kvm_controller_run ${template-kvm-controller:target}
raw template_kvm_run ${template-kvm-run:target}
raw template_monitor ${monitor2-template:rendered}
raw websockify_executable_location ${buildout:directory}/bin/websockify
raw wipe_disk_wrapper ${buildout:directory}/bin/securedelete
......@@ -134,7 +134,7 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
raw kvm_template $${dynamic-template-kvm:rendered}
raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
raw template_kvm_export ${template-kvm-export-script:target}
key pbsready_export_template template-pbsready-export:rendered
raw gzip_binary ${gzip:location}/bin/gzip
key slapparameter_dict slap-configuration:configuration
......@@ -149,7 +149,7 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
raw qemu_location ${qemu:location}
raw template_kvm_import ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
raw template_kvm_import ${template-kvm-import-script:target}
key pbsready_import_template template-pbsready-import:rendered
key slapparameter_dict slap-configuration:configuration
raw zcat_binary ${gzip:location}/bin/zcat
......
#!/usr/bin/env python
# Parse Ansible result log file and and generate a report
import os, time
import json
import sqlite3
from datetime import datetime, timedelta
import sys
FIELDS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout', 'stderr',
'response', 'status_code', 'url', 'dest']
class ansibleReport(object):
def __init__(self, db_path,
ansible_log_dir,
name):
self.db_path = db_path
self.ansible_log_dir = ansible_log_dir
self.name = name
self.result_OK = '127.0.0.1_OK'
self.result_failed = '127.0.0.1_FAILED'
self.result_failed_ignore = '127.0.0.1_FAILED_INGORED' # tipo in ansible log upload pluging
self.date_format = '%Y-%m-%d %H:%M:%S'
self.day_format = '%Y-%m-%d'
self._init_db()
def _init_db(self):
db = sqlite3.connect(self.db_path)
c = db.cursor()
c.executescript("""
CREATE TABLE IF NOT EXISTS ansible_report (
name VARCHAR(40),
reportdate VARCHAR(15),
createdate VARCHAR(15),
status VARCHAR(20),
success_count INTEGER,
ignored_count INTEGER,
failed_count INTEGER,
ignored TEXT,
failed TEXT,
success TEXT);
""")
db.commit()
db.close()
def connect_db(self):
db = sqlite3.connect(self.db_path)
return db
def insertEntryDb(self, table_name, data_dict):
db = self.connect_db()
columns = data_dict.keys()
entries = ', '.join(columns)
values = '?' + ', ?' * (len(columns)-1)
sql_string = "insert into %s(%s) values (%s)" % (
table_name, entries, values)
tuple_data = ()
for key in columns:
tuple_data += (data_dict[key],)
db.execute(sql_string, tuple_data)
db.commit()
db.close()
def selectEntriesDb(self, fields=[], start_date=None, limit=0, success=None, order='DESC', where=""):
db = self.connect_db()
entries = ', '.join(fields) if fields else '*'
query = "select %s from ansible_report " % entries
where = " and %s" % where if where else ""
if not start_date:
start_date = datetime.utcnow().strftime(self.day_format)
tuple_values = (start_date,)
if success is not None:
status = 'OK' if success else 'FAILLED'
query += "where createdate>=? and status=? %s order by createdate %s" % (where, order)
tuple_values += (status,)
else:
query += "where createdate>=? %s order by createdate %s" % (where, order)
if limit:
query += " limit ?"
tuple_values += (limit,)
rows = db.cursor().execute(query, tuple_values)
#db.close()
if rows:
return [list(row) for row in rows]
return []
def truncateEntriesDb(self, table_name, on_field, to_value, operator='<'):
db = self.connect_db()
query = "delete from %s where %s%s?" % (table_name, on_field,
operator)
db.execute(query, (to_value,))
db.commit()
db.close()
def getLogString(self, res, head=False):
log = ""
if type(res) == type(dict()):
log = '%s, args [%s]\n' % (res['invocation']['module_name'],
res['invocation']['module_args'])
if head:
return log
for field in FIELDS:
if field in res.keys():
# use default encoding, check out sys.setdefaultencoding
log += '\n{0}:\n{1}'.format(field, res[field])
return log
def _read_file(self, filepath):
content = '[]'
with open(filepath, 'r') as f:
content = f.read()
return content
def saveResult(self):
date = datetime.utcnow().strftime(self.date_format)
files_list = os.listdir(self.ansible_log_dir)
if not len(files_list):
return
to_date = (datetime.now() - timedelta(days=2)).strftime(self.date_format)
cmp_file = os.path.join(self.ansible_log_dir, files_list.pop())
modification_date = datetime.fromtimestamp(
os.path.getmtime(cmp_file)
).strftime(self.date_format)
# Get the latest insert date
result = self.selectEntriesDb(['reportdate'], start_date=to_date, limit=1)
if len(result):
latest_date = result[0][0]
if latest_date >= modification_date:
return
file_map = dict(
success_file=os.path.join(self.ansible_log_dir, self.result_OK),
failed_file=os.path.join(self.ansible_log_dir, self.result_failed),
ignored_file=os.path.join(self.ansible_log_dir, self.result_failed_ignore))
data = dict(name=self.name, status='FAILED',
reportdate=modification_date, createdate=date,
success_count=0, ignored_count=0,
failed_count=0, success="",
failed="", ignored="")
for category in ('failed', 'success', 'ignored'):
file_category = file_map['%s_file' % category]
if os.path.exists(file_category):
text_content = self._read_file(file_category)
count = len(json.loads(text_content))
if count > 0:
data['%s_count' % category] = count
data[category] = text_content
if data['failed_count'] == 0:
data['status'] = 'OK'
self.insertEntryDb('ansible_report', data)
def getAnsibleReport(self, start_date=None, limit=0, success=None, order='DESC', category=None, head=False, only_state=True):
"""Get one or many entries from the ansible report table.
"""
where = ""
get_content = category is not None
fields = ['name', 'reportdate', 'createdate', 'status', 'success_count',
'ignored_count', 'failed_count']
if category:
where = " %s_count>0" % category
if not only_state:
fields.append(category)
rows = self.selectEntriesDb(fields=fields, start_date=start_date,
limit=limit, success=success, order=order,
where=where)
result_dict = {}
if category and not only_state:
last_pos = len(fields) -1
for i in range (0, len(rows)):
message = ""
message_list = json.loads(rows[i][last_pos])
for msg in message_list:
message += '%s\n\n' % self.getLogString(msg, head=head)
rows[i][last_pos] = message
else:
return {}
return rows
if __name__ == "__main__":
json = """{
"status": "OK",
"message": "kvm-1: OK(114) FAILED(0) IGNORED(2)",
"description": "Ansible playbook report in kvm-1. Execution date is: 2015-08-28 17:42:01."
}"""
parameter_dict = json.loads(sys.argv[1])
with open(parameter_dict['status_path'], 'w') as status_file:
status_file.write(json)
\ No newline at end of file
......@@ -58,95 +58,55 @@ scripts =
# Create all templates that will be used to deploy instances
[download-base]
recipe = hexagonit.recipe.download
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[download-file-base]
<= download-base
ignore-existing = true
download-only = true
[download-template-base]
<= download-file-base
url = ${:_profile_base_location_}/template/${:path}
path = ${:filename}
[template-file-base]
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[template]
<= template-file-base
output = ${buildout:directory}/template.cfg
[template-kvm]
<= download-file-base
on-update = true
<= download-base
[template-kvm-cluster]
<= download-file-base
on-update = true
<= download-base
[template-kvm-resilient]
<= download-file-base
on-update = true
<= download-base
[template-kvm-import]
<= download-file-base
on-update = true
<= download-base
[template-kvm-import-script]
<= download-template-base
filename = kvm-import.sh.jinja2
mode = 0755
<= download-base
[template-kvm-export]
<= download-file-base
on-update = true
<= download-base
[template-kvm-export-script]
<= download-template-base
filename = kvm-export.sh.jinja2
mode = 0755
<= download-base
[template-nbd]
<= download-file-base
on-update = true
<= download-base
[template-ansible-promise]
<= download-template-base
filename = ansible-promise.in
<= download-base
[template-kvm-run]
<= download-template-base
filename = template-kvm-run.in
on-update = true
<= download-base
[template-kvm-controller]
<= download-template-base
filename = kvm-controller-run.in
on-update = true
<= download-base
[template-apache-conf]
<= download-template-base
filename = apache.conf.in
on-update = true
<= download-base
[template-content]
<= download-template-base
filename = template-content.in
on-update = true
<= download-base
[template-qemu-ready]
<= download-template-base
filename = qemu-is-ready.in
on-update = true
[whitelist-domains-default]
<= download-template-base
filename = whitelist-domains-default
<= download-base
[template-httpd]
recipe = slapos.recipe.template:jinja2
......@@ -155,24 +115,22 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/instance-kvm-
context =
key apache_location apache:location
raw openssl_executable_location ${openssl:location}/bin/openssl
raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename}
key template_apache_conf template-apache-conf:target
[image-download-controller]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
<= download-base
[image-download-config-creator]
<= image-download-controller
<= download-base
[whitelist-firewall-download-controller]
<= image-download-controller
<= download-base
[whitelist-domains-default]
<= download-base
[versions]
websockify = 0.9.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
pycurl = 7.43.0
smmap = 0.9.0
......@@ -17,8 +17,9 @@ result_failed_ignore = '127.0.0.1_FAILED_IGNORED'
def get_log(res):
log = ""
if type(res) == type(dict()):
log = '>> Running task: %s, args [%s]\n' % (res['invocation']['module_name'],
res['invocation']['module_args'])
module_name = res['invocation'].get('module_name', 'unkown_module_name')
module_args = res['invocation'].get('module_args', 'unkown_module_args')
log = '>> Running task: %s, args [%s]\n' % (module_name, module_args)
for field in FIELDS:
if field in res.keys():
# use default encoding, check out sys.setdefaultencoding
......
......@@ -1437,7 +1437,7 @@ class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
self.image_download_controller = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '0',
'software_release', 'parts', 'image-download-controller',
'image-download-controller')
'image-download-controller.py')
def tearDown(self):
self.stopImageHttpServer()
......
......@@ -15,7 +15,6 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template/REPLACE_ME_BY_APPLICATION_TEMPLATE_NAME
#md5sum = Student may put here md5sum of this file, this is good idea
filename = template.in
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[application-configuration]
......
......@@ -22,8 +22,8 @@ part = python3
[metabase.jar]
recipe = slapos.recipe.build:download
url = https://downloads.metabase.com/v0.41.2/metabase.jar
md5sum = 630068d1ccbdc95556931fe9cfc12e61
url = https://downloads.metabase.com/v0.41.4/metabase.jar
md5sum = 9b81838e5c40302b552c66df5a767f8e
[instance-profile]
recipe = slapos.recipe.template
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
md5sum = 37966f5f7c6b82137c0388ca3520cb71
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......
......@@ -25,7 +25,7 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:target}
mode = 0644
[instance-template]
......@@ -93,7 +93,7 @@ context = import json_module json
key template_surykatka_ini surykatka:ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:target}
mode = 0644
......
......@@ -15,7 +15,6 @@ parts =
template
template-monitor-edgetest
template-monitor
monitor-collect-csv-dump
[python]
part = python3
......@@ -24,27 +23,22 @@ part = python3
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[template-monitor]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest-basic]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgebot]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-surykatka-ini]
recipe = slapos.recipe.build:download
......@@ -53,19 +47,15 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
[json-test-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[network-bench-cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 0644
[monitor-collect-csv-dump]
<= monitor-template-script
url = ${:_profile_base_location_}/script/${:filename}
filename = collect_csv_dump.py
output = ${:destination}/${:filename}
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
[surykatka]
recipe = zc.recipe.egg
......@@ -76,7 +66,6 @@ recipe = zc.recipe.egg
eggs +=
slapos.cookbook
slapos.recipe.cmmi
hexagonit.recipe.download
plone.recipe.command
scripts =
......
......@@ -59,8 +59,6 @@ adapter-egg = ${python-mysqlclient:egg}
[download-base-neo]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
# XXX: following mode should be the default
mode = 644
# XXX: must be rendered, not just dled
[instance-common]
......@@ -127,7 +125,6 @@ template =
[versions]
coverage = 4.5.1
mock = 3.0.5
ecdsa = 0.13
mysqlclient = 1.3.12
pycrypto = 2.6.1
......
......@@ -4,26 +4,22 @@ extends =
../../component/redis/buildout.cfg
../../stack/lamp/buildout.cfg
[nc-download-base]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
[nc-download]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[application]
url = https://download.nextcloud.com/server/releases/nextcloud-16.0.3.tar.bz2
md5sum = d81902d2dec5d547779bec6336a438be
[template-nextcloud-install.sh]
<= nc-download-base
<= nc-download
[template-apache-httpd]
<= nc-download-base
<= nc-download
[template-nextcloud-config.json]
<= nc-download-base
<= nc-download
[template-nextcloud-instance]
recipe = slapos.recipe.template:jinja2
......@@ -37,48 +33,46 @@ context =
key php_location apache-php:location
raw redis_bin ${redis:location}/bin/redis-server
raw redis_cli ${redis:location}/bin/redis-cli
raw nextcloud_install_sh ${template-nextcloud-install.sh:location}/${template-nextcloud-install.sh:filename}
raw nextcloud_apache_httpd ${template-apache-httpd:location}/${template-apache-httpd:filename}
raw nextcloud_parameter_json ${template-nextcloud-config.json:location}/${template-nextcloud-config.json:filename}
key nextcloud_install_sh template-nextcloud-install.sh:target
key nextcloud_apache_httpd template-apache-httpd:target
key nextcloud_parameter_json template-nextcloud-config.json:target
section nextcloud_apps nextcloud-apps
[custom-application-deployment]
path = ${template-nextcloud-instance:rendered}
part-list = nextcloud-install.sh
[nc-download-unpacked]
recipe = slapos.recipe.build:download-unpacked
shared = true
[news-updater]
recipe = hexagonit.recipe.download
<= nc-download-unpacked
url = https://github.com/nextcloud/news-updater/archive/10.0.1.tar.gz
md5sum = 37387199c0482e08d01e9294cd95eaad
strip-top-level-dir = true
[nc-download-app]
<= nc-download-base
download-only = false
strip-top-level-dir = true
[nextcloud-app-spreed]
<= nc-download-app
<= nc-download-unpacked
url = https://github.com/nextcloud/spreed/releases/download/v6.0.5/spreed-6.0.5.tar.gz
md5sum = 002c09e543edc141f6ca848782573376
[nextcloud-app-richdocuments]
<= nc-download-app
<= nc-download-unpacked
url = https://github.com/nextcloud/richdocuments/releases/download/v3.7.17/richdocuments.tar.gz
md5sum = 5559cd14a4a0a93d2a39b260538839f8
[nextcloud-app-calendar]
<= nc-download-app
<= nc-download-unpacked
url = https://github.com/nextcloud/calendar/releases/download/v1.7.3/calendar.tar.gz
md5sum = ab398d943eb6939e3e71df5b1a1abf87
[nextcloud-app-rainloop]
<= nc-download-app
<= nc-download-unpacked
url = https://github.com/pierre-alain-b/rainloop-nextcloud/releases/download/6.1.4/rainloop.tar.gz
md5sum = 7cefc3dd3bd52b42d381de7d7447691f
[nextcloud-app-news]
<= nc-download-app
<= nc-download-unpacked
url = https://github.com/nextcloud/news/releases/download/14.2.2/news.tar.gz
md5sum = f48d4b5dcbc078131bb86a4ae619da99
......
# ORS Amarisoft software release
How to deploy from scratch
1. Compile and install kernel module lte_trx_sdr by `# cd trx_sdr*/kernel/ && make && sh init.sh`
2. Make sure to have "create_tun = True" in /etc/opt/slapos/slapos.cfg
3. Install ors playbook
4. Deploy this SR
## Generated buildout configurations and json input schemas
Since there are many ors-amarisoft softwares releases and software types, the following files are
generated with jinja2 templates with the render-templates script before being pushed to gitlab:
* instance-tdd1900-enb-epc-input-schema.json
* instance-tdd1900-enb-input-schema.json
* instance-tdd1900-gnb-epc-input-schema.json
* instance-tdd1900-gnb-input-schema.json
* instance-tdd2600-enb-epc-input-schema.json
* instance-tdd2600-enb-input-schema.json
* instance-tdd2600-gnb-epc-input-schema.json
* instance-tdd2600-gnb-input-schema.json
* instance-tdd3500-enb-epc-input-schema.json
* instance-tdd3500-enb-input-schema.json
* instance-tdd3500-gnb-epc-input-schema.json
* instance-tdd3500-gnb-input-schema.json
* instance-tdd3700-enb-epc-input-schema.json
* instance-tdd3700-enb-input-schema.json
* instance-tdd3700-gnb-epc-input-schema.json
* instance-tdd3700-gnb-input-schema.json
* software-tdd1900.cfg
* software-tdd1900.cfg.json
* software-tdd2600.cfg
* software-tdd2600.cfg.json
* software-tdd3500.cfg
* software-tdd3500.cfg.json
* software-tdd3700.cfg
* software-tdd3700.cfg.json
These files should not be modified directly, and the render-templates scripts should be run along
with update-hash before each commit.
## Services
instance.cfg is rather complicated because Amarisoft LTE stack consists of 4 binaries
* **lteenb** - eNodeB software is the server accepting connection from UI (user interfaces)
* **ltemme** - Mobile Management Entity in other words core network which handles orchestration of
eNodeBs in case UI switches from one to another
* **lteims** - IP Multimedia System is another protocol such as LTE but designed for services over
IP. Please read http://www.differencebetween.com/difference-between-lte-and-vs-ims-2/
* **ltembmsgw** - Multimedia Broadcast Multicast Services (Gateway) is technology which broadcast
the same multimedia content into multiple IP addresses at once to save bandwidth.
Those binaries are started in foreground, originaly in screen. We don't want the binaries inside one
screen because then we cannot easily control their resource usage. Thus we make 4 on-watch services.
### MME
Is the core network. This binary keep track of UEs and to which eNodeB they are currently connected.
It reroutes traffic when UE switches between eNodeBs.
MME also serves as a service bus thus all services must register within MME.
### IMS
Service connected into MME bus. IMS handles circuit-ish services over IP whereas LTE would have
failed because it is intended as data-over-IP service.
### MBMSGW
MBMS Gateway is a standalone component connected to BMSC (Broadcast Multicast Service Centre), server
supporting streaming content from providers, which is another component inside our core network not
provided by Amarisoft.
MBMS Gateway is connected to MME which then manages MBMS sessions.
## Gotchas!
**trx_sdr.so** provided from archive MUST be placed next to `lteenb` binary. This library is the
only one which does not follow standard `ld` path resolution.
**rf_driver** has to be compiled and installed. Inside trx_sdr/kernel folder issue `# make` to compile the
kernel module, and then `# ./init.sh` to create devices `/dev/sdr<N>` and insert compiled module.
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 96a76300b2f714b6c47157920fe79a53
[template-lte-enb-epc]
_update_hash_filename_ = instance-enb-epc.jinja2.cfg
md5sum = 762d55291e75e8b61e35f9f28d29915a
[template-lte-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = 83c37b43d7b70584bf71aed9289ea13c
[template-lte-gnb-epc]
_update_hash_filename_ = instance-gnb-epc.jinja2.cfg
md5sum = e43a726dd3023a4bbaa474bb2d7a6ebe
[template-lte-gnb]
_update_hash_filename_ = instance-gnb.jinja2.cfg
md5sum = fc59d15a7f7f942951f9e38d7a1cca2c
[template-lte-epc]
_update_hash_filename_ = instance-epc.jinja2.cfg
md5sum = 0585b4f8fd42538595e6abb61d07fd93
[ue_db.jinja2.cfg]
filename = config/ue_db.jinja2.cfg
md5sum = d33163012d6c98efc59161974c649557
[enb.jinja2.cfg]
filename = config/enb.jinja2.cfg
md5sum = d841debc51d9f12555a47d1556a6a3c1
[gnb.jinja2.cfg]
filename = config/gnb.jinja2.cfg
md5sum = da64ea9c5003f40987a8bba3f18e8839
[ltelogs.jinja2.sh]
filename = ltelogs.jinja2.sh
md5sum = 1ba2e065bdf14a6411e95e80db17dcfd
[mme.jinja2.cfg]
filename = config/mme.jinja2.cfg
md5sum = 518c71ce57204304b703b977c665a164
[ims.jinja2.cfg]
filename = config/ims.jinja2.cfg
md5sum = e561ec26a70943c61557def1781cf65f
/* lteenb configuration file version 2021-09-18
* Copyright (C) 2015-2021 Amarisoft
*/
#define TDD 1 // Values: 0 (FDD), 1(TDD)
#define N_RB_DL {{ slapparameter_dict.get('n_rb_dl', slap_configuration['configuration.default_lte_n_rb_dl']) }} // Values: 6 (1.4 MHz), 15 (3MHz), 25 (5MHz), 50 (10MHz), 75 (15MHz), 100 (20MHz)
#define N_ANTENNA_DL 2 // Values: 1 (SISO), 2 (MIMO 2x2)
#define N_ANTENNA_UL 2 // Values: 1, 2
#define CHANNEL_SIM 0 // Values: 0 (channel simulator disabled), 1 (channel simulator enabled)
{
/* Log filter: syntax: layer.field=value[,...]
Possible layers are phy, mac, rlc, pdcp, rrc, nas, s1ap, x2ap, gtpu and
all. The 'all' layer is used to address all the layers at the
same time.
field values:
- 'level': the log level of each layer can be set to 'none',
'error', 'info' or 'debug'. Use 'debug' to log all the messages.
- 'max_size': set the maximum size of the hex dump. 0 means no
hex dump. -1 means no limit.
*/
log_options: "all.level=debug,all.max_size=32",
log_filename: "{{ directory['log'] }}/enb.log",
/* Enable remote API and Web interface */
com_addr: "[{{ slap_configuration['ipv6-random'] }}]:{{ slap_configuration['configuration.enb_ws_port'] }}",
/* RF driver configuration */
rf_driver: {
name: "sdr",
/* list of devices. 'dev0' is always the master. */
args: "dev0=/dev/sdr0",
/* synchronisation source: none, internal, gps, external (default = none) */
// sync: "gps",
rx_antenna:"tx_rx",
},
tx_gain: {{ slapparameter_dict.get('tx_gain', slap_configuration['configuration.default_lte_tx_gain']) }}, /* TX gain (in dB) */
rx_gain: {{ slapparameter_dict.get('rx_gain', slap_configuration['configuration.default_lte_rx_gain']) }}, /* RX gain (in dB) */
#if CHANNEL_SIM == 1
rf_ports: [
{
channel_dl: {
type: "awgn",
noise_level: -30,
},
}
],
#endif
mme_list: [
{
/* address of MME for S1AP connection. Must be modified if the MME
runs on a different host. */
mme_addr: "127.0.1.100"
},
],
/* GTP bind address (=address of the ethernet interface connected to
the MME). Must be modified if the MME runs on a different host. */
gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
/* high 20 bits of SIB1.cellIdentifier */
enb_id: 0x1A2D0,
/* list of cells */
cell_list: [
{
/* Broadcasted PLMN identities */
plmn_list: [
"00101",
],
dl_earfcn: {{ slapparameter_dict.get('dl_earfcn', slap_configuration['configuration.default_dl_earfcn']) }},
n_id_cell: 1,
cell_id: 0x01,
tac: 0x0001,
root_sequence_index: 204, /* PRACH root sequence index */
},
], /* cell_list */
/* default cell parameters */
cell_default: {
n_antenna_dl: N_ANTENNA_DL, /* number of DL antennas */
n_antenna_ul: N_ANTENNA_UL, /* number of UL antennas */
#if TDD == 1
uldl_config: 2, /* TDD only */
sp_config: 7, /* TDD only */
#endif
n_rb_dl: N_RB_DL, /* Bandwidth: 25: 5 MHz, 50: 10 MHz, 75: 15 MHz, 100: 20 MHz */
cyclic_prefix: "normal",
phich_duration: "normal",
phich_resource: "1", /* ratio of NG */
/* SIB1 */
si_value_tag: 0, /* increment modulo 32 if SI is modified */
cell_barred: false, /* SIB1.cellBarred-r13 */
intra_freq_reselection: true, /* SIB1.intraFreqReselection */
q_rx_lev_min: -70, /* SIB1.q-RxLevMin */
p_max: 10, /* maximum power allowed for the UE (dBm) */
si_window_length: 40, /* ms */
sib_sched_list: [
{
filename: "{{ directory['software'] }}/enb/config/sib23.asn",
si_periodicity: 16, /* frames */
},
],
#if N_RB_DL == 6
si_coderate: 0.30, /* maximum code rate for SI/RA/P-RNTI messages */
#else
si_coderate: 0.20, /* maximum code rate for SI/RA/P-RNTI messages */
#endif
si_pdcch_format: 2, /* 2 or 3. Log2 of the number of CCEs for PDCCH
for SI/RA/P-RNTI */
n_symb_cch: 0, /* number of symbols for CCH (0 = auto) */
/* PDSCH dedicated config (currently same for all UEs) */
pdsch_dedicated: {
#if N_ANTENNA_DL == 4
p_a: -6,
#elif N_ANTENNA_DL == 2
p_a: -3,
#else
p_a: 0,
#endif
p_b: -1, /* -1 means automatic */
},
/* If defined, force for number of CCEs for UE specific PDCCH to
2^pdcch_format. Otherwise it is computed from the reported
CQI. Range: 0 to 3. */
#if N_RB_DL == 6
pdcch_format: 1,
#else
pdcch_format: 2,
#endif
/* if defined, force the PDSCH MCS for all UEs. Otherwise it is
computed from the reported CQI */
/* pdsch_mcs: 12, */
#if N_RB_DL == 6
prach_config_index: 15, /* subframe 9 every 20 ms */
#else
prach_config_index: 4, /* subframe 4 every 10 ms */
#endif
prach_freq_offset: -1, /* -1 means automatic */
/* PUCCH dedicated config (currently same for all UEs) */
pucch_dedicated: {
n1_pucch_sr_count: 11, /* increase if more UEs are needed */
cqi_pucch_n_rb: 1, /* increase if more UEs are needed */
#if TDD == 1
//tdd_ack_nack_feedback_mode: "bundling", /* TDD only */
tdd_ack_nack_feedback_mode: "multiplexing", /* TDD only */
#endif
},
/* PUSCH dedicated config (currently same for all UEs) */
pusch_dedicated: {
beta_offset_ack_index: 9,
beta_offset_ri_index: 6,
beta_offset_cqi_index: 6,
},
pusch_hopping_offset: -1, /* -1 means automatic */
/* MCS for Msg3 (=CCCH RRC Connection Request) */
pusch_msg3_mcs: 0,
/* this CQI value is assumed when none is received from the UE */
#if N_RB_DL == 6
initial_cqi: 5,
#else
initial_cqi: 3,
#endif
/* if defined, force the PUSCH MCS for all UEs. Otherwise it is
computed from the last received SRS/PUSCH. */
// pusch_mcs: 18,
dl_256qam: true,
ul_64qam: true,
/* Scheduling request period (ms). Must be >= 40 for HD-FDD */
sr_period: 20,
/* CQI report config */
cqi_period: 40, /* period (ms). Must be >= 32 for HD-FDD */
#if N_ANTENNA_DL >= 2
/* RI reporting is done with a period of m_ri * cqi_period.
m_ri = 0 (default) disables RI reporting. */
m_ri: 8,
/* transmission mode */
transmission_mode: 3,
#endif
/* SRS dedicated config. All UEs share these
parameters. srs_config_index and freq_domain_position are
allocated for each UE) */
srs_dedicated: {
#if N_RB_DL == 6
srs_bandwidth_config: 7,
srs_bandwidth: 1,
#elif N_RB_DL == 15
srs_bandwidth_config: 6,
srs_bandwidth: 1,
#elif N_RB_DL == 25
srs_bandwidth_config: 3,
srs_bandwidth: 1,
#elif N_RB_DL == 50
srs_bandwidth_config: 2,
srs_bandwidth: 2,
#elif N_RB_DL == 75
srs_bandwidth_config: 2,
srs_bandwidth: 2,
#else
srs_bandwidth_config: 2,
srs_bandwidth: 3,
#endif
srs_subframe_config: 3, /* 0 - 15 */
srs_period: 40, /* period (ms). Must be >= 40 for HD-FDD */
srs_hopping_bandwidth: 0,
},
/* MAC configuration (same for all UEs) */
mac_config: {
ul_max_harq_tx: 5, /* max number of HARQ transmissions for uplink */
dl_max_harq_tx: 5, /* max number of HARQ transmissions for downlink */
},
/* CPU load limitation */
pusch_max_its: 6, /* max number of turbo decoder iterations */
/* dynamic power control */
dpc: true,
dpc_pusch_snr_target: 25,
dpc_pucch_snr_target: 20,
/* RRC/UP ciphering algorithm preference. EEA0 is always the last. */
cipher_algo_pref: [],
/* RRC integrity algorithm preference. EIA0 is always the last. */
integ_algo_pref: [2, 1],
/* (in ms) send RRC connection release after this time of network
inactivity */
inactivity_timer: 10000,
/* SRB configuration */
srb_config: [
{
id: 1,
maxRetxThreshold: 32,
t_Reordering: 45,
t_PollRetransmit: 60,
},
{
id: 2 ,
maxRetxThreshold: 32,
t_Reordering: 45,
t_PollRetransmit: 60,
}
],
/* DRB configuration */
drb_config: "{{ directory['software'] }}/enb/config/drb.cfg",
},
}
/* lteenb configuration file version 2021-09-18
* Copyright (C) 2019-2021 Amarisoft
* NR SA FDD or TDD cell */
#define NR_TDD 1 // Values: 0 (NR FDD), 1(NR TDD)
#define FR2 0 // Values: 0 (FR1), 1 (FR2)
#define NR_TDD_CONFIG 2 // Values: FR1: 1, 2, 3, 4 (compatible with LTE TDD config 2) FR2: 10
#define N_ANTENNA_DL 2 // Values: 1 (SISO), 2 (MIMO 2x2), 4 (MIMO 4x4)
#define N_ANTENNA_UL 2 // Values: 1, 2, 4
#define NR_BANDWIDTH {{ slapparameter_dict.get('nr_bandwidth', slap_configuration['configuration.default_nr_bandwidth']) }} // NR cell bandwidth
#define NR_LONG_PUCCH_FORMAT 2 // Values: 2, 3, 4
/* define to 1 to enable periodic SRS with N_ANTENNA_UL ports. Uplink
SU-MIMO is also enabled if N_ANTENNA_UL >= 2. Not all UEs support
uplink SU-MIMO. */
#define USE_SRS 0
{
log_options: "all.level=debug,all.max_size=32",
log_filename: "{{ directory['log'] }}/gnb.log",
/* Enable remote API and Web interface */
com_addr: "[{{ slap_configuration['ipv6-random'] }}]:{{ slap_configuration['configuration.enb_ws_port'] }}",
rf_driver: {
name: "sdr",
/* list of devices. 'dev0' is always the master. */
args: "dev0=/dev/sdr0",
/* synchronisation source: none, internal, gps, external (default = none) */
// sync: "gps",
rx_antenna: "tx_rx",
},
tx_gain: {{ slapparameter_dict.get('tx_gain', slap_configuration['configuration.default_nr_tx_gain']) }}, /* TX gain (in dB) */
rx_gain: {{ slapparameter_dict.get('rx_gain', slap_configuration['configuration.default_nr_rx_gain']) }}, /* RX gain (in dB) */
amf_list: [
{
/* address of AMF for NGAP connection. Must be modified if the AMF runs on a different host. */
amf_addr: "127.0.1.100",
},
],
/* GTP bind address (=address of the ethernet interface connected to
the AMF). Must be modified if the AMF runs on a different host. */
gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
gnb_id_bits: 28,
gnb_id: 0x12345,
nr_support: true,
rf_ports: [
{
#if FR2
/* an external frequency translator must be used for FR2 */
rf_dl_freq: 3500, /* MHz */
rf_ul_freq: 3500, /* MHz */
/* uncomment to have a higher per-UE bitrate at the expense of
higher gNB real time constraints. The default value is 4
ms. 1 ms gives the maximum per-UE bitrate. */
// rx_to_tx_latency: 1, /* ms */
#endif
},
],
/* list of cells */
cell_list: [],
nr_cell_list: [
{
rf_port: 0,
cell_id: 0x01,
#if NR_TDD == 1
#if FR2
band: 257,
dl_nr_arfcn: 2079167, /* 28000.08 MHz */
subcarrier_spacing: 120, /* kHz */
ssb_pos_bitmap: "0100000000000000000000000000000000000000000000000000000000000000",
#else
band: {{ slapparameter_dict.get('nr_band', slap_configuration['configuration.default_nr_band']) }},
dl_nr_arfcn: {{ slapparameter_dict.get('dl_nr_arfcn', slap_configuration['configuration.default_dl_nr_arfcn']) }},
subcarrier_spacing: 30, /* kHz */
ssb_pos_bitmap: "10000000",
#endif
#else
band: 7,
dl_nr_arfcn: 536020, /* 2680 MHz */
ssb_subcarrier_spacing: 15,
subcarrier_spacing: 30, /* kHz */
ssb_pos_bitmap: "1000",
#endif
},
], /* nr_cell_list */
nr_cell_default: {
bandwidth: NR_BANDWIDTH, /* MHz */
n_antenna_dl: N_ANTENNA_DL,
n_antenna_ul: N_ANTENNA_UL,
/* force the timing TA offset (optional) */
// n_timing_advance_offset: 39936,
/* subframe offset to align with the LTE TDD pattern (optional) */
// subframe_offset: 2,
#if NR_TDD == 1
tdd_ul_dl_config: {
#if NR_TDD_CONFIG == 1
pattern1: {
period: 5, /* in ms */
dl_slots: 7,
dl_symbols: /* 6 */ 2,
ul_slots: 2,
ul_symbols: 2,
},
#elif NR_TDD_CONFIG == 2
pattern1: {
period: 5, /* in ms */
dl_slots: 7,
dl_symbols: 6,
ul_slots: 2,
ul_symbols: 4,
},
#elif NR_TDD_CONFIG == 3
pattern1: {
period: 5, /* in ms */
dl_slots: 6,
dl_symbols: 2,
ul_slots: 3,
ul_symbols: 2,
},
#elif NR_TDD_CONFIG == 4
pattern1: {
period: 3, /* in ms */
dl_slots: 3,
dl_symbols: 6,
ul_symbols: 4,
ul_slots: 2,
},
pattern2: {
period: 2, /* in ms */
dl_slots: 4,
dl_symbols: 0,
ul_symbols: 0,
ul_slots: 0,
},
#elif NR_TDD_CONFIG == 10
/* only for FR2 */
pattern1: {
period: 0.625, /* in ms */
dl_slots: 3,
dl_symbols: 10,
ul_slots: 1,
ul_symbols: 2,
},
#endif
},
#endif
ssb_period: 20, /* in ms */
n_id_cell: 500,
plmn_list: [ {
tac: 100,
plmn: "00101",
reserved: false,
nssai: [
{
sst: 1,
},
/*{
sst: 2,
},
{
sst: 3,
sd: 50,
},*/
],
},
],
/*sib_sched_list: [
{
filename: "{{ directory['software'] }}/enb/config/sib2_nr.asn",
si_periodicity: 16,
},
{
filename: "{{ directory['software'] }}/enb/config/sib3_nr.asn",
si_periodicity: 16,
},
{
filename: "{{ directory['software'] }}/enb/config/sib4_nr.asn",
si_periodicity: 32,
},
],
sib9: {
si_periodicity: 32
},*/
si_window_length: 40,
cell_barred: false,
intra_freq_reselection: true,
q_rx_lev_min: -70,
q_qual_min: -20,
p_max: 10, /* dBm */
root_sequence_index: 1, /* PRACH root sequence index */
/* Scheduling request period (slots). */
sr_period: 40,
dmrs_type_a_pos: 2,
/* to limit the number of HARQ feedback in UL, use pdsch_harq_ack_max;
allows to workaround issues with SM-G977N for example */
//pdsch_harq_ack_max: 2,
prach: {
#if NR_TDD == 1
#if FR2
prach_config_index: 149, /* format C0, every 4 frames */
msg1_subcarrier_spacing: 120, /* kHz */
#else
#if NR_TDD_CONFIG == 4
prach_config_index: 156, /* format B4, subframe 2 */
#else
prach_config_index: 160, /* format B4, subframe 9 */
#endif
msg1_subcarrier_spacing: 30, /* kHz */
#endif
#else
prach_config_index: 16, /* subframe 1 every frame */
#endif
msg1_fdm: 1,
msg1_frequency_start: -1,
zero_correlation_zone_config: 15,
preamble_received_target_power: -110, /* in dBm */
preamble_trans_max: 7,
power_ramping_step: 4, /* in dB */
#if FR2
ra_response_window: 40, /* in slots */
#else
ra_response_window: 20, /* in slots */
#endif
restricted_set_config: "unrestricted_set",
ra_contention_resolution_timer: 64, /* in ms */
ssb_per_prach_occasion: 1,
cb_preambles_per_ssb: 8,
},
pdcch: {
search_space0_index: 0,
dedicated_coreset: {
rb_start: -1, /* -1 to have the maximum bandwidth */
l_crb: -1, /* -1 means all the bandwidth */
duration: 0, /* 0 means to automatically set it from the coreset bandwidth */
precoder_granularity: "sameAsREG_bundle",
},
css: {
n_candidates: [ 0, 0, 1, 0, 0 ],
},
rar_al_index: 2,
si_al_index: 2,
uss: {
n_candidates: [ 0, 2, 1, 0, 0 ],
dci_0_1_and_1_1: true,
},
al_index: 1,
},
pdsch: {
mapping_type: "typeA",
dmrs_add_pos: 1,
dmrs_type: 1,
dmrs_max_len: 1,
/* k0 delay in slots from DCI to PDSCH: automatic setting */
/* k1 delay in slots from PDSCH to PUCCH/PUSCH ACK/NACK: automatic setting */
mcs_table: "qam256",
rar_mcs: 2,
si_mcs: 6,
/* If defined, force the PDSCH MCS for all UEs. Otherwise it is computed
* based on DL channel quality estimation */
/* mcs: 24, */
},
csi_rs: {
nzp_csi_rs_resource: [
{
csi_rs_id: 0,
#if N_ANTENNA_DL == 1
n_ports: 1,
frequency_domain_allocation: "row2",
bitmap: "100000000000",
cdm_type: "no_cdm",
#elif N_ANTENNA_DL == 2
n_ports: 2,
frequency_domain_allocation: "other",
bitmap: "100000",
cdm_type: "fd_cdm2",
#elif N_ANTENNA_DL == 4
n_ports: 4,
frequency_domain_allocation: "row4",
bitmap: "100",
cdm_type: "fd_cdm2",
#elif N_ANTENNA_DL == 8
n_ports: 8,
frequency_domain_allocation: "other",
bitmap: "110011",
cdm_type: "fd_cdm2",
#else
#error unsupported number of DL antennas
#endif
density: 1,
first_symb: 4,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
power_control_offset: 0, /* dB */
power_control_offset_ss: 0, /* dB */
period: 80,
offset: 1, /* != 0 to avoid collision with SSB */
qcl_info_periodic_csi_rs: 0,
},
#if FR2 == 0
#define USE_TRS
#endif
#ifdef USE_TRS
/* TRS : period of 40 ms, slots 1 & 2, symbols 4 and 8 */
{
csi_rs_id: 1,
n_ports: 1,
frequency_domain_allocation: "row1",
bitmap: "0001",
cdm_type: "no_cdm",
density: 3,
first_symb: 4,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
power_control_offset: 0, /* dB */
power_control_offset_ss: 0, /* dB */
period: 40,
offset: 11,
qcl_info_periodic_csi_rs: 0,
},
{
csi_rs_id: 2,
n_ports: 1,
frequency_domain_allocation: "row1",
bitmap: "0001",
cdm_type: "no_cdm",
density: 3,
first_symb: 8,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
power_control_offset: 0, /* dB */
power_control_offset_ss: 0, /* dB */
period: 40,
offset: 11,
qcl_info_periodic_csi_rs: 0,
},
{
csi_rs_id: 3,
n_ports: 1,
frequency_domain_allocation: "row1",
bitmap: "0001",
cdm_type: "no_cdm",
density: 3,
first_symb: 4,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
power_control_offset: 0, /* dB */
power_control_offset_ss: 0, /* dB */
period: 40,
offset: 12,
qcl_info_periodic_csi_rs: 0,
},
{
csi_rs_id: 4,
n_ports: 1,
frequency_domain_allocation: "row1",
bitmap: "0001",
cdm_type: "no_cdm",
density: 3,
first_symb: 8,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
power_control_offset: 0, /* dB */
power_control_offset_ss: 0, /* dB */
period: 40,
offset: 12,
qcl_info_periodic_csi_rs: 0,
},
#endif
],
nzp_csi_rs_resource_set: [
{
csi_rs_set_id: 0,
nzp_csi_rs_resources: [ 0 ],
repetition: false,
},
#ifdef USE_TRS
{
csi_rs_set_id: 1,
nzp_csi_rs_resources: [ 1, 2, 3, 4 ],
repetition: false,
trs_info: true,
},
#endif
],
csi_im_resource: [
{
csi_im_id: 0,
pattern: 1,
subcarrier_location: 8,
symbol_location: 8,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
period: 80,
offset: 1, /* != 0 to avoid collision with SSB */
},
],
csi_im_resource_set: [
{
csi_im_set_id: 0,
csi_im_resources: [ 0 ],
}
],
/* ZP CSI-RS to set the CSI-IM REs to zero */
zp_csi_rs_resource: [
{
csi_rs_id: 0,
frequency_domain_allocation: "row4",
bitmap: "100",
n_ports: 4,
cdm_type: "fd_cdm2",
first_symb: 8,
density: 1,
rb_start: 0,
l_crb: -1, /* -1 means from rb_start to the end of the bandwidth */
period: 80,
offset: 1,
},
],
p_zp_csi_rs_resource_set: [
{
zp_csi_rs_resources: [ 0 ],
},
],
csi_resource_config: [
{
csi_rsc_config_id: 0,
nzp_csi_rs_resource_set_list: [ 0 ],
resource_type: "periodic",
},
{
csi_rsc_config_id: 1,
csi_im_resource_set_list: [ 0 ],
resource_type: "periodic",
},
#ifdef USE_TRS
{
csi_rsc_config_id: 2,
nzp_csi_rs_resource_set_list: [ 1 ],
resource_type: "periodic",
},
#endif
],
csi_report_config: [
{
resources_for_channel_measurement: 0,
csi_im_resources_for_interference: 1,
report_config_type: "periodic",
period: 80,
report_quantity: "CRI_RI_PMI_CQI",
#if N_ANTENNA_DL > 1
codebook_config: {
codebook_type: "type1",
sub_type: "typeI_SinglePanel",
#if N_ANTENNA_DL == 2
#elif N_ANTENNA_DL == 4
n1: 2,
n2: 1,
codebook_mode: 1,
#elif N_ANTENNA_DL == 8
n1: 4,
n2: 1,
codebook_mode: 1,
#endif
},
#endif
cqi_table: 2,
subband_size: "value1",
},
],
},
pucch: {
pucch_group_hopping: "neither",
hopping_id: -1, /* -1 = n_cell_id */
p0_nominal: -90,
#if 0
pucch0: {
initial_cyclic_shift: 1,
n_symb: 1,
},
#else
pucch1: {
n_cs: 3,
n_occ: 3,
freq_hopping: true,
#if USE_SRS && NR_TDD == 0
n_symb: 13,
#endif
},
#endif
#if NR_LONG_PUCCH_FORMAT == 2
pucch2: {
n_symb: 2,
n_prb: 1,
freq_hopping: true,
simultaneous_harq_ack_csi: false,
max_code_rate: 0.25,
},
#elif NR_LONG_PUCCH_FORMAT == 3
pucch3: {
bpsk: false,
additional_dmrs: false,
freq_hopping: true,
n_prb: 1,
simultaneous_harq_ack_csi: true,
max_code_rate: 0.25,
},
#elif NR_LONG_PUCCH_FORMAT == 4
pucch4: {
occ_len: 4,
bpsk: false,
additional_dmrs: false,
freq_hopping: true,
simultaneous_harq_ack_csi: true,
max_code_rate: 0.25,
},
#endif
},
#if USE_SRS
srs: {
#if NR_TDD
#if NR_TDD_CONFIG == 1 || NR_TDD_CONFIG == 2
srs_symbols: [ 0, 0, 0, 0, 0, 0, 0, 2, 0, 0 ],
#elif NR_TDD_CONFIG == 3
srs_symbols: [ 0, 0, 0, 0, 0, 0, 2, 0, 0, 0 ],
#elif NR_TDD_CONFIG == 4
srs_symbols: [ 0, 0, 0, 4, 0, 0, 0, 0, 0, 0 ],
#elif NR_TDD_CONFIG == 10
srs_symbols: [ 0, 0, 0, 2, 0 ],
#endif
#else
srs_symbols: [ 1, 0, 0, 0, 0, 1, 0, 0, 0, 0 ],
#endif
srs_resource: [
{
srs_resource_id: 0,
n_ports: N_ANTENNA_UL,
resource_type: "periodic",
period: 80, /* in slots */
}
],
srs_resource_set: [
{
srs_resource_id_list: [ 0 ],
},
],
},
#endif
pusch: {
mapping_type: "typeA",
n_symb: 14,
dmrs_add_pos: 1,
dmrs_type: 1,
dmrs_max_len: 1,
tf_precoding: false,
mcs_table: "qam256", /* without transform precoding */
mcs_table_tp: "qam256", /* with transform precoding */
ldpc_max_its: 5,
/* k2, msg3_k2 delay in slots from DCI to PUSCH: automatic setting */
p0_nominal_with_grant: -76,
msg3_mcs: 4,
msg3_delta_power: 0, /* in dB */
beta_offset_ack_index: 9,
/* if defined, force the PUSCH MCS for all UEs. Otherwise it is
computed from the last received PUSCH. */
/* mcs: 16, */
},
/* MAC configuration */
mac_config: {
msg3_max_harq_tx: 5,
ul_max_harq_tx: 5, /* max number of HARQ transmissions for uplink */
dl_max_harq_tx: 5, /* max number of HARQ transmissions for downlink */
ul_max_consecutive_retx: 30, /* disconnect UE if reached */
dl_max_consecutive_retx: 30, /* disconnect UE if reached */
periodic_bsr_timer: 20,
retx_bsr_timer: 320,
periodic_phr_timer: 500,
prohibit_phr_timer: 200,
phr_tx_power_factor_change: "dB3",
sr_prohibit_timer: 0, /* in ms, 0 to disable the timer */
sr_trans_max: 64,
},
cipher_algo_pref: [],
integ_algo_pref: [2, 1],
inactivity_timer: 10000,
drb_config: "{{ directory['software'] }}/enb/config/drb_nr.cfg",
},
}
/* lteims configuration file
* version 2021-09-18
* Copyright (C) 2015-2021 Amarisoft
*/
{
log_options: "all.level=debug,all.max_size=32",
log_filename: "{{ directory['log'] }}/ims.log",
/* SIP bind address */
//sip_addr: [
// {addr: "192.168.4.1", bind_addr: "192.168.4.1", port_min: 10000, port_max: 20000},
// "2001:468:3000:1::",
// {addr: "192.168.5.1", bind_addr: "192.168.5.1", port_min: 10000, port_max: 20000},
// "2001:468:4000:1::"
//],
/* SIP bind address */
sip_addr: [
{addr: "{{ slap_configuration['tun-ipv4-addr'] }}", bind_addr: "0.0.0.0", port_min: 10000, port_max: 20000},
{#" slap_configuration['tun-ipv6-addr'] ",#}
],
mms_server_bind_addr: "{{ netaddr.IPAddress(netaddr.IPNetwork(slap_configuration['tun-ipv4-network']).first) + 1 }}:1111",
/* MME connection for SMS over SG */
sctp_addr: "{{ slap_configuration['configuration.ims_addr'] }}",
/* Cx connection */
cx_server_addr: "127.0.1.100",
cx_bind_addr: "{{ slap_configuration['configuration.ims_addr'] }}",
/* Rx connection */
rx_server_addr: "127.0.1.100",
rx_bind_addr: "{{ slap_configuration['configuration.ims_addr'] }}",
/* Remote API */
com_addr: "[{{ slap_configuration['ipv6-random'] }}]:{{ slap_configuration['configuration.ims_ws_port'] }}",
/* Global domain name (May be overriden for each user) */
domain: "{{ slap_configuration['configuration.domain'] }}",
include "{{ slap_configuration['ue_db_path'] }}",
{# Example of of s6a connection #}
{# s6: { #}
{# server_addr: "", #}
{# bind_addr: "", #}
{# origin_realm: "", #}
{# origin_host: "", #}
{# }, #}
/* Echo phone number */
echo: [
"tel:666",
"tel:+666",
{impu: "tel:404", code: 404}, /* 404 test */
{impu: "urn:service:sos", anonymous: true, authentication: false}, /* Emergency call */
{impu: "urn:service:sos.police", anonymous: true, authentication: false}, /* Emergency call */
],
/* Delay */
//sms_expires: 86400,
//binding_expires: 3600,
/* on: 3GPP mode allowed
* silent: 3GPP mode forced
* off: IETF mode
*/
precondition: "on",
"100rel": true,
/* IPsec */
ipsec_aalg_list: ["hmac-md5-96", "hmac-sha-1-96"],
ipsec_ealg_list: ["null", "aes-cbc", "des-cbc", "des-ede3-cbc"],
mt_call_sdp_file: "{{ directory['software'] }}/mme/config/mt_call.sdp",
ue_db_filename: "{{ directory['var'] }}/lte_ue_ims.db",
}
/* ltemme configuration file for ims
* version 2021-09-18
* Copyright (C) 2015-2021 Amarisoft
*/
{
/* Log filter: syntax: layer.field=value[,...]
Possible layers are nas, ip, s1ap, gtpu and all. The 'all' layer
is used to address all the layers at the same time.
field values:
- 'level': the log level of each layer can be set to 'none',
'error', 'info' or 'debug'. Use 'debug' to log all the messages.
- 'max_size': set the maximum size of the hex dump. 0 means no
hex dump. -1 means no limit.
*/
log_options: "all.level=debug,all.max_size=32",
log_filename: "{{ directory['log'] }}/mme.log",
/* Enable remote API and Web interface */
com_addr: "[{{ slap_configuration['ipv6-random'] }}]:{{ slap_configuration['configuration.mme_ws_port'] }}",
/* bind address for GTP-U. Normally = address of the PC, here bound
on local interface to be able to run ltemme on the same PC as
lteenb. By default, the S1AP SCTP connection is bound on the same
address. */
gtp_addr: "127.0.1.100",
plmn: "00101",
mme_group_id: 32769,
mme_code: 1,
ims_vops_eps: true, /* IMS supported */
ims_vops_5gs_3gpp: true, /* IMS supported */
ims_vops_5gs_n3gpp: true, /* IMS supported */
//emc_bs: true, /* emergency calls supported */
//emc: 3, /* NR/E-UTRA connected to 5GCN emergency calls supported */
//emc_n3gpp: true, /* non-3GPP emergency calls supported */
emergency_number_list: [
/* Category bits: (Table 10.5.135d/3GPP TS 24.008)
Bit 1 Police
Bit 2 Ambulance
Bit 3 Fire Brigade
Bit 4 Marine Guard
Bit 5 Mountain Rescue
*/
{ category: 0x1f, digits: "911" },
{ category: 0x1f, digits: "112" },
],
rx: {
qci: {audio: 1, video: 2},
},
/* network name and network short name sent in the EMM information
message to the UE */
network_name: "{{ slap_configuration['configuration.network_name'] }}",
network_short_name: "{{ slap_configuration['configuration.network_name'] }}",
/* Control Plane Cellular IoT EPS optimization support */
cp_ciot_opt: true,
/* DCNR support */
nr_support: true,
eps_5gs_interworking: "with_n26",
/* 15 bearers support */
fifteen_bearers: false,
ims_list: [
{
ims_addr: "{{ slap_configuration['configuration.ims_addr'] }}",
bind_addr: "{{ slap_configuration['configuration.ims_bind'] }}"
}
],
/* AMF slices configuration */
/*nssai: [
{
sst: 1,
},
{
sst: 2,
},
{
sst: 3,
sd: 50,
}
],*/
/* ePDG configuration */
//epdg: {
// bind_addr: "127.0.1.100:500",
// esp_duration: 900,
// certificate: "epdg.pem",
// /* required for some buggy Mediatek phones */
// //omit_auth_in_first_auth_rsp: true
//},
/* Public Data Networks. The first one is the default. */
pdn_list: [
{
pdn_type: "ipv4",
tun_ifname: "{{ slap_configuration['tun-name'] }}",
access_point_name: ["default", "internet", "ims", "sos"],
first_ip_addr: "{{ netaddr.IPAddress(netaddr.IPNetwork(slap_configuration['tun-ipv4-network']).first) + 2 }}",
last_ip_addr: "{{ netaddr.IPAddress(netaddr.IPNetwork(slap_configuration['tun-ipv4-network']).last) - 1 }}",
ip_addr_shift: 2, /* difference between allocated IP addresses is 4 */
dns_addr: "8.8.8.8", /* Google DNS address */
// TODO: enable this instead of Google's DNS above
// dns_addr: "{{ slap_configuration['tun-ipv4-addr'] }}",
p_cscf_addr: ["{{ slap_configuration['tun-ipv4-addr'] }}"],
erabs: [
{
qci: 9,
priority_level: 15,
pre_emption_capability: "shall_not_trigger_pre_emption",
pre_emption_vulnerability: "not_pre_emptable",
},
],
},
],
/* Setup script for the network interface.
If no script is given, no network interface is created.
Script is called for each PDN with following parameters:
1) Interface name
2) PDN index
3) Access Point Name
4) IP version: 'ipv4' or 'ipv6'
5) IP address: first IP address for ipv4 and link local address for IPv6
6) First IP address
7) Last IP address
SlapOS: We suplement the script by preparing the TUN interface in slapos format phase.
Gotcha: The script has to be presented
*/
tun_setup_script: "{{ ifup_empty }}",
/* If true, inter-UE routing is done inside the MME (hence no IP
packet is output to the virtual network interface in case of
inter-UE communication). Otherwise, it is done by the Linux IP
layer. */
ue_to_ue_forwarding: false,
/* NAS ciphering algorithm preference. EEA0 is always the last. */
nas_cipher_algo_pref: [ ],
/* NAS integrity algorithm preference. EIA0 is always the last. */
nas_integ_algo_pref: [ 2, 1 ],
include "{{ slap_configuration['ue_db_path'] }}",
/* persistent user database */
ue_db_filename: "{{ directory['var'] }}/lte_ue.db"
}
{%- set filtered_slave_instance_list = [] %}
{%- for slave_instance in slave_instance_list %}
{%- if slave_instance.get('imsi', '') != '' %}
{%- do filtered_slave_instance_list.append(slave_instance) %}
{%- endif %}
{%- endfor -%}
ue_db: [
{%- for i, slave_instance in enumerate(filtered_slave_instance_list) %}
{%- if i == 0 -%}
{
{%- else -%}
, {
{%- endif %}
sim_algo: "{{ slave_instance.get('sim_algo', 'milenage') }}",
imsi: "{{ slave_instance.get('imsi', '') }}",
opc: "{{ slave_instance.get('opc', '') }}",
amf: {{ slave_instance.get('amf', '0x9001') }},
sqn: "{{ slave_instance.get('sqn', '000000000000') }}",
K: "{{ slave_instance.get('k', '') }}",
impu: "{{ slave_instance.get('impu', '') }}",
impi: "{{ slave_instance.get('impi', '') }}",
}
{%- endfor -%}
]
[buildout]
parts =
directory
lte-enb-request
lte-epc-request
publish-connection-information
extends = {{ monitor_template }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
[directory]
recipe = slapos.cookbook:mkdirectory
software = {{ buildout_directory }}
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
etc = ${:home}/etc
bin = ${:home}/bin
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[request-common-base]
recipe = slapos.cookbook:request.serialised
software-url = {{ slap_connection['software-release-url'] }}
server-url = {{ slap_connection['server-url'] }}
computer-id = {{ slap_connection['computer-id'] }}
partition-id = {{ slap_connection['partition-id'] }}
key-file = {{ slap_connection['key-file'] }}
cert-file = {{ slap_connection['cert-file'] }}
config-monitor-password = ${monitor-htpasswd:passwd}
return = monitor-base-url
[lte-epc-request]
<= request-common-base
name = EPC
software-type = epc
config-name = epc
[lte-enb-request]
<= request-common-base
name = eNB
software-type = enb
config-name = enb
{% if slapparameter_dict.get("tx_gain", None) %}
config-tx_gain = {{ dumps(slapparameter_dict["tx_gain"]) }}
{% endif %}
{% if slapparameter_dict.get("rx_gain", None) %}
config-rx_gain = {{ dumps(slapparameter_dict["rx_gain"]) }}
{% endif %}
{% if slapparameter_dict.get("dl_earfcn", None) %}
config-dl_earfcn = {{ dumps(slapparameter_dict["dl_earfcn"]) }}
{% endif %}
{% if slapparameter_dict.get("n_rb_dl", None) %}
config-n_rb_dl = {{ dumps(slapparameter_dict["n_rb_dl"]) }}
{% endif %}
[monitor-base-url-dict]
lte-epc-request = ${lte-epc-request:connection-monitor-base-url}
lte-enb-request = ${lte-enb-request:connection-monitor-base-url}
[publish-connection-information]
recipe = slapos.cookbook:publish
<= monitor-publish
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": {{ default_lte_tx_gain }}
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": {{ default_lte_rx_gain }}
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": {{ default_dl_earfcn }}
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": {{ default_lte_n_rb_dl }}
}
}
}
[buildout]
parts =
directory
ltelogs
lte-enb-config
lte-enb-service
# Temporarily extend monitor-base until promises are added
monitor-base
publish-connection-information
extends = {{ monitor_template }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
configuration.ims_addr = 127.0.0.1
configuration.ims_bind = 127.0.0.2
configuration.enb_addr = 127.0.1.1
configuration.default_dl_earfcn = {{ default_dl_earfcn }}
configuration.default_lte_dl_freq = {{ default_lte_dl_freq }}
configuration.default_lte_band = {{ default_lte_band }}
configuration.default_lte_n_rb_dl = {{ default_lte_n_rb_dl }}
configuration.default_lte_tx_gain = {{ default_lte_tx_gain }}
configuration.default_lte_rx_gain = {{ default_lte_rx_gain }}
[directory]
recipe = slapos.cookbook:mkdirectory
software = {{ buildout_directory }}
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
etc = ${:home}/etc
bin = ${:home}/bin
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[ltelogs]
recipe = slapos.recipe.template:jinja2
template = {{ ltelogs_template }}
rendered = ${directory:home}/ltelogs.sh
mode = 0775
extensions = jinja2.ext.do
context =
section directory directory
### eNodeB (enb)
[lte-enb-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
command-line = {{ enb }}/lteenb ${directory:etc}/enb.cfg
wrapper-path = ${directory:service}/lte-enb
mode = 0775
reserve-cpu = True
pidfile = ${directory:run}/enb.pid
hash-files =
${lte-enb-config:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
[config-base]
recipe = slapos.recipe.template:jinja2
mode = 0664
extensions = jinja2.ext.do
context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
import netaddr netaddr
[lte-enb-config]
<= config-base
template = {{ enb_template }}
rendered = ${directory:etc}/enb.cfg
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
[monitor-instance-parameter]
monitor-title = {{ slapparameter_dict['name'] | string }}
password = {{ slapparameter_dict['monitor-password'] | string }}
{
"$schema": "http://json-schema.org/draft-04/schema",
"title": "EPC Input Parameters",
"type": "object",
"properties": {}
}
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "EPC SIM Card Parameters",
"required": [
"sim_algo",
"imsi",
"opc",
"amf",
"sqn",
"k",
"impu",
"impi"
],
"properties": {
"sim_algo": {
"title": "Sim Algorithm",
"description": "xor, milenage or tuak. Set the USIM authentication algorithm.",
"type": "string",
"default": "milenage"
},
"imsi": {
"title": "IMSI",
"description": "IMSI",
"type": "string",
"default": ""
},
"opc": {
"title": "OPC",
"description": "Operator key preprocessed with the user secret key (as a 16 byte hexadecimal string). When the Milenage authentication algorithm is used, opc must be set.",
"type": "string",
"default": ""
},
"amf": {
"title": "AMF",
"description": "Range: 0 to 65535. Set the Authentication Management Field.",
"type": "string",
"default": "0x9001"
},
"sqn": {
"title": "SQN",
"description": "Optional String (6 byte hexadecimal string). Set the initial sequence number. For the XOR algorithm, the actual value does not matter. For the Milenage or TUAK algorithm, a sequence number resynchronization is initiated if the sequence number does not match the one stored in the USIM.",
"type": "string",
"default": "000000000000"
},
"k": {
"title": "K",
"description": "Set the user secret key (as a 16 bytes hexadecimal string, or eventually 32 bytes hexadecimal string for TUAK).",
"type": "string",
"default": ""
},
"impu": {
"title": "IMPU",
"description": "sip URI or a telephone number. Note that sip URI must not include hostname. If IMPU does not start by a scheme, it is assumed to be a sip URI.",
"type": "string",
"default": ""
},
"impi": {
"title": "IMPI",
"description": "Defines user IMPI. Must be fully filled with hostname if necessary.",
"type": "string",
"default": ""
}
}
}
[buildout]
parts =
directory
ltelogs
lte-mme-config
lte-mme-service
# Temporarily extend monitor-base until promises are added
monitor-base
publish-connection-information
extends = {{ monitor_template }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.log_size = 50M
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
configuration.mme_addr = 127.0.1.100
configuration.ims_addr = 127.0.0.1
configuration.ims_bind = 127.0.0.2
configuration.enb_addr = 127.0.1.1
ue_db_path = {{ ue_db_path }}
[directory]
recipe = slapos.cookbook:mkdirectory
software = {{ buildout_directory }}
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
etc = ${:home}/etc
bin = ${:home}/bin
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[ltelogs]
recipe = slapos.recipe.template:jinja2
template = {{ ltelogs_template }}
rendered = ${directory:home}/ltelogs.sh
mode = 0775
extensions = jinja2.ext.do
context =
section directory directory
### IMS
[lte-ims-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/ims.log; sleep 1
command-line = rm -f ${directory:var}/lte_ue.db; {{ mme }}/lteims ${directory:etc}/ims.cfg
wrapper-path = ${directory:service}/lte-ims
mode = 0775
pidfile = ${directory:run}/ims.pid
hash-files =
${lte-ims-config:rendered}
{{ ue_db_path }}
environment = AMARISOFT_PATH=/opt/amarisoft/.amarisoft
### MME
[lte-mme-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/mme.log
command-line = {{ mme }}/ltemme ${directory:etc}/mme.cfg
wrapper-path = ${directory:service}/lte-mme
mode = 0775
pidfile = ${directory:run}/mme.pid
hash-files =
${lte-mme-config:rendered}
{{ ue_db_path }}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib:{{ nghttp2_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
### EMPTY mme-ifup script
[lte-mme-ifup-empty]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/mme-ifup-empty
command-line = echo Using interface
mode = 775
[config-base]
recipe = slapos.recipe.template:jinja2
mode = 0664
extensions = jinja2.ext.do
context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
import netaddr netaddr
[lte-ims-config]
<= config-base
template = {{ ims_template }}
rendered = ${directory:etc}/ims.cfg
[lte-mme-config]
<= config-base
template = {{ mme_template }}
rendered = ${directory:etc}/mme.cfg
context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
import netaddr netaddr
key ifup_empty lte-mme-ifup-empty:wrapper-path
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
{% if slapparameter_dict.get("monitor-password", None) %}
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
{% else %}
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
[monitor-instance-parameter]
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif %}
[buildout]
parts =
directory
lte-gnb-request
lte-epc-request
publish-connection-information
extends = {{ monitor_template }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
[directory]
recipe = slapos.cookbook:mkdirectory
software = {{ buildout_directory }}
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
etc = ${:home}/etc
bin = ${:home}/bin
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[request-common-base]
recipe = slapos.cookbook:request.serialised
software-url = {{ slap_connection['software-release-url'] }}
server-url = {{ slap_connection['server-url'] }}
computer-id = {{ slap_connection['computer-id'] }}
partition-id = {{ slap_connection['partition-id'] }}
key-file = {{ slap_connection['key-file'] }}
cert-file = {{ slap_connection['cert-file'] }}
config-monitor-password = ${monitor-htpasswd:passwd}
return = monitor-base-url
[lte-epc-request]
<= request-common-base
name = EPC
software-type = epc
config-name = epc
[lte-gnb-request]
<= request-common-base
name = gNB
software-type = gnb
config-name = gnb
{% if slapparameter_dict.get("tx_gain", None) %}
config-tx_gain = {{ dumps(slapparameter_dict["tx_gain"]) }}
{% endif %}
{% if slapparameter_dict.get("rx_gain", None) %}
config-rx_gain = {{ dumps(slapparameter_dict["rx_gain"]) }}
{% endif %}
{% if slapparameter_dict.get("dl_nr_arfcn", None) %}
config-dl_nr_arfcn = {{ dumps(slapparameter_dict["dl_nr_arfcn"]) }}
{% endif %}
{% if slapparameter_dict.get("nr_band", None) %}
config-nr_band = {{ dumps(slapparameter_dict["nr_band"]) }}
{% endif %}
{% if slapparameter_dict.get("nr_bandwidth", None) %}
config-nr_bandwidth = {{ dumps(slapparameter_dict["nr_bandwidth"]) }}
{% endif %}
[monitor-base-url-dict]
lte-epc-request = ${lte-epc-request:connection-monitor-base-url}
lte-gnb-request = ${lte-gnb-request:connection-monitor-base-url}
[publish-connection-information]
recipe = slapos.cookbook:publish
<= monitor-publish
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": {{ default_nr_tx_gain }}
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": {{ default_nr_rx_gain }}
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": {{ default_dl_nr_arfcn }}
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": {{ default_nr_band }}
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": {{ default_nr_bandwidth }}
}
}
}
[buildout]
parts =
directory
ltelogs
lte-gnb-config
lte-enb-service
# Temporarily extend monitor-base until promises are added
monitor-base
publish-connection-information
extends = {{ monitor_template }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
configuration.ims_addr = 127.0.0.1
configuration.ims_bind = 127.0.0.2
configuration.enb_addr = 127.0.1.1
configuration.default_dl_nr_arfcn = {{ default_dl_nr_arfcn }}
configuration.default_nr_band = {{ default_nr_band }}
configuration.default_nr_dl_freq = {{ default_nr_dl_freq }}
configuration.default_nr_bandwidth = {{ default_nr_bandwidth }}
configuration.default_nr_tx_gain = {{ default_nr_tx_gain }}
configuration.default_nr_rx_gain = {{ default_nr_rx_gain }}
[directory]
recipe = slapos.cookbook:mkdirectory
software = {{ buildout_directory }}
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
etc = ${:home}/etc
bin = ${:home}/bin
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[ltelogs]
recipe = slapos.recipe.template:jinja2
template = {{ ltelogs_template }}
rendered = ${directory:home}/ltelogs.sh
mode = 0775
extensions = jinja2.ext.do
context =
section directory directory
### eNodeB (enb)
[lte-enb-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
command-line = {{ enb }}/lteenb ${directory:etc}/gnb.cfg
wrapper-path = ${directory:service}/lte-enb
mode = 0775
reserve-cpu = True
pidfile = ${directory:run}/enb.pid
hash-files =
${lte-gnb-config:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
[config-base]
recipe = slapos.recipe.template:jinja2
mode = 0664
extensions = jinja2.ext.do
context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
import netaddr netaddr
[lte-gnb-config]
<= config-base
template = {{ gnb_template }}
rendered = ${directory:etc}/gnb.cfg
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
[monitor-instance-parameter]
monitor-title = {{ slapparameter_dict['name'] | string }}
password = {{ slapparameter_dict['monitor-password'] | string }}
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 38350
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 38350
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 378000
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 39
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 378000
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 39
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 38050
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 38050
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 520000
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 38
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 520000
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 38
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 42590
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 42590
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 632628
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 78
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 632628
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 78
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 44590
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 30
},
"dl_earfcn": {
"title": "DL EARFCN",
"description": "Downlink E-UTRA Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 44590
},
"n_rb_dl": {
"title": "DL RB",
"description": "number of DL resource blocks",
"type": "number",
"default": 100
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 646666
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 78
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"tx_gain": {
"title": "Tx gain",
"description": "Tx gain (in dB)",
"type": "number",
"default": 70
},
"rx_gain": {
"title": "Rx gain",
"description": "Rx gain (in dB)",
"type": "number",
"default": 50
},
"dl_nr_arfcn": {
"title": "DL NR ARFCN",
"description": "Downlink NR Absolute Radio Frequency Channel Number of the cell",
"type": "number",
"default": 646666
},
"nr_band": {
"title": "NR band",
"description": "NR band number",
"type": "number",
"default": 78
},
"nr_bandwidth": {
"title": "Bandwidth",
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
}
}
}
\ No newline at end of file
[buildout]
parts =
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
raw buildout_directory ${buildout:directory}
section slap_connection slap-connection
key slapparameter_dict slap-configuration:configuration
$${:extra-context}
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
enb-epc = dynamic-template-lte-enb-epc:rendered
gnb-epc = dynamic-template-lte-gnb-epc:rendered
enb = dynamic-template-lte-enb:rendered
gnb = dynamic-template-lte-gnb:rendered
epc = dynamic-template-lte-epc:rendered
RootSoftwareInstance = $${:enb-epc}
[dynamic-template-lte-enb-epc]
< = jinja2-template-base
template = ${template-lte-enb-epc:target}
filename = instance-lte-enb-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
[dynamic-template-lte-gnb-epc]
< = jinja2-template-base
template = ${template-lte-gnb-epc:target}
filename = instance-lte-gnb-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
[dynamic-template-lte-enb]
< = jinja2-template-base
template = ${template-lte-enb:target}
filename = instance-lte-enb.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
raw enb ${enb:destination}
raw enb_template ${enb.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw openssl_location ${openssl:location}
raw default_dl_earfcn ${enb:default-dl-earfcn}
raw default_lte_dl_freq ${enb:default-lte-dl-freq}
raw default_lte_band ${enb:default-lte-band}
raw default_lte_n_rb_dl ${enb:default-lte-n-rb-dl}
raw default_lte_tx_gain ${enb:default-lte-tx-gain}
raw default_lte_rx_gain ${enb:default-lte-rx-gain}
raw min_frequency ${enb:min-frequency}
raw max_frequency ${enb:max-frequency}
[dynamic-template-lte-gnb]
< = jinja2-template-base
template = ${template-lte-gnb:target}
filename = instance-lte-gnb.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
raw enb ${enb:destination}
raw gnb_template ${gnb.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw openssl_location ${openssl:location}
raw default_dl_nr_arfcn ${enb:default-dl-nr-arfcn}
raw default_nr_band ${enb:default-nr-band}
raw default_nr_dl_freq ${enb:default-nr-dl-freq}
raw default_nr_bandwidth ${enb:default-nr-bandwidth}
raw default_nr_tx_gain ${enb:default-nr-tx-gain}
raw default_nr_rx_gain ${enb:default-nr-rx-gain}
raw min_frequency ${enb:min-frequency}
raw max_frequency ${enb:max-frequency}
[dynamic-template-lte-epc]
< = jinja2-template-base
template = ${template-lte-epc:target}
filename = instance-lte-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
raw mme ${mme:destination}
raw mme_template ${mme.jinja2.cfg:target}
raw ims_template ${ims.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw openssl_location ${openssl:location}
raw nghttp2_location ${nghttp2:location}
key ue_db_path ue-db-config:rendered
[ue-db-config]
recipe = slapos.recipe.template:jinja2
template = ${ue_db.jinja2.cfg:target}
filename = ue_db.cfg
extensions = jinja2.ext.do
rendered = $${buildout:directory}/$${:filename}
context =
key slave_instance_list slap-configuration:slave-instance-list
#!/bin/bash
# Copyright (C) 2012-2015 Amarisoft
# LTE system logger version 2016-10-13
# Path for multi environment support
export PATH="$PATH:/bin/:/usr/bin/:/usr/local/bin"
while [ "$1" != "" ] ; do
if [ -e "$1" ] ; then
# Avoid storing logs with comments only
HAS_LOG=$(grep -v -l "#" $1)
if [ "$HAS_LOG" != "" ] ; then
DATE=$(date -u +%Y%m%d.%H:%M:%S | sed -e "s/ /-/g")
FILE=$(basename $1)
mv $1 "{{ directory['log'] }}/${FILE}.${DATE}"
else
rm -f $1
fi
fi
shift
done
#!/usr/bin/env python3
from jinja2 import Template
import os
# Values: 6 (1.4 MHz), 15 (3MHz), 25 (5MHz), 50 (10MHz), 75 (15MHz), 100 (20MHz)
LTE_N_RB_DL=100
LTE_TX_GAIN=70
LTE_RX_GAIN=30
NR_BANDWIDTH=40
NR_TX_GAIN=70
NR_RX_GAIN=50
global_context = {
'generated_file_message': "This file was generated using a jinja2 template and the render-templates script, don't modify directly."
}
rf_mode_context_list = [
{
'rf_mode': 'tdd1900',
'default_dl_earfcn': 38350,
'default_lte_dl_freq': 1890.0,
'default_lte_band': 39,
'default_lte_n_rb_dl': LTE_N_RB_DL,
'default_lte_tx_gain': LTE_TX_GAIN,
'default_lte_rx_gain': LTE_RX_GAIN,
'default_dl_nr_arfcn': 378000,
'default_nr_band': 39,
'default_nr_dl_freq': 1890.0,
'default_nr_bandwidth': NR_BANDWIDTH,
'default_nr_tx_gain': NR_TX_GAIN,
'default_nr_rx_gain': NR_RX_GAIN,
'min_frequency': 1880.0,
'max_frequency': 1920,
}, {
'rf_mode': 'tdd2600',
'default_dl_earfcn': 38050,
'default_lte_dl_freq': 2600.0,
'default_lte_band': 38,
'default_lte_n_rb_dl': LTE_N_RB_DL,
'default_lte_tx_gain': LTE_TX_GAIN,
'default_lte_rx_gain': LTE_RX_GAIN,
'default_dl_nr_arfcn': 520000,
'default_nr_band': 38,
'default_nr_dl_freq': 2600.0,
'default_nr_bandwidth': NR_BANDWIDTH,
'default_nr_tx_gain': NR_TX_GAIN,
'default_nr_rx_gain': NR_RX_GAIN,
'min_frequency': 2570,
'max_frequency': 2620,
}, {
'rf_mode': 'tdd3500',
'default_dl_earfcn': 42590,
'default_lte_dl_freq': 3500.0,
'default_lte_band': 42,
'default_lte_n_rb_dl': LTE_N_RB_DL,
'default_lte_tx_gain': LTE_TX_GAIN,
'default_lte_rx_gain': LTE_RX_GAIN,
'default_dl_nr_arfcn': 632628,
'default_nr_band': 78,
'default_nr_dl_freq': 3489.42,
'default_nr_bandwidth': NR_BANDWIDTH,
'default_nr_tx_gain': NR_TX_GAIN,
'default_nr_rx_gain': NR_RX_GAIN,
'min_frequency': 3400,
'max_frequency': 3600,
}, {
'rf_mode': 'tdd3700',
'default_dl_earfcn': 44590,
'default_lte_dl_freq': 3700.0,
'default_lte_band': 43,
'default_lte_n_rb_dl': LTE_N_RB_DL,
'default_lte_tx_gain': LTE_TX_GAIN,
'default_lte_rx_gain': LTE_RX_GAIN,
'default_dl_nr_arfcn': 646666,
'default_nr_band': 78,
'default_nr_dl_freq': 3699.99,
'default_nr_bandwidth': NR_BANDWIDTH,
'default_nr_tx_gain': NR_TX_GAIN,
'default_nr_rx_gain': NR_RX_GAIN,
'min_frequency': 3600,
'max_frequency': 3800,
}
]
with open('software.cfg.jinja2.json', 'r') as f:
software_json_template = Template(f.read())
with open('instance-enb-input-schema.jinja2.json', 'r') as f:
instance_enb_json_template = Template(f.read())
with open('instance-gnb-input-schema.jinja2.json', 'r') as f:
instance_gnb_json_template = Template(f.read())
with open('software.jinja2.cfg', 'r') as f:
software_template = Template(f.read())
for rf_mode_context in rf_mode_context_list:
with open('software-{}.cfg.json'.format(rf_mode_context['rf_mode']),
'w+') as f:
f.write(software_json_template.render(**rf_mode_context, **global_context))
with open('software-{}.cfg'.format(rf_mode_context['rf_mode']),
'w+') as f:
f.write(software_template.render(**rf_mode_context, **global_context))
for software_type in ['enb', 'enb-epc', 'gnb', 'gnb-epc']:
with open('instance-{}-{}-input-schema.json'.format(
rf_mode_context['rf_mode'],
software_type),
'w+') as f:
if software_type in ['enb', 'enb-epc']:
f.write(instance_enb_json_template.render(**rf_mode_context, **global_context))
else:
f.write(instance_gnb_json_template.render(**rf_mode_context, **global_context))
# This file was generated using a jinja2 template and the render-templates script, don't modify directly.
[buildout]
extends =
software.cfg
[enb]
default-dl-earfcn = 38350
default-lte-dl-freq = 1890.0
default-lte-band = 39
default-lte-n-rb-dl = 100
default-lte-tx-gain = 70
default-lte-rx-gain = 30
default-dl-nr-arfcn = 378000
default-nr-band = 39
default-nr-dl-freq = 1890.0
default-nr-bandwidth = 40
default-nr-tx-gain = 70
default-nr-rx-gain = 50
min-frequency = 1880.0
max-frequency = 1920
\ No newline at end of file
{
"name": "ORS Amarisoft",
"description": "4G and 5G amarisoft stack for ORS",
"serialisation": "xml",
"software-type": {
"enb": {
"title": "eNB",
"software-type": "enb",
"description": "eNodeB Configuration",
"request": "instance-tdd1900-enb-input-schema.json",
"response": "instance-tdd1900-enb-schema.json",
"index": 0
},
"enb-epc": {
"title": "eNB and EPC",
"software-type": "enb-epc",
"description": "eNodeB and EPC Configuration",
"request": "instance-tdd1900-enb-epc-input-schema.json",
"response": "instance-tdd1900-enb-epc-schema.json",
"index": 1
},
"gnb": {
"title": "gNB",
"software-type": "gnb",
"description": "gNodeB Configuration",
"request": "instance-tdd1900-gnb-input-schema.json",
"response": "instance-tdd1900-gnb-schema.json",
"index": 2
},
"gnb-epc": {
"title": "gNB and EPC",
"software-type": "gnb-epc",
"description": "gNodeB and EPC Configuration",
"request": "instance-tdd1900-gnb-epc-input-schema.json",
"response": "instance-tdd1900-gnb-epc-schema.json",
"index": 3
},
"epc": {
"title": "EPC",
"software-type": "epc",
"description": "EPC Configuration",
"request": "instance-epc-input-schema.json",
"response": "instance-epc-schema.json",
"index": 4
},
"epc-slave": {
"title": "Sim Card",
"description": "EPC Configuration",
"software-type": "epc",
"request": "instance-epc-slave-input-schema.json",
"response": "instance-epc-slave-schema.json",
"shared": true,
"index": 5
}
}
}
\ No newline at end of file
# This file was generated using a jinja2 template and the render-templates script, don't modify directly.
[buildout]
extends =
software.cfg
[enb]
default-dl-earfcn = 38050
default-lte-dl-freq = 2600.0
default-lte-band = 38
default-lte-n-rb-dl = 100
default-lte-tx-gain = 70
default-lte-rx-gain = 30
default-dl-nr-arfcn = 520000
default-nr-band = 38
default-nr-dl-freq = 2600.0
default-nr-bandwidth = 40
default-nr-tx-gain = 70
default-nr-rx-gain = 50
min-frequency = 2570
max-frequency = 2620
\ No newline at end of file
{
"name": "ORS Amarisoft",
"description": "4G and 5G amarisoft stack for ORS",
"serialisation": "xml",
"software-type": {
"enb": {
"title": "eNB",
"software-type": "enb",
"description": "eNodeB Configuration",
"request": "instance-tdd2600-enb-input-schema.json",
"response": "instance-tdd2600-enb-schema.json",
"index": 0
},
"enb-epc": {
"title": "eNB and EPC",
"software-type": "enb-epc",
"description": "eNodeB and EPC Configuration",
"request": "instance-tdd2600-enb-epc-input-schema.json",
"response": "instance-tdd2600-enb-epc-schema.json",
"index": 1
},
"gnb": {
"title": "gNB",
"software-type": "gnb",
"description": "gNodeB Configuration",
"request": "instance-tdd2600-gnb-input-schema.json",
"response": "instance-tdd2600-gnb-schema.json",
"index": 2
},
"gnb-epc": {
"title": "gNB and EPC",
"software-type": "gnb-epc",
"description": "gNodeB and EPC Configuration",
"request": "instance-tdd2600-gnb-epc-input-schema.json",
"response": "instance-tdd2600-gnb-epc-schema.json",
"index": 3
},
"epc": {
"title": "EPC",
"software-type": "epc",
"description": "EPC Configuration",
"request": "instance-epc-input-schema.json",
"response": "instance-epc-schema.json",
"index": 4
},
"epc-slave": {
"title": "Sim Card",
"description": "EPC Configuration",
"software-type": "epc",
"request": "instance-epc-slave-input-schema.json",
"response": "instance-epc-slave-schema.json",
"shared": true,
"index": 5
}
}
}
\ No newline at end of file
# This file was generated using a jinja2 template and the render-templates script, don't modify directly.
[buildout]
extends =
software.cfg
[enb]
default-dl-earfcn = 42590
default-lte-dl-freq = 3500.0
default-lte-band = 42
default-lte-n-rb-dl = 100
default-lte-tx-gain = 70
default-lte-rx-gain = 30
default-dl-nr-arfcn = 632628
default-nr-band = 78
default-nr-dl-freq = 3489.42
default-nr-bandwidth = 40
default-nr-tx-gain = 70
default-nr-rx-gain = 50
min-frequency = 3400
max-frequency = 3600
\ No newline at end of file
{
"name": "ORS Amarisoft",
"description": "4G and 5G amarisoft stack for ORS",
"serialisation": "xml",
"software-type": {
"enb": {
"title": "eNB",
"software-type": "enb",
"description": "eNodeB Configuration",
"request": "instance-tdd3500-enb-input-schema.json",
"response": "instance-tdd3500-enb-schema.json",
"index": 0
},
"enb-epc": {
"title": "eNB and EPC",
"software-type": "enb-epc",
"description": "eNodeB and EPC Configuration",
"request": "instance-tdd3500-enb-epc-input-schema.json",
"response": "instance-tdd3500-enb-epc-schema.json",
"index": 1
},
"gnb": {
"title": "gNB",
"software-type": "gnb",
"description": "gNodeB Configuration",
"request": "instance-tdd3500-gnb-input-schema.json",
"response": "instance-tdd3500-gnb-schema.json",
"index": 2
},
"gnb-epc": {
"title": "gNB and EPC",
"software-type": "gnb-epc",
"description": "gNodeB and EPC Configuration",
"request": "instance-tdd3500-gnb-epc-input-schema.json",
"response": "instance-tdd3500-gnb-epc-schema.json",
"index": 3
},
"epc": {
"title": "EPC",
"software-type": "epc",
"description": "EPC Configuration",
"request": "instance-epc-input-schema.json",
"response": "instance-epc-schema.json",
"index": 4
},
"epc-slave": {
"title": "Sim Card",
"description": "EPC Configuration",
"software-type": "epc",
"request": "instance-epc-slave-input-schema.json",
"response": "instance-epc-slave-schema.json",
"shared": true,
"index": 5
}
}
}
\ No newline at end of file
# This file was generated using a jinja2 template and the render-templates script, don't modify directly.
[buildout]
extends =
software.cfg
[enb]
default-dl-earfcn = 44590
default-lte-dl-freq = 3700.0
default-lte-band = 43
default-lte-n-rb-dl = 100
default-lte-tx-gain = 70
default-lte-rx-gain = 30
default-dl-nr-arfcn = 646666
default-nr-band = 78
default-nr-dl-freq = 3699.99
default-nr-bandwidth = 40
default-nr-tx-gain = 70
default-nr-rx-gain = 50
min-frequency = 3600
max-frequency = 3800
\ No newline at end of file
{
"name": "ORS Amarisoft",
"description": "4G and 5G amarisoft stack for ORS",
"serialisation": "xml",
"software-type": {
"enb": {
"title": "eNB",
"software-type": "enb",
"description": "eNodeB Configuration",
"request": "instance-tdd3700-enb-input-schema.json",
"response": "instance-tdd3700-enb-schema.json",
"index": 0
},
"enb-epc": {
"title": "eNB and EPC",
"software-type": "enb-epc",
"description": "eNodeB and EPC Configuration",
"request": "instance-tdd3700-enb-epc-input-schema.json",
"response": "instance-tdd3700-enb-epc-schema.json",
"index": 1
},
"gnb": {
"title": "gNB",
"software-type": "gnb",
"description": "gNodeB Configuration",
"request": "instance-tdd3700-gnb-input-schema.json",
"response": "instance-tdd3700-gnb-schema.json",
"index": 2
},
"gnb-epc": {
"title": "gNB and EPC",
"software-type": "gnb-epc",
"description": "gNodeB and EPC Configuration",
"request": "instance-tdd3700-gnb-epc-input-schema.json",
"response": "instance-tdd3700-gnb-epc-schema.json",
"index": 3
},
"epc": {
"title": "EPC",
"software-type": "epc",
"description": "EPC Configuration",
"request": "instance-epc-input-schema.json",
"response": "instance-epc-schema.json",
"index": 4
},
"epc-slave": {
"title": "Sim Card",
"description": "EPC Configuration",
"software-type": "epc",
"request": "instance-epc-slave-input-schema.json",
"response": "instance-epc-slave-schema.json",
"shared": true,
"index": 5
}
}
}
\ No newline at end of file
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/nghttp2/buildout.cfg
parts +=
template
slapos-cookbook
ltelogs.jinja2.sh
# copy all configs by default
mme.jinja2.cfg
ims.jinja2.cfg
enb.jinja2.cfg
gnb.jinja2.cfg
ue_db.jinja2.cfg
# sdr driver is dependent on ENB thus should be added explicitely by software.cfg
sdr-driver
lteenb-cap-sys-nice
lteenb-avx2-cap-sys-nice
# unimplemented parts - the http monitor and better log handling using logrotate
# apache-php
# logrotate
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-lte-enb-epc]
<= download-base
[template-lte-gnb-epc]
<= download-base
[template-lte-enb]
<= download-base
[template-lte-gnb]
<= download-base
[template-lte-epc]
<= download-base
[amarisoft]
recipe = slapos.recipe.build
path = /opt/amarisoft/lte
init =
import os
options['lte-version'] = os.readlink(options["path"])[:-1]
[copy-to-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
[copy-config-to-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/config/${:_buildout_section_name_}
[unpack-to-instance]
recipe = slapos.recipe.build:download-unpacked
url = ${amarisoft:path}/lte${:_buildout_section_name_}-linux-${amarisoft:lte-version}.tar.gz
destination = ${buildout:directory}/${:_buildout_section_name_}
strip-top-level-dir = true
[enb.jinja2.cfg]
<= copy-config-to-instance
filename = enb.jinja2.cfg
[gnb.jinja2.cfg]
<= copy-config-to-instance
filename = gnb.jinja2.cfg
[ltelogs.jinja2.sh]
<= copy-to-instance
filename = ltelogs.jinja2.sh
[ue_db.jinja2.cfg]
<= copy-config-to-instance
filename = ue_db.jinja2.cfg
[mme.jinja2.cfg]
<= copy-config-to-instance
filename = mme.jinja2.cfg
[ims.jinja2.cfg]
<= copy-config-to-instance
filename = ims.jinja2.cfg
[sdr]
<= unpack-to-instance
url = ${amarisoft:path}/trx_${:_buildout_section_name_}-linux-${amarisoft:lte-version}.tar.gz
destination = ${enb:destination}/x86_64
md5sum = ${trx_sdr-linux:md5sum}
[sdr-driver]
# move trx_sdr.so next to lteenb binary
recipe = plone.recipe.command
command = cp -p ${sdr:destination}/trx_sdr.so ${enb:destination}
[enb]
<= unpack-to-instance
md5sum = ${lteenb-linux:md5sum}
[mme]
<= unpack-to-instance
md5sum = ${ltemme-linux:md5sum}
[lteenb-linux]
filename = ${amarisoft:lte-version}/lteenb-linux-${amarisoft:lte-version}.tar.gz
md5sum = 842b1526073472a30cb0b286d3b1528c
[ltemme-linux]
filename = ${amarisoft:lte-version}/ltemme-linux-${amarisoft:lte-version}.tar.gz
md5sum = 9d7917f90c7c7b2a8ba624d874595351
[ltewww-linux]
filename = ${amarisoft:lte-version}/ltewww-linux-${amarisoft:lte-version}.tar.gz
md5sum = 416b6167f70b12910fbbb9293038554c
[trx_sdr-linux]
filename = ${amarisoft:lte-version}/trx_sdr-linux-${amarisoft:lte-version}.tar.gz
md5sum = e6960e3460f1a32c2436f36b2082995d
[base-lteenb-cap-sys-nice]
recipe = plone.recipe.command
command =
getcap ${amarisoft:path}/${:binary} | grep cap_sys_nice+ep && exit 0;
# Make a copy or restore the copy, as patchelf will irreversibly change the md5sum
stat ${enb:destination}/${:binary}-unpriviledged ||
cp ${enb:destination}/${:binary} ${enb:destination}/${:binary}-unpriviledged &&
cp ${enb:destination}/${:binary}-unpriviledged ${enb:destination}/${:binary}
# ORS are pre-configured to have sudo allow slapsoft to run give-cap-sys-nice-lteenb script
# with root permissions
sudo -n ${amarisoft:path}/../give-cap-sys-nice-lteenb ${enb:destination}/${:binary} || true;
update-command = ${:command}
[lteenb-avx2-cap-sys-nice]
<= base-lteenb-cap-sys-nice
binary=lteenb
[lteenb-cap-sys-nice]
<= base-lteenb-cap-sys-nice
binary=lteenb-avx2
{
"name": "ORS Amarisoft",
"description": "4G and 5G amarisoft stack for ORS",
"serialisation": "xml",
"software-type": {
"enb": {
"title": "eNB",
"software-type": "enb",
"description": "eNodeB Configuration",
"request": "instance-{{ rf_mode }}-enb-input-schema.json",
"response": "instance-{{ rf_mode }}-enb-schema.json",
"index": 0
},
"enb-epc": {
"title": "eNB and EPC",
"software-type": "enb-epc",
"description": "eNodeB and EPC Configuration",
"request": "instance-{{ rf_mode }}-enb-epc-input-schema.json",
"response": "instance-{{ rf_mode }}-enb-epc-schema.json",
"index": 1
},
"gnb": {
"title": "gNB",
"software-type": "gnb",
"description": "gNodeB Configuration",
"request": "instance-{{ rf_mode }}-gnb-input-schema.json",
"response": "instance-{{ rf_mode }}-gnb-schema.json",
"index": 2
},
"gnb-epc": {
"title": "gNB and EPC",
"software-type": "gnb-epc",
"description": "gNodeB and EPC Configuration",
"request": "instance-{{ rf_mode }}-gnb-epc-input-schema.json",
"response": "instance-{{ rf_mode }}-gnb-epc-schema.json",
"index": 3
},
"epc": {
"title": "EPC",
"software-type": "epc",
"description": "EPC Configuration",
"request": "instance-epc-input-schema.json",
"response": "instance-epc-schema.json",
"index": 4
},
"epc-slave": {
"title": "Sim Card",
"description": "EPC Configuration",
"software-type": "epc",
"request": "instance-epc-slave-input-schema.json",
"response": "instance-epc-slave-schema.json",
"shared": true,
"index": 5
}
}
}
# {{ generated_file_message }}
[buildout]
extends =
software.cfg
[enb]
default-dl-earfcn = {{ default_dl_earfcn }}
default-lte-dl-freq = {{ default_lte_dl_freq }}
default-lte-band = {{ default_lte_band }}
default-lte-n-rb-dl = {{ default_lte_n_rb_dl }}
default-lte-tx-gain = {{ default_lte_tx_gain }}
default-lte-rx-gain = {{ default_lte_rx_gain }}
default-dl-nr-arfcn = {{ default_dl_nr_arfcn }}
default-nr-band = {{ default_nr_band }}
default-nr-dl-freq = {{ default_nr_dl_freq }}
default-nr-bandwidth = {{ default_nr_bandwidth }}
default-nr-tx-gain = {{ default_nr_tx_gain }}
default-nr-rx-gain = {{ default_nr_rx_gain }}
min-frequency = {{ min_frequency }}
max-frequency = {{ max_frequency }}
......@@ -26,33 +26,27 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[template-powerdns]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-powerdns.cfg
mode = 0644
[template-pdns-configuration]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
[template-dns-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[iso-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-zones-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[versions]
PyRSS2Gen = 1.1
......
......@@ -15,11 +15,11 @@
[instance-profile]
filename = instance.cfg.in
md5sum = efb4238229681447aa7fe73898dffad4
md5sum = b62dd8dd89ef6627ebac20987a89a069
[instance-default]
filename = instance-default.cfg.in
md5sum = 4df64032e14c19363ad3dfe9aecf8e0c
md5sum = 16d592805c6c1756e54924242148ba85
[proftpd-config-file]
filename = proftpd-config-file.cfg.in
......
......@@ -102,7 +102,7 @@ config-port = ${proftpd:sftp-port}
recipe = slapos.cookbook:wrapper
wrapper-path =${buildout:bin-directory}/${:_buildout_section_name_}
command-line =
{{ perl_bin }} {{ ftpasswd_bin }} --passwd --home=${proftpd:data-dir} --shell=/bin/false --uid=${proftpd-userinfo:pw-uid} --gid=${proftpd-userinfo:gr-gid} --file ${auth-user-file:output}
{{ ftpasswd_bin }} --passwd --home=${proftpd:data-dir} --shell=/bin/false --uid=${proftpd-userinfo:pw-uid} --gid=${proftpd-userinfo:gr-gid} --file ${auth-user-file:output}
[auth-user-file]
recipe = plone.recipe.command
......
......@@ -24,7 +24,6 @@ context =
raw ftpasswd_bin {{ ftpasswd_bin }}
raw ftpdctl_bin {{ ftpdctl_bin }}
raw ssh_keygen_bin {{ ssh_keygen_bin }}
raw perl_bin {{ perl_bin }}
raw template_monitor {{ template_monitor }}
[instance-default]
......
......@@ -18,8 +18,6 @@ part = python3
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[proftpd-config-file]
<= download-file-base
......@@ -31,7 +29,6 @@ mode = 0644
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/instance.cfg
mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
......@@ -40,10 +37,7 @@ context =
key ftpasswd_bin proftpd-output:ftpasswd
key ftpdctl_bin proftpd-output:ftpdctl
key ssh_keygen_bin openssh-output:keygen
key perl_bin proftpd-output:perl
raw template_monitor ${monitor2-template:rendered}
[versions]
collective.recipe.environment = 1.1.0
collective.recipe.grp = 1.1.0
plone.recipe.command = 1.1
......@@ -476,6 +476,5 @@ class TestBanLog(ProFTPdTestCase, LogRotationMixin):
expected_logged_text = 'denied due to host ban'
def _access(self) -> None:
for _ in range(6):
with self.assertRaisesRegex(
Exception, '(Authentication failed|Connection reset by peer)'):
with self.assertRaises(Exception):
self._getConnection(password='wrong')
......@@ -72,7 +72,6 @@ context =
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 644
[template]
recipe = slapos.recipe.template:jinja2
......
......@@ -62,7 +62,6 @@ extra-paths +=
[download-base-part]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 644
[template-erp5]
< = download-base-part
......
......@@ -234,6 +234,7 @@ setup = ${slapos.core-repository:location}
eggs += ${:extra-eggs}
extra-eggs =
${lxml-python:egg}
${python-PyYAML:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
${pycurl:egg}
......
......@@ -115,6 +115,7 @@ setup = ${rubygemsrecipe-repository:location}
<= python-interpreter
eggs +=
${lxml-python:egg}
${python-PyYAML:egg}
${python-cryptography:egg}
${backports.lzma:egg}
${pycurl:egg}
......
......@@ -14,11 +14,11 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
md5sum = ba3539d959143a3be76fcb54196a3aa8
[template-runner]
filename = instance-runner.cfg
md5sum = 384285ab789396b6e674a8125ce2d030
md5sum = efa0d1647dee2689485264daf256c76d
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......
......@@ -362,7 +362,7 @@ work_dir = $${slaprunner:working-directory}
[nginx_conf]
recipe = slapos.recipe.template:jinja2
template = ${template_nginx_conf:location}/${template_nginx_conf:filename}
template = ${template_nginx_conf:target}
rendered = $${nginx-frontend:path_nginx_conf}
context =
key shellinabox_socket shellinabox:socket
......@@ -372,7 +372,7 @@ context =
[nginx-launcher]
recipe = slapos.recipe.template:jinja2
template = ${template_launcher:location}/${template_launcher:filename}
template = ${template_launcher:target}
rendered = $${nginx-frontend:bin_launcher}
mode = 700
context =
......@@ -404,7 +404,7 @@ httpd_cors_file = $${slaprunner-httpd-cors:location}
[httpd-conf]
recipe = slapos.recipe.template:jinja2
template = ${template_httpd_conf:location}/${template_httpd_conf:filename}
template = ${template_httpd_conf:target}
rendered = $${directory:etc}/httpd.conf
context =
section parameters httpd-parameters
......@@ -663,7 +663,7 @@ buildout-shared-folder = $${runnerdirectory:home}/shared
[slapos-cfg]
recipe = slapos.recipe.template:jinja2
template = ${template-slapos-cfg:location}/${template-slapos-cfg:filename}
template = ${template-slapos-cfg:target}
rendered = $${slaprunner:slapos.cfg}
mode = 700
context =
......@@ -673,7 +673,7 @@ context =
[slapos-test-cfg]
recipe = slapos.recipe.template:jinja2
template = ${template-slapos-cfg:location}/${template-slapos-cfg:filename}
template = ${template-slapos-cfg:target}
rendered = $${test-runner:slapos.cfg}
mode = 700
context =
......@@ -681,7 +681,7 @@ context =
[slapformat-definition.cfg]
recipe = slapos.recipe.template:jinja2
template = ${template-slapformat-definition.cfg:location}/${template-slapformat-definition.cfg:filename}
template = ${template-slapformat-definition.cfg:target}
rendered = $${slaprunner:slapformat-definition.cfg}
mode = 700
context =
......@@ -727,7 +727,7 @@ command = $${prepare-software:wrapper-path}
[instance-parameters]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
template = ${template-parameters:location}/${template-parameters:filename}
template = ${template-parameters:target}
rendered = $${directory:etc}/.parameter.xml.default
mode = 0644
context =
......@@ -759,7 +759,7 @@ stop-on-error = true
[bash-profile]
recipe = slapos.recipe.template:jinja2
template = ${template-bash-profile:location}/${template-bash-profile:filename}
template = ${template-bash-profile:target}
rendered = $${buildout:directory}/.bash_profile
context =
raw path $${shell-environment:path}
......@@ -826,7 +826,7 @@ slapformat-definition.cfg = $${slaprunner:slapformat-definition.cfg}
[supervisord-conf]
recipe = slapos.recipe.template:jinja2
template = ${template-supervisord:location}/${template-supervisord:filename}
template = ${template-supervisord:target}
rendered = $${directory:etc}/supervisord.conf
context =
import multiprocessing multiprocessing
......@@ -836,7 +836,7 @@ context =
[listener-slapgrid-bin]
recipe = slapos.recipe.template:jinja2
template = ${template-listener-slapgrid:location}/${template-listener-slapgrid:filename}
template = ${template-listener-slapgrid:target}
rendered = $${directory:bin}/listener_slapgrid.py
mode = 0744
context =
......@@ -905,7 +905,7 @@ private-path-list +=
[monitor-check-webrunner-internal-instance]
recipe = slapos.recipe.template:jinja2
template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
template = ${monitor-check-webrunner-internal-instance:target}
rendered = $${monitor-directory:bin}/$${:filename}
filename = monitor-check-webrunner-internal-instance
mode = 0744
......@@ -913,7 +913,7 @@ mode = 0744
## Slapuser slapos command script
[template-slapuser-script]
recipe = slapos.recipe.template:jinja2
template = ${template-slapuser-script:location}/${template-slapuser-script:filename}
template = ${template-slapuser-script:target}
rendered = $${buildout:bin-directory}/slapos
mode = 0744
context =
......
......@@ -63,7 +63,7 @@ context =
key slapparameter_dict slap-configuration:configuration
raw software_release_bin ${buildout:bin-directory}
raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:target}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw rsync_bin_folder ${rsync:location}/bin
......@@ -78,12 +78,12 @@ context =
key template_runner_path instance-base-runner:rendered
key slapparameter_dict slap-configuration:configuration
raw software_release_bin ${buildout:bin-directory}
raw importer_script_path ${template-runner-import-script:location}/${template-runner-import-script:filename}
raw importer_script_path ${template-runner-import-script:target}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw sqlite3_executable_location ${sqlite3:location}/bin/sqlite3
raw rsync_executable_location ${rsync:location}/bin/rsync
raw software_release_information_template ${template-resilient-software-release-information:destination}/${template-resilient-software-release-information:filename}
raw software_release_information_template ${template-resilient-software-release-information:target}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
......
......@@ -54,23 +54,10 @@ parts =
[template-base]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[download-base]
recipe = hexagonit.recipe.download
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[download-only-base]
< = download-base
ignore-existing = true
download-only = true
[template-download-base]
# Downloads from template directory into current directory
< = download-only-base
url = ${:_profile_base_location_}/template/${:filename}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template]
< = template-base
......@@ -81,66 +68,52 @@ output = ${buildout:directory}/template.cfg
output = ${buildout:directory}/template-runner.cfg.in
[template-runner-import-script]
< = template-download-base
filename = runner-import.sh.jinja2
< = download-base
[instance-runner-import]
< = download-base
recipe = slapos.recipe.build:download
[instance-runner-export]
< = download-base
recipe = slapos.recipe.build:download
[template-resilient]
< = download-base
recipe = slapos.recipe.build:download
[template_nginx_conf]
< = download-only-base
< = download-base
[template_httpd_conf]
< = download-only-base
< = download-base
[template_launcher]
< = download-base
recipe = slapos.recipe.build:download
[template-slapos-cfg]
< = template-download-base
filename = slapos.cfg.in
< = download-base
[template-slapformat-definition.cfg]
< = template-download-base
filename = slapformat-definition.cfg.in
< = download-base
[template-parameters]
< = download-only-base
< = download-base
[template-bash-profile]
< = template-download-base
filename = bash_profile.in
< = download-base
[template-supervisord]
< = template-download-base
filename = supervisord.conf.in
< = download-base
[template-listener-slapgrid]
< = template-download-base
filename = listener_slapgrid.py.in
< = download-base
[monitor-check-webrunner-internal-instance]
< = template-download-base
destination = ${:location}/${:filename}
filename = monitor-check-webrunner-internal-instances.py
< = download-base
[template-resilient-software-release-information]
< = template-download-base
filename = resilient_software_release_information.py.in
< = download-base
[template-slapuser-script]
< = template-download-base
filename = slapos-slapuser-script.in
< = download-base
[template-buildout-shared-part-list]
< = template-base
......@@ -175,5 +148,4 @@ gitdb = 0.6.4
gunicorn = 19.10.0
prettytable = 0.7.2
pycurl = 7.43.0
collective.recipe.environment = 0.2.0
smmap = 0.9.0
......@@ -15,48 +15,40 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 8e4f43e603a5dd57752758c987465d41
md5sum = 4df9f0d76a134a8abec9060a0c1be50b
[instance]
_update_hash_filename_ = instance.cfg.in
md5sum = a7d78b4002266c69ece05a476df82791
md5sum = f2f01a47d98a980177dc1755e618bbb7
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = 57b707cf0ed83be1959d26a88c131906
md5sum = b0a2c2b3d59fd6c8ba76c634b83a1ba2
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = 190a736471f0e0cffcb2838968e01d84
md5sum = b3f1dd83033d6a45def0bd26e70d5a9c
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = d78a9f885bdebf6720197209e0c21aa0
md5sum = b1e338973bc9cfe1bb4e16d46b3c6da9
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = e57396473b4b6a17d26a747f0030293c
md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
[theia-export]
_update_hash_filename_ = theia_export.py
md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
[theia-import]
_update_hash_filename_ = theia_import.py
md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
[yarn.lock]
_update_hash_filename_ = yarn.lock
md5sum = 067d2db611b21f77885f3adfd7f81453
md5sum = 1a668d6203d42b4d46d56e24c7606cb2
[python-language-server-requirements.txt]
_update_hash_filename_ = python-language-server-requirements.txt
md5sum = 9f478fd1b03b7738f3de549cb899bf54
[preloadTemplate.html]
_update_hash_filename_ = preloadTemplate.html
md5sum = 8157c22134200bd862a07c6521ebf799
[slapos.css.in]
_update_hash_filename_ = slapos.css.in
md5sum = d2930ec3ef973b7908f0fa896033fd64
......
......@@ -33,7 +33,7 @@ mode = 0700
exitcode-file = $${directory:srv}/export-exitcode-file
error-file = $${directory:srv}/export-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw python ${software-info:python-for-resiliency}
raw theia_export ${software-info:theia-export}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
......
......@@ -29,6 +29,11 @@ name = Import {{ parameter_dict['additional-frontend-name'] }}
{%- endif %}
# Change standalone socket path to avoid collisions
[slapos-standalone-config]
abstract-socket-path = $${directory:home}/standalone-import-ready
# Change port ranges to avoid race conditions on port allocation
[frontend-instance-port]
minimum = 3200
......@@ -79,7 +84,7 @@ mode = 0700
exitcode-file = $${directory:srv}/import-exitcode-file
error-file = $${directory:srv}/import-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw python ${software-info:python-for-resiliency}
raw theia_import ${software-info:theia-import}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
......
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
{% import 'parts' as parts -%}
{% import 'replicated' as replicated with context -%}
{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
{% set clones_amount = slapparameter_dict.get('resilient-clone-number', 1)|int + 1 -%}
[buildout]
eggs-directory = {{ eggs_directory }}
......@@ -11,59 +11,38 @@ extends =
{{ monitor_template }}
parts +=
# Generate the parts to request theia-export, pull-backup and theia-import
# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
# See below for the generation of the sections corresponding to the parts generated here
{{ parts.replicate("theia", number_of_instances + 1) }}
# Also publish some connection parameters
publish-connection-parameter
publish
{#- Generate the parts to request the main theia, the clones and the PBS. #}
{#- See ../../stack/resilient/template-parts.cfg.in #}
{{ parts.replicate("theia", clones_amount) }}
[ArgLeader]
[ArgBackup]
{#- Prepare monitoring information to transmit to and request from the main theia, the clones and the PBS #}
{%- set monitor_cors_domains = slapparameter_dict.pop('monitor-cors-domains', 'monitor.app.officejs.com') %}
{%- set monitor_username = slapparameter_dict.get('monitor-username', '${monitor-instance-parameter:username}') %}
{%- set monitor_password = slapparameter_dict.get('monitor-password', '${monitor-htpasswd:passwd}') %}
{%- set monitor_return = ['monitor-base-url'] %}
{%- set monitor_parameter = {'monitor-cors-domains': monitor_cors_domains, 'monitor-username' : monitor_username, 'monitor-password': monitor_password} %}
{%- set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return} %}
# Generate sections to request theia-export, pull-backup and theia-import
# See stack/resilient/template-replicated.cfg.in
# In particular:
#
# [request-theia]
# <= ArgLeader
# software-type = export
# ...
#
# [request-theia-pseudo-replicating-1]
# <= ArgBackup
# software-type = import
# ...
#
# [request-pbs-theia-1]
# software-type = pull-backup
# ...
#
{{ replicated.replicate("theia", number_of_instances + 1,
"export", "import",
"ArgLeader", "ArgBackup",
slapparameter_dict=slapparameter_dict) }}
# Extend the list of return parameters for the export request
# The monitor parameters are only there to assert they are
# actually published by the export instance
{# Generate the sections to request the main theia, the clones and the PBS. #}
{#- See ../../stack/resilient/template-replicated.cfg.in #}
{{ replicated.replicate("theia", clones_amount, "export", "import", slapparameter_dict=slapparameter_dict, monitor_parameter_dict=monitor_dict) }}
# Ask for the connection parameters of the main theia
[request-theia]
return += url username password backend-url monitor-base-url monitor-setup-url
return += url username password backend-url
# Extend the list of return parameters for the import request
# with the monitor parameters to assert they are actually published
[request-theia-pseudo-replicating-1]
return += monitor-base-url monitor-setup-url
# Publish some parameters from the export instance
[publish-connection-parameter]
# Publish connection parameters of the main theia and resiliency parameters
[publish]
recipe = slapos.cookbook:publish
url = ${request-theia:connection-url}
username = ${request-theia:connection-username}
password = ${request-theia:connection-password}
backend-url = ${request-theia:connection-backend-url}
# Publish resiliency parameters fetched by the resilient stack
[publish-connection-parameter]
monitor-base-url = ${monitor-publish:monitor-base-url}
monitor-setup-url = ${monitor-publish:monitor-setup-url}
<= publish-connection-information
......@@ -63,6 +63,25 @@ bash-completions = $${:home}/.local/share/bash-completion/completions/
fish-completions = $${:home}/.config/fish/completions/
# Monitor
# -------
[monitor-instance-parameter]
monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
{%- for k in ('monitor-cors-domains', 'monitor-username', 'monitor-password') %}
{%- set v = parameter_dict.get(k) %}
{%- if v %}
{{ k[8:] }} = {{ v }}
{%- endif %}
{%- endfor %}
{%- for k in ('monitor-url-list', ) %}
{%- set v = parameter_dict.get(k) %}
{%- if v %}
{{ k }} = {{ v }}
{%- endif %}
{%- endfor %}
# Promises
# --------
......@@ -132,7 +151,7 @@ config-port = $${slapos-standalone-instance:port}
<= monitor-promise-base
promise = check_socket_listening
name = standalone-ready-promise.py
config-abstract = $${directory:runner}/standalone_ready
config-abstract = $${slapos-standalone-config:abstract-socket-path}
[slapos-autorun-promise]
<= monitor-promise-base
......@@ -183,7 +202,6 @@ sla-instance_guid = {{ parameter_dict['additional-frontend-guid'] }}
[frontend-instance-password]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_}
[frontend-instance-port]
......@@ -378,7 +396,7 @@ recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${theia-service:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
hash-existing-files =
${yarn.lock:output}
${yarn.lock:target}
${theia-wrapper:rendered}
ip = {{ ipv4_random }}
hostname = $${:ip}
......@@ -426,6 +444,42 @@ command =
${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
# Embedded Instance
# -----------------
{%- set embedded_sr = parameter_dict['embedded-sr'] %}
{%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
{%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
{%- if embedded_sr %}
{%- if embedded_sr.startswith('~/') %}
{%- set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
{%- set embedded_sr = os_module.path.normpath(embedded_sr) %}
{%- endif %}
[request-embedded-instance-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/request_embedded.sh
mode = 0700
template =
inline:#!/bin/sh
slapos supply {{ embedded_sr }} slaprunner
slapos request "embedded_instance" {{ embedded_sr }}
{%- if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
{%- if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:rendered}
[embedded-instance-parameters]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.json
template =
inline:{{ embedded_instance_parameters | indent(2) }}
{%- endif %}
{%- endif %}
{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
# SlapOS Standalone
# -----------------
......@@ -442,6 +496,7 @@ port = $${slapos-standalone-port:port}
local-software-release-root = $${directory:home}
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
......@@ -453,6 +508,100 @@ template =
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template =
inline:#!${python-for-standalone:executable}
import glob
import json
import os
import signal
import socket
import subprocess
import sys
import time
import traceback
import slapos.slap.standalone
# Include this hash, so that if it changes the standalone service will be restarted
# {{ embedded_digest_hash }}
shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url="$${slap-connection:server-url}",
computer="$${slap-connection:computer-id}",
partition="$${slap-connection:partition-id}",
cert="$${slap-connection:cert-file}",
key="$${slap-connection:key-file}",
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
standalone = slapos.slap.standalone.StandaloneSlapOS(
"$${directory:runner}",
"$${slapos-standalone-config:ipv4}",
$${slapos-standalone-config:port},
computer_id="$${slapos-standalone-config:computer-id}",
shared_part_list=shared_part_list,
software_root="$${directory:runner}/software",
instance_root="$${directory:runner}/instance",
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
)
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start()
try:
partition_count = 20
print("Standalone SlapOS: Formatting %d partitions" % partition_count)
standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
{%- if embedded_sr %}
# Compatibility layer
try:
for cp in standalone.computer.getComputerPartitionList():
if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
print("Renaming 'Embedded Instance' into 'embedded_instance'")
cp.rename(new_name="embedded_instance")
break
except Exception:
print("Exception in compatibility layer, printing and moving on")
traceback.print_exc()
# Run request script
print("Running SlapOS script $${request-embedded-instance-script:rendered}")
slapos_env = {
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
}
subprocess.call(("$${request-embedded-instance-script:rendered}",), env=slapos_env)
{%- endif %}
s = socket.socket(socket.AF_UNIX)
s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
[slapos-standalone]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
......@@ -463,25 +612,7 @@ template =
#XXX find out where the extra nodejs in theia's PATH comes from
export PATH=${nodejs:location}/bin/:$PATH
. $${slapos-standalone-activate:rendered}
exec ${slapos-standalone:script-path} \
$${directory:runner} \
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:local-software-release-root} \
$${slapos-standalone-config:computer-id} \
{%- if parameter_dict.get('embedded-sr') %}
--sr='{{ parameter_dict['embedded-sr'] }}' \
{%- if parameter_dict.get('embedded-sr-type') %}
--srtype='{{ parameter_dict['embedded-sr-type'] }}' \
{%- endif %}
--srparams='$${embedded-instance-parameters:rendered}' \
{%- endif %}
$${slap-connection:server-url} \
$${slap-connection:computer-id} \
$${slap-connection:partition-id} \
--key='$${slap-connection:key-file}' \
--cert='$${slap-connection:cert-file}'
exec $${slapos-standalone-script:rendered}
[slapos-standalone-instance]
recipe = slapos.cookbook:wrapper
......@@ -489,6 +620,7 @@ wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${slapos-standalone:rendered}
hash-files =
$${slapos-standalone:rendered}
$${slapos-standalone-script:rendered}
hostname = $${slapos-standalone-config:ipv4}
port = $${slapos-standalone-config:port}
......@@ -604,14 +736,6 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
{% if parameter_dict.get('embedded-sr') -%}
[embedded-instance-parameters]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}.json
template =
inline:{{ parameter_dict['embedded-instance-parameters'] | indent(2) }}
{%- endif %}
[request-script-template]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh
......
......@@ -36,14 +36,17 @@ context =
jsonkey default_parameter_dict :default-parameters
key parameter_dict slap-configuration:configuration
key root_title slap-configuration:root-instance-title
key partition_root_path buildout:directory
key ipv6_random slap-configuration:ipv6-random
key ipv4_random slap-configuration:ipv4-random
import os_module os
import hashlib_module hashlib
default-parameters =
{
"autorun": "running",
"embedded-sr": null,
"embedded-sr-type": null,
"embedded-instance-parameters": "null",
"embedded-instance-parameters": null,
"frontend-name": "Theia Frontend",
"frontend-sr": "$${:frontend-sr}",
"frontend-sr-type": "RootSoftwareInstance",
......@@ -51,7 +54,8 @@ default-parameters =
"additional-frontend-name":"Theia Additional Frontend",
"additional-frontend-sr": "$${:frontend-sr}",
"additional-frontend-sr-type": "RootSoftwareInstance",
"additional-frontend-guid": null
"additional-frontend-guid": null,
"monitor-httpd-port": 8386
}
frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
......
......@@ -10,22 +10,19 @@ extends =
../../component/vim/buildout.cfg
../../component/curl/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/java-jdk/buildout.cfg
../../component/fonts/buildout.cfg
../../component/libsecret/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../stack/nodejs.cfg
../../component/theia/buildout.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../stack/resilient/buildout.cfg
../../component/defaults.cfg
./download-plugins.cfg
./buildout.hash.cfg
parts =
theia-wrapper
slapos-cookbook
python-with-eggs
python-for-resiliency
python-for-standalone
instance-theia
instance
instance-import
......@@ -37,149 +34,28 @@ parts =
# default for slapos-standalone
shared-part-list =
# Versions
# --------
[gcc]
# We keep the gcc part in sync with the one from erp5 software, so that when we install
# erp5 inside theia's slapos parts can be shared.
[gcc]
max_version = 0
[nodejs]
<= nodejs-12.18.3
<= nodejs-14.16.0
[yarn]
<= yarn-1.17.3
[slapos-standalone]
recipe = zc.recipe.egg
eggs =
slapos.core
scripts = ${:_buildout_section_name_}
script-path = ${buildout:bin-directory}/${:scripts}
# XXX generate a fake entry point for a non existant module, that will not
# be used because we exit in initialization step
entry-points =
${:scripts}=not_used:main
initialization =
import argparse
import glob
import json
import os
import signal
import socket
import sys
import time
import slapos.slap.standalone
parser = argparse.ArgumentParser()
parser.add_argument('base_directory')
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
parser.add_argument('local_software_release_root')
parser.add_argument('computer_id')
parser.add_argument('--sr')
parser.add_argument('--srtype')
parser.add_argument('--srparams')
forwarded_arguments = parser.add_argument_group('forwarded')
forwarded_arguments.add_argument('master_url')
forwarded_arguments.add_argument('computer')
forwarded_arguments.add_argument('partition')
# cert and key are optional
forwarded_arguments.add_argument('--cert')
forwarded_arguments.add_argument('--key')
args = parser.parse_args()
shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url=args.master_url,
computer=args.computer,
partition=args.partition,
cert=args.cert,
key=args.key,
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
standalone = slapos.slap.standalone.StandaloneSlapOS(
args.base_directory,
args.ipv4,
args.server_port,
computer_id=args.computer_id,
shared_part_list=shared_part_list,
software_root="%s/software" % args.base_directory,
instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root=args.local_software_release_root,
)
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start()
try:
partition_count = 20
if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
print("Standalone SlapOS: Formatting {partition_count} partitions".format(
partition_count=partition_count))
standalone.format(
partition_count,
args.ipv4,
args.ipv6,
)
print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
if args.sr:
try:
with open(args.srparams) as f:
params = json.load(f)
except Exception:
params = None
if not isinstance(params, dict):
params = None
print("Supplying and Requesting Embedded Software {sr} with type {srtype}".format(
sr=args.sr, srtype=args.srtype))
print("With parameters {param_dict} parsed from '{srparams}'".format(
param_dict=params, srparams=args.srparams))
standalone.supply(args.sr)
standalone.request(
args.sr,
"Embedded Instance",
args.srtype,
partition_parameter_kw=params,
)
s = socket.socket(socket.AF_UNIX)
s.bind('\0' + os.path.join(args.base_directory, 'standalone_ready'))
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
[gowork]
install +=
golang.org/x/tools/gopls@v0.6.6
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
${slapos-command:recipe}
[supervisor]
recipe = zc.recipe.egg
eggs =
supervisor
setuptools
# Downloads and templates
# -----------------------
[template-base]
recipe = slapos.recipe.template
......@@ -192,199 +68,15 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
output = ${:destination}
[python-language-server]
version = 0.19.0
recipe = plone.recipe.command
command =
PATH=${git:location}/bin/:$PATH bash -c "${python3:executable} -m venv --clear ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${python-language-server-requirements.txt:output}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
[python-language-server-requirements.txt]
<= download-base
[theia]
recipe = plone.recipe.command
command = ${bash:location}/bin/bash -c "
export \
TMPDIR=${:location}/tmp \
PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:$PATH \
PKG_CONFIG_PATH=${libsecret:pkg-config-path} \
LDFLAGS='-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib' && \
mkdir -p ${:location} && \
mkdir -p \$TMPDIR && \
cd ${:location} && \
cp ${package.json:rendered} . &&
cp ${yarn.lock:output} . &&
${yarn:location}/bin/yarn && \
${yarn:location}/bin/yarn theia build"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
uses = ${yarn.lock:recipe}
[theia-plugins]
recipe = slapos.recipe.build
urls = ${theia-download-plugins:urls}
install =
import os
for line in options['urls'].splitlines():
extension_name, url, md5sum = line.split()
extract_dir = self.extract(self.download(url, md5sum))
destination_dir = os.path.join(options['location'], extension_name)
self.copyTree(guessworkdir(extract_dir), destination_dir)
os.chmod(destination_dir, 0o750)
[yarn.lock]
<= download-base
[preloadTemplate.html]
<= download-base
[slapos.css.in]
<= download-base
[logo.png]
<= download-base
[package.json]
recipe = slapos.recipe.template:jinja2
template =
inline:{
"private": true,
"theia": {
"backend": {
"config": {
"warnOnPotentiallyInsecureHostPattern": false
}
},
"frontend": {
"config": {
"applicationName": "Theia SlapOS",
"preferences": {
"application.confirmExit": "always",
"files.associations": {
"*.cfg": "zc-buildout"
},
"files.enableTrash": false,
"files.exclude": {
"**.pyc": true,
"**.egg-info": true,
"__pycache__": true,
".git": true,
".env": true,
"**/node_modules/**": true
},
"files.watcherExclude": {
"**/.eggs/**": true,
"**/.env/**": true,
"**/.git/**": true,
"**/node_modules/**": true
},
"editor.multiCursorModifier": "ctrlCmd",
"editor.tabSize": 2,
"plantuml.server": "https://plantuml.host.vifib.net/svg/",
"plantuml.render": "PlantUMLServer",
"gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }],
"java.home": "${java-jdk:location}"
},
"warnOnPotentiallyInsecureHostPattern": false
}
},
"generator": {
"config": {
"preloadTemplate": "${preloadTemplate.html:output}"
}
}
},
"dependencies": {
"@theia/bulk-edit": "latest",
"@theia/callhierarchy": "latest",
"@theia/console": "latest",
"@theia/core": "latest",
"@theia/debug": "latest",
"@theia/editor": "latest",
"@theia/editor-preview": "latest",
"@theia/file-search": "latest",
"@theia/filesystem": "latest",
"@theia/getting-started": "latest",
"@theia/keymaps": "latest",
"@theia/markers": "latest",
"@theia/messages": "latest",
"@theia/metrics": "latest",
"@theia/mini-browser": "latest",
"@theia/monaco": "latest",
"@theia/navigator": "latest",
"@theia/outline-view": "latest",
"@theia/output": "latest",
"@theia/plugin-dev": "latest",
"@theia/plugin-ext": "latest",
"@theia/plugin-ext-vscode": "latest",
"@theia/preferences": "latest",
"@theia/preview": "latest",
"@theia/process": "latest",
"@theia/property-view": "latest",
"@theia/scm": "latest",
"@theia/scm-extra": "latest",
"@theia/search-in-workspace": "latest",
"@theia/task": "latest",
"@theia/terminal": "latest",
"@theia/timeline": "latest",
"@theia/typehierarchy": "latest",
"@theia/userstorage": "latest",
"@theia/variable-resolver": "latest",
"@theia/vsx-registry": "latest",
"@theia/workspace": "latest",
"@perrinjerome/theia-open": "latest",
"@perrinjerome/theia-open-cli": "latest"
},
"devDependencies": {
"@theia/cli": "latest"
}
}
rendered = ${buildout:directory}/${:_buildout_section_name_}
[gowork]
install +=
golang.org/x/tools/gopls@v0.6.6
[cli-utilities]
PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${buildout:bin-directory}
[theia-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:
#!/bin/sh
cd ${theia:location}
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libsecret:location}/lib:${gettext:location}/lib:${glib:location}/lib"
exec ${yarn:location}/bin/yarn theia start "$@"
[theia-open]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:
#!/bin/sh
exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
[instance-theia]
<= template-base
output = ${buildout:directory}/instance-theia.cfg.jinja
......@@ -416,9 +108,57 @@ destination = ${buildout:directory}/theia_export.py
<= download-base
destination = ${buildout:directory}/theia_import.py
# Utilities
# ---------
[supervisor]
recipe = zc.recipe.egg
eggs =
supervisor
setuptools
[python-language-server]
version = 0.19.0
recipe = plone.recipe.command
command =
PATH=${git:location}/bin/:$PATH bash -c "${python3:executable} -m venv --clear ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${python-language-server-requirements.txt:output}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
[cli-utilities]
PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python:location}/bin:${buildout:bin-directory}
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
executable = ${buildout:bin-directory}/${:interpreter}
[python-for-resiliency]
<= python-with-eggs
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
[python-for-standalone]
<= python-with-eggs
eggs =
slapos.core
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
${slapos-command:recipe}
[software-info]
slapos = ${buildout:bin-directory}/slapos
python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
python-for-resiliency = ${python-for-resiliency:executable}
python = ${python:location}/bin/python
rsync = ${rsync:location}/bin/rsync
sqlite3 = ${sqlite3:location}/bin/sqlite3
......
......@@ -60,7 +60,7 @@ def setUpModule():
class ERP5Mixin(object):
_test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{\s*'_'\s*:\s*'(.*)'\s*}")
_connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type)
......@@ -69,7 +69,7 @@ class ERP5Mixin(object):
stderr=subprocess.STDOUT,
)
print(out)
return json.loads(self._connexion_parameters_regex.search(out).group(1))
return json.loads(self._connexion_parameters_regex.search(out).group(0).replace("'", '"'))
def _getERP5Url(self, connexion_parameters, path=''):
return urljoin(connexion_parameters['family-default-v6'], path)
......@@ -121,6 +121,10 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
backup_max_tries = 480
backup_wait_interval = 60
def test_twice(self):
# do nothing
pass
def _prepareExport(self):
super(TestTheiaResilienceERP5, self)._prepareExport()
......
......@@ -40,7 +40,7 @@ import six
from six.moves.urllib.parse import urlparse, urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass, SlapOSNodeCommandError
from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
......@@ -229,29 +229,82 @@ class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
self.assertFalse(embedded_slapos_process.is_running())
class TestTheiaWithSR(TheiaTestCase):
sr_url = 'bogus/software.cfg'
class ReRequestMixin(object):
def rerequest(self, parameter_dict=None, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
name = self.default_partition_reference
self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=name,
partition_parameter_kw=parameter_dict,
state=state)
def reinstantiate(self):
# Process at least twice to propagate parameter changes
try:
self.slap.waitForInstance()
except SlapOSNodeCommandError:
pass
self.slap.waitForInstance(self.instance_max_retry)
class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
sr_url = '~/bogus/software.cfg'
sr_type = 'bogus_type'
instance_parameters = '{\n"bogus_param": "bogus_value"\n}'
instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
@classmethod
def getInstanceParameterDict(cls):
return {
'embedded-sr': cls.sr_url,
'embedded-sr-type': cls.sr_type,
'embedded-instance-parameters': cls.instance_parameters
}
def proxy_show(self, slapos):
return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
def test(self):
slapos = self._getSlapos()
info = subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
instance_name = "Embedded Instance"
home = self.computer_partition_root_path
# Check that no request script was generated
request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
self.assertFalse(os.path.exists(request_script))
# Manually request old-name 'Embedded Instance'
old_instance_name = "Embedded Instance"
subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
self.assertIn(old_instance_name, self.proxy_show(slapos))
# Update Theia instance parameters
embedded_request_parameters = {
'embedded-sr': self.sr_url,
'embedded-sr-type': self.sr_type,
'embedded-instance-parameters': self.instance_parameters
}
self.rerequest(embedded_request_parameters)
self.reinstantiate()
# Check that embedded instance was requested
instance_name = "embedded_instance"
info = self.proxy_show(slapos)
try:
self.assertIn(instance_name, info)
except AssertionError:
for filename in os.listdir(home):
if 'standalone' in filename and '.log' in filename:
filepath = os.path.join(home, filename)
with open(filepath) as f:
print("Contents of filepath: " + filepath)
print(f.read())
raise
# Check that old-name instance was renamed
self.assertNotIn(old_instance_name, info)
# Check embedded instance parameters
bogus_sr = os.path.join(home, self.sr_url[2:])
self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (self.sr_url,), info), info)
self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (self.sr_url, self.sr_type, instance_name), info), info)
self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)
service_info = subprocess.check_output((slapos, 'service', 'info', instance_name), universal_newlines=True)
self.assertIn("{'bogus_param': 'bogus_value'}", service_info)
self.assertIn("{'bogus_param': 'bogus_value', 'bogus_param2': 'bogus_value2'}", service_info)
class TestTheiaFrontend(TheiaTestCase):
......
......@@ -225,6 +225,8 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
script_relpath = os.path.join(
'srv', 'runner', 'instance', 'slappart0',
'srv', '.backup_identity_script')
signature_relpath = os.path.join(
'srv', 'backup', 'theia', 'backup.signature')
def assertPromiseFailure(self, *msg):
# Force promises to recompute regardless of periodicity
......@@ -291,6 +293,10 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
self.customSignatureScript(content=None)
self.customRestoreScript(content=None)
self.cleanupExitfiles()
try:
os.remove(self._getPartitionPath('import', self.signature_relpath))
except OSError:
pass
def test_export_promise(self):
self.writeFile(self.getExportExitfile(), '1')
......@@ -303,17 +309,14 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
def test_custom_hash_script(self):
errmsg = 'Bye bye'
self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
backup_script = self._getPartitionPath(
'export', 'srv', 'backup', 'theia', self.script_relpath)
self.assertExportFailure('Compute backup signature\n ... ERROR !',
'Custom signature script %s failed' % os.path.abspath(backup_script),
custom_script = self._getPartitionPath('export', self.script_relpath)
self.assertExportFailure('Compute partitions backup signatures\n ... ERROR !',
'Custom signature script %s failed' % os.path.abspath(custom_script),
'and stderr:\n%s' % errmsg)
def test_signature_mismatch(self):
signature_file = self._getPartitionPath('import', 'srv', 'backup', 'theia', 'backup.signature')
moved_file = self._getPartitionPath('import', 'srv', 'backup', 'backup.signature.moved')
self.writeFile(moved_file, 'Bogus Hash\n', mode='a')
os.rename(moved_file, signature_file)
signature_file = self._getPartitionPath('import', self.signature_relpath)
self.writeFile(signature_file, 'Bogus Hash\n', mode='a')
self.assertImportFailure('ERROR the backup signatures do not match')
def test_restore_script_error(self):
......@@ -363,12 +366,15 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self.writeFile(os.path.join(dummy_root, 'exclude', 'excluded'),
'This file should be excluded from resilient backup')
# Check that ~/srv/exporter.exclude and ~/srv/runner-import-restore
# Check that ~/srv/exporter.exclude and ~/srv/runner-import-restore exist
# As well as ~/srv/.backup_identity_script
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', 'exporter.exclude')))
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', 'runner-import-restore')))
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
def _doSync(self):
self._doExport()
self._doTransfer()
......@@ -384,14 +390,20 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self.assertIn(adapted_test_url, proxy_content)
self.assertNotIn(self._test_software_url, proxy_content)
# Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
# Check that ~/srv/project was exported
self.assertTrue(os.path.exists(adapted_test_url))
# Check that the dummy instance is not yet started
self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
# Check that ~/srv/.backup_identity_script was called
signature = self._getPartitionPath('import', 'srv', 'backup', 'backup.signature.proof')
# Check that ~/srv/.backup_identity_script was detected and called
signature = self._getPartitionPath(
'import', 'srv', 'backup', 'theia', 'slappart0.backup.signature.custom')
self.assertTrue(os.path.exists(signature))
with open(signature) as f:
self.assertIn('Custom script', f.read())
......@@ -477,6 +489,14 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
_test_software_url = dummy_software_url
def test_twice(self):
# Run two synchronisations on the same instances
# to make sure everything still works the second time
# Check ~/etc in import theia again
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self._doSync()
self._checkSync()
def _prepareExport(self):
# Deploy test instance
self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
......@@ -485,6 +505,9 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
self.export_id = self._getPartitionId('export')
self.import_id = self._getPartitionId('import')
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
def _doSync(self):
start = time.time()
......@@ -499,6 +522,11 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
# Wait for takoever to be ready
self._waitTakeoverReady(takeover_url, start, self.backup_max_tries, self.backup_wait_interval)
def _checkSync(self):
# Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
def _doTakeover(self):
# Takeover
takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
......
......@@ -4,6 +4,7 @@ import glob
import hashlib
import os
import re
import shutil
import subprocess as sp
import sqlite3
......@@ -21,13 +22,19 @@ EXCLUDE_FLAGS = ['--exclude={}'.format(x) for x in sorted(EXCLUDE_PATTERNS)]
def makedirs(path):
try:
os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
def copyfile(src, dst):
dst = os.path.abspath(dst)
makedirs(os.path.dirname(dst))
shutil.copy2(src, dst)
def copytree(rsyncbin, src, dst, exclude=(), extrargs=(), verbosity='-v'):
# Ensure there is a trailing slash in the source directory
# to avoid creating an additional directory level at the destination
src = os.path.join(src, '')
......@@ -60,21 +67,20 @@ def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
def copydb(sqlite3bin, src_db, dst_db):
makedirs(dst_db)
makedirs(os.path.dirname(dst_db))
sp.check_output((sqlite3bin, src_db, '.backup ' + dst_db))
def remove(path):
try:
os.remove(path)
except OSError:
if os.path.exists(path):
except OSError as e:
if e.errno != errno.ENOENT:
raise
def parse_installed(partition):
paths = []
custom_script = os.path.join(partition, 'srv', '.backup_identity_script')
for cfg in glob.glob(os.path.join(partition, '.installed*.cfg')):
try:
with open(cfg) as f:
......@@ -86,7 +92,7 @@ def parse_installed(partition):
for section in six.itervalues(installed_cfg):
for p in section.get('__buildout_installed__', '').splitlines():
p = p.strip()
if p and p != custom_script:
if p:
paths.append(p)
return paths
......@@ -101,31 +107,44 @@ def sha256sum(file_path, chunk_size=1024 * 1024):
return sha256.hexdigest()
def hashwalk(backup_dir, mirror_partitions):
scripts = {}
for p in mirror_partitions:
script_path = os.path.join(p, 'srv', '.backup_identity_script')
if os.path.exists(script_path):
scripts[os.path.abspath(p)] = script_path
for dirpath, dirnames, filenames in os.walk(backup_dir):
filenames.sort()
def fast_hashwalk(root_dir):
for dirpath, dirnames, filenames in os.walk(root_dir):
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=backup_dir)
displaypath = os.path.relpath(filepath, start=root_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
remaining_dirnames = []
for subdir in dirnames:
subdirpath = os.path.abspath(os.path.join(dirpath, subdir))
custom_hashscript = scripts.get(subdirpath)
if custom_hashscript:
print('Using custom signature script %s' % custom_hashscript)
for s in hashcustom(subdirpath, backup_dir, custom_hashscript):
yield s
else:
remaining_dirnames.append(subdir)
remaining_dirnames.sort()
dirnames[:] = remaining_dirnames
def exclude_hashwalk(root_dir, instance_dir):
root_dir = os.path.abspath(root_dir)
instance_dir = os.path.abspath(instance_dir)
for dirpath, dirnames, filenames in os.walk(root_dir):
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=root_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
if dirpath == instance_dir:
remaining_dirs = []
for d in dirnames:
if not d.startswith('slappart'):
remaining_dirs.append(d)
dirnames[:] = remaining_dirs
def hashwalk(root_dir, instance_dir=None):
if instance_dir and not os.path.relpath(
instance_dir, start=root_dir).startswith(os.pardir):
return exclude_hashwalk(root_dir, instance_dir)
return fast_hashwalk(root_dir)
def hashscript(partition):
script = os.path.join(partition, 'srv', '.backup_identity_script')
if os.path.exists(script):
return script
return None
@contextlib.contextmanager
......@@ -138,10 +157,11 @@ def cwd(path):
os.chdir(old_path)
def hashcustom(mirrordir, backup_dir, custom_hashscript):
workingdir = os.path.join(mirrordir, os.pardir, os.pardir, os.pardir)
def hashcustom(partition, script):
workingdir = os.path.join(partition, os.pardir, os.pardir, os.pardir)
with cwd(os.path.abspath(workingdir)):
for dirpath, _, filenames in os.walk(mirrordir):
for dirpath, dirnames, filenames in os.walk(partition):
dirnames.sort()
filepaths = []
for f in filenames:
path = os.path.join(dirpath, f)
......@@ -150,16 +170,16 @@ def hashcustom(mirrordir, backup_dir, custom_hashscript):
if not filepaths:
continue
hashprocess = sp.Popen(
custom_hashscript, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
script, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = hashprocess.communicate(str2bytes('\0'.join(filepaths)))
if hashprocess.returncode != 0:
template = "Custom signature script %s failed on inputs:\n%s"
msg = template % (custom_hashscript, '\n'.join(filepaths))
msg = template % (script, '\n'.join(filepaths))
msg += "\nwith stdout:\n%s" % bytes2str(out)
msg += "\nand stderr:\n%s" % bytes2str(err)
raise Exception(msg)
signatures = bytes2str(out).strip('\n').split('\n')
signatures.sort()
displaypath = os.path.relpath(dirpath, start=backup_dir)
displaypath = os.path.relpath(dirpath, start=partition)
for s in signatures:
yield '%s %s/ (custom)' % (s, displaypath)
yield '%s %s' % (s, displaypath)
......@@ -9,8 +9,8 @@ import traceback
import six
from six.moves import configparser
sys.path.append(os.path.dirname(__file__))
from theia_common import copytree, copydb, hashwalk, parse_installed, remove
sys.path.insert(0, os.path.dirname(__file__))
from theia_common import *
os.environ['LC_ALL'] = 'C'
......@@ -55,45 +55,74 @@ class TheiaExport(object):
self.copytree_partitions_args = {}
self.logs = []
def mirrorpath(self, src):
def mirror_path(self, src):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(src, start=self.root_dir)))
def backuptree(self, src, exclude=[], extrargs=[], verbosity='-v'):
dst = self.mirrorpath(src)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def backup_tree(self, src):
return copytree(self.rsync_bin, src, self.mirror_path(src))
def backupdb(self):
copydb(self.sqlite3_bin, self.proxy_db, self.mirrorpath(self.proxy_db))
def backup_file(self, src):
return copyfile(src, self.mirror_path(src))
def backuppartition(self, partition):
def backup_db(self):
copydb(self.sqlite3_bin, self.proxy_db, self.mirror_path(self.proxy_db))
def backup_partition(self, partition):
installed = parse_installed(partition)
rules = os.path.join(partition, 'srv', 'exporter.exclude')
extrargs = ('--filter=.-/ ' + rules,) if os.path.exists(rules) else ()
self.backuptree(partition, exclude=installed, extrargs=extrargs)
self.copytree_partitions_args[partition] = (installed, extrargs)
dst = self.mirror_path(partition)
copytree(self.rsync_bin, partition, dst, installed, extrargs)
self.copytree_partitions_args[partition] = (dst, installed, extrargs)
def sign(self, signaturefile):
def sign(self, signaturefile, signatures):
remove(signaturefile)
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
tmpfile = os.path.join(pardir, 'backup.signature.tmp')
mirror_partitions = [self.mirrorpath(p) for p in self.partition_dirs]
tmpfile = os.path.join(pardir, os.path.basename(signaturefile) + '.tmp')
with open(tmpfile, 'w') as f:
for s in hashwalk(self.backup_dir, mirror_partitions):
for s in signatures:
f.write(s + '\n')
os.rename(tmpfile, signaturefile)
def checkpartition(self, partition, pattern='/srv/backup/'):
installed, extrargs = self.copytree_partitions_args[partition]
output = self.backuptree(
def sign_root(self):
signaturefile = os.path.join(self.backup_dir, 'backup.signature')
signatures = hashwalk(self.backup_dir, self.mirror_path(self.instance_dir))
self.sign(signaturefile, signatures)
def sign_partition(self, partition):
dst = self.mirror_path(partition)
filename = os.path.basename(partition) + '.backup.signature'
signaturefile = os.path.join(self.backup_dir, filename)
script = hashscript(partition)
if script:
signaturefile += '.custom'
self.sign(signaturefile, hashcustom(dst, script))
else:
self.sign(signaturefile, hashwalk(dst))
def remove_signatures(self):
pattern = os.path.join(self.backup_dir, '*backup.signature*')
signature_files = glob.glob(pattern)
for f in signature_files:
try:
os.remove(f)
except OSError:
pass
def check_partition(self, partition, pattern='/srv/backup/'):
dst, installed, extrargs = self.copytree_partitions_args[partition]
output = copytree(
self.rsync_bin,
partition,
dst,
exclude=installed,
extrargs=extrargs + ('--dry-run', '--update'),
verbosity='--out-format=%n',
)
return [path for path in output.splitlines() if pattern in path]
def loginfo(self, msg):
def log(self, msg):
print(msg)
self.logs.append(msg)
......@@ -118,40 +147,46 @@ class TheiaExport(object):
def export(self):
export_start_date = int(time.time())
etc_dir = os.path.join(self.root_dir, 'etc')
with open(os.path.join(etc_dir, '.resilient_timestamp'), 'w') as f:
timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
with open(timestamp, 'w') as f:
f.write(str(export_start_date))
self.loginfo('Backup directory ' + etc_dir)
self.backuptree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
self.remove_signatures()
self.log('Backup resilient timestamp ' + timestamp)
self.backup_file(timestamp)
for d in self.dirs:
self.loginfo('Backup directory ' + d)
self.backuptree(d)
self.log('Backup directory ' + d)
self.backup_tree(d)
self.loginfo('Backup slapproxy database')
self.backupdb()
self.log('Backup slapproxy database')
self.backup_db()
self.loginfo('Backup partitions')
self.log('Backup partitions')
for p in self.partition_dirs:
self.backuppartition(p)
self.backup_partition(p)
self.loginfo('Compute backup signature')
self.sign(os.path.join(self.backup_dir, 'backup.signature'))
self.log('Compute root backup signature')
self.sign_root()
self.log('Compute partitions backup signatures')
for p in self.partition_dirs:
self.sign_partition(p)
time.sleep(10)
self.loginfo('Check partitions')
self.log('Check partitions')
modified = list(itertools.chain.from_iterable(
self.checkpartition(p) for p in self.partition_dirs))
self.check_partition(p) for p in self.partition_dirs))
if modified:
msg = 'Some files have been modified since the backup started'
self.loginfo(msg + ':')
self.loginfo('\n'.join(modified))
self.loginfo("Let's wait %d minutes and try again" % BACKUP_WAIT)
self.log(msg + ':')
self.log('\n'.join(modified))
self.log("Let's wait %d minutes and try again" % BACKUP_WAIT)
time.sleep(BACKUP_WAIT * 60)
raise Exception(msg)
self.loginfo('Done')
self.log('Done')
if __name__ == '__main__':
......
......@@ -10,7 +10,7 @@ import six
from six.moves import configparser
sys.path.append(os.path.dirname(__file__))
from theia_common import copytree, copydb, hashwalk, parse_installed, remove
from theia_common import *
os.environ['LC_ALL'] = 'C'
......@@ -57,28 +57,32 @@ class TheiaImport(object):
configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri')
self.instance_dir = configp.get('slapos', 'instance_root')
mirror_dir = self.mirrorpath(self.instance_dir)
mirror_dir = self.mirror_path(self.instance_dir)
partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
self.mirror_partition_dirs = [p for p in partitions if os.path.isdir(p)]
self.logs = []
def mirrorpath(self, dst):
def mirror_path(self, dst):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(dst, start=self.root_dir)))
def dstpath(self, src):
def dst_path(self, src):
return os.path.abspath(os.path.join(
self.root_dir, os.path.relpath(src, start=self.backup_dir)))
def restoretree(self, dst, exclude=[], extrargs=[], verbosity='-v'):
src = self.mirrorpath(dst)
def restore_tree(self, dst, exclude=(), extrargs=(), verbosity='-v'):
src = self.mirror_path(dst)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def restoredb(self):
copydb(self.sqlite3_bin, self.mirrorpath(self.proxy_db), self.proxy_db)
def restore_file(self, dst):
src = self.mirror_path(dst)
return copyfile(src, dst)
def restorepartition(self, mirror_partition):
p = self.dstpath(mirror_partition)
def restore_db(self):
copydb(self.sqlite3_bin, self.mirror_path(self.proxy_db), self.proxy_db)
def restore_partition(self, mirror_partition):
p = self.dst_path(mirror_partition)
installed = parse_installed(p) if os.path.exists(p) else []
copytree(self.rsync_bin, mirror_partition, p, exclude=installed)
......@@ -86,38 +90,67 @@ class TheiaImport(object):
supervisor_command = (self.supervisorctl_bin, '-c', self.supervisord_conf)
command = supervisor_command + args
print(' '.join(command))
sp.check_call(command)
print(sp.check_output(command, stderr=sp.STDOUT, universal_newlines=True))
def slapos(self, *args):
command = (self.slapos_bin,) + args + ('--cfg', self.slapos_cfg)
print(' '.join(command))
sp.check_call(command)
print(sp.check_output(command, stderr=sp.STDOUT, universal_newlines=True))
def verify(self, signaturefile):
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
moved = os.path.join(pardir, 'backup.signature.moved')
proof = os.path.join(pardir, 'backup.signature.proof')
if os.path.exists(signaturefile):
os.rename(signaturefile, moved)
if not os.path.exists(moved):
msg = 'ERROR the backup signature file is missing'
print(msg)
def sign(self, signaturefile, root_dir):
with open(signaturefile, 'r') as f:
for line in f:
try:
_, relpath = line.strip().split(None, 1)
except ValueError:
yield 'Could not parse: %s' % line
continue
filepath = os.path.join(root_dir, relpath)
try:
signature = sha256sum(filepath)
except IOError:
yield 'Could not read: %s' % filepath
continue
yield '%s %s' % (signature, relpath)
def sign_custom(self, root_dir):
partition = self.dst_path(root_dir)
script = hashscript(partition)
if not script:
msg = 'ERROR: missing custom signature script for partition ' + partition
raise Exception(msg)
return hashcustom(root_dir, script)
def find_signature_file(self, partition):
filename = os.path.basename(partition) + '.backup.signature'
signaturefile = os.path.join(self.backup_dir, filename)
if os.path.exists(signaturefile):
return signaturefile, False
signaturefile += '.custom'
if os.path.exists(signaturefile):
return signaturefile, True
raise Exception('ERROR: missing signature file for partition ' + partition)
def verify(self, signaturefile, root_dir, custom=False):
proof = signaturefile + '.proof'
if custom:
signatures = self.sign_custom(root_dir)
else:
signatures = self.sign(signaturefile, root_dir)
with open(proof, 'w') as f:
for s in hashwalk(self.backup_dir, self.mirror_partition_dirs):
for s in signatures:
f.write(s + '\n')
diffcommand = ('diff', moved, proof)
print(' '.join(diffcommand))
diffcommand = ('diff', signaturefile, proof)
try:
sp.check_output(
diffcommand, stderr=sp.STDOUT, universal_newlines=True)
except sp.CalledProcessError as e:
template = 'ERROR the backup signatures do not match\n\n%s'
msg = template % e.output
template = 'ERROR the backup signatures do not match\n\n%s\n%s'
msg = template % (' '.join(diffcommand), e.output)
print(msg)
raise Exception(msg)
def loginfo(self, msg):
def log(self, msg):
print(msg)
self.logs.append(msg)
......@@ -126,9 +159,11 @@ class TheiaImport(object):
exitcode = 0
try:
self.restore()
except Exception:
except Exception as e:
exitcode = 1
exc = traceback.format_exc()
if isinstance(e, sp.CalledProcessError) and e.output:
exc = "%s\n\n%s" % (exc, e.output)
with open(self.error_file, 'w') as f:
f.write('\n ... OK\n\n'.join(self.logs))
f.write('\n ... ERROR !\n\n')
......@@ -140,44 +175,54 @@ class TheiaImport(object):
sys.exit(exitcode)
def restore(self):
self.loginfo('Verify backup signature')
self.verify(os.path.join(self.backup_dir, 'backup.signature'))
self.log('Verify main backup signature')
signaturefile = os.path.join(self.backup_dir, 'backup.signature')
self.verify(signaturefile, self.backup_dir)
self.loginfo('Stop slapproxy')
custom_partition_signatures = []
for m in self.mirror_partition_dirs:
signaturefile, custom = self.find_signature_file(m)
if custom:
custom_partition_signatures.append((signaturefile, m))
else:
self.log('Verify backup signature for ' + m)
self.verify(signaturefile, m)
self.log('Stop slapproxy')
self.supervisorctl('stop', 'slapos-proxy')
self.loginfo('Restore partitions')
self.log('Restore partitions')
for m in self.mirror_partition_dirs:
self.restorepartition(m)
self.restore_partition(m)
for d in self.dirs:
self.loginfo('Restore directory ' + d)
self.restoretree(d)
self.log('Restore directory ' + d)
self.restore_tree(d)
self.loginfo('Restore slapproxy database')
self.restoredb()
self.log('Restore slapproxy database')
self.restore_db()
etc_dir = os.path.join(self.root_dir, 'etc')
self.loginfo('Restore directory ' + etc_dir)
self.restoretree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
timestamp = os.path.join(self.root_dir, 'etc', '.resilient_timestamp')
self.log('Restore resilient timestamp ' + timestamp)
self.restore_file(timestamp)
custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
if os.path.exists(custom_script):
self.loginfo('Run custom restore script %s' % custom_script)
sp.check_call(custom_script)
self.log('Run custom restore script %s' % custom_script)
print(sp.check_output(custom_script))
self.loginfo('Start slapproxy again')
self.log('Start slapproxy again')
self.supervisorctl('start', 'slapos-proxy')
self.loginfo('Reformat partitions')
self.log('Reformat partitions')
self.slapos('node', 'format', '--now')
self.loginfo('Remove old supervisord configuration files')
self.log('Remove old supervisord configuration files')
conf_dir = os.path.join(self.instance_dir, 'etc', 'supervisor.conf.d')
for f in glob.glob(os.path.join(conf_dir, '*')):
os.remove(f)
self.loginfo('Build Software Releases')
self.log('Build Software Releases')
for i in range(3):
try:
self.slapos('node', 'software', '--all', '--logfile', self.sr_log)
......@@ -187,18 +232,18 @@ class TheiaImport(object):
else:
break
self.loginfo('Remove old custom instance scripts')
self.log('Remove old custom instance scripts')
partitions_glob = os.path.join(self.instance_dir, 'slappart*')
scripts = os.path.join(partitions_glob, 'srv', 'runner-import-restore')
for f in glob.glob(scripts):
remove(f)
self.loginfo('Remove partition timestamps')
self.log('Remove partition timestamps')
timestamps = os.path.join(partitions_glob, '.timestamp')
for f in glob.glob(timestamps):
remove(f)
self.loginfo('Build Instances')
self.log('Build Instances')
cp_log = self.cp_log
for i in range(3):
try:
......@@ -209,11 +254,15 @@ class TheiaImport(object):
else:
break
self.log('Verify custom backup signatures')
for signaturefile, m in custom_partition_signatures:
self.verify(signaturefile, m, True)
for custom_script in glob.glob(scripts):
self.loginfo('Running custom instance script %s' % custom_script)
sp.check_call(custom_script)
self.log('Running custom instance script %s' % custom_script)
print(sp.check_output(custom_script))
self.loginfo('Done')
self.log('Done')
if __name__ == '__main__':
......
......@@ -18,7 +18,6 @@ parts =
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 644
[instance-cfg]
recipe = slapos.recipe.template
......
......@@ -10,9 +10,7 @@ md5sum = 409889c98b13cbdbb9fd121df859ae3e
[application-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
download-only = True
filename = template.in
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[application-configuration]
......
......@@ -32,7 +32,6 @@ scripts =
[caucase-jinja2-library]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
depends = ${caucase-jinja2-library-eggs:eggs}
[versions]
......
......@@ -98,15 +98,12 @@ url = ${:_profile_base_location_}/${:filename}
[mariadb-start-clone-from-backup]
<= download-base
mode = 755
[mariadb-resiliency-after-import-script]
<= download-base
mode = 755
[mariadb-slow-query-report-script]
<= download-base
mode = 755
[template-mariadb]
<= download-base
......
......@@ -74,7 +74,7 @@ md5sum = bbef65b4edeb342f08309604ca3717d5
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = c10634353841bb09a847168b4add8d2f
md5sum = cb41e15a1585973896a7645dc2ae3a5c
[template-zeo]
filename = instance-zeo.cfg.in
......
......@@ -297,7 +297,7 @@ config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 1)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
......
......@@ -32,17 +32,11 @@ extends =
[template-download-base]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[application]
recipe = hexagonit.recipe.download
ignore-existing = true
# If the provided tarball does not contain top directory, this option should be changed to false
strip-top-level-dir = true
recipe = slapos.recipe.build:download-unpacked
#----------------
#--
......@@ -93,10 +87,10 @@ context =
key groonga_plugin_dir groonga:groonga-plugin-dir
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key percona_toolkit_location percona-toolkit:location
key template_php_ini template-php.ini:output
key template_apache_conf template-apache.conf:output
key template_apache_php instance-apache-php:output
key template_lamp instance-lamp:output
key template_php_ini template-php.ini:target
key template_apache_conf template-apache.conf:target
key template_apache_php instance-apache-php:target
key template_lamp instance-lamp:target
key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_mysqld_wrapper template-mysqld-wrapper:rendered
......@@ -107,19 +101,15 @@ context =
[instance-apache-php]
<= template-download-base
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
[instance-lamp]
<= template-download-base
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
[template-apache.conf]
<= template-download-base
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
[template-php.ini]
<= template-download-base
output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
[erp5]
# lamp stack reuses erp5 stack to have mariadb, but we don't need to checkout erp5 here.
......
......@@ -10,8 +10,6 @@ parts =
[logrotate-download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
# XXX: following mode should be the default
mode = 644
[logrotate-conf-template]
< = logrotate-download-base
......
......@@ -17,17 +17,8 @@ parts =
monitor2-template
[monitor-download-base]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[monitor-template-script]
<= monitor-download-base
url = ${:_profile_base_location_}/scripts/${:filename}
destination = ${buildout:parts-directory}/monitor-scripts
on-update = true
[monitor-eggs]
recipe = zc.recipe.egg
......
......@@ -26,7 +26,6 @@ parts =
recipe = zc.recipe.egg
eggs =
collective.recipe.template
collective.recipe.environment
#----------------
#--
......@@ -40,7 +39,6 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/pbsready.cfg
mode = 0644
[pbsready-import]
# An import instance has an importer script, which is called
......@@ -48,7 +46,6 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg
mode = 0644
[pbsready-export]
# An export instance has an exporter script, and communicates
......@@ -56,29 +53,24 @@ mode = 0644
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/pbsready-export.cfg.in
filename = pbsready-export.cfg.in
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/instance-pull-backup.cfg
mode = 0644
[template-replicated]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template-replicated.cfg.in
mode = 0644
[template-parts]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template-parts.cfg.in
mode = 0644
[template-resilient-templates]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}.in
output = ${buildout:directory}/${:filename}
mode = 0644
filename = template-resilient-templates.cfg
[instance-frozen]
......@@ -93,32 +85,26 @@ output = ${buildout:directory}/instance-frozen.cfg
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
filename = resilient-web-takeover-cgi-script.py.in
mode = 0644
# Provide an empty wrapper
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[notifier-feed-promise-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/notifier-feed-promise.py.in
filename = notifier-feed-promise.py.in
mode = 0644
##################
# Monitor element
#
[template-monitor-check-resilient-feed]
recipe = hexagonit.recipe.download
ignore-existing = true
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/monitor-check-resilient-feed.in
download-only = true
filename = monitor-check-resilient-feed.in
mode = 0644
[exporter-default-configuration]
# Time in minutes to wait for the backup to finish
......
......@@ -26,7 +26,7 @@ md5sum = 8f15263c4a27ec315eb3a12dbf7a7b34
[template-pull-backup]
filename = instance-pull-backup.cfg.in
md5sum = 4425db50d551fb8a974e547308990bac
md5sum = d1af7f8a5d4f0846e4c016253fa31f3d
[template-replicated]
filename = template-replicated.cfg.in
......
......@@ -250,6 +250,7 @@ monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password}
monitor-setup-url = $${monitor-publish:monitor-setup-url}
#----------------
#--
......@@ -269,7 +270,7 @@ private-path-list +=
[monitor-check-resilient-feed-file]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
template = ${template-monitor-check-resilient-feed:target}
rendered = $${monitor-directory:bin}/check-create-resilient-feed-files
mode = 700
context =
......
......@@ -38,7 +38,6 @@ exec-sitecustomize = false
# Add location for modified non-official slapos.buildout
find-links +=
http://www.nexedi.org/static/packages/source/
http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
http://www.nexedi.org/static/packages/source/slapos.buildout/
# Use only quite well working sites.
......@@ -126,6 +125,7 @@ eggs =
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
${python-PyYAML:egg}
slapos.core
slapos.libnetworkcache
......@@ -135,15 +135,13 @@ setuptools = 44.1.1
zc.buildout = 2.7.1+slapos014
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives
hexagonit.recipe.download = 1.7.post4
traitlets = 4.3.3
Jinja2 = 2.11.3
Importing = 1.10
MarkupSafe = 1.0
PyYAML = 5.4.1
Werkzeug = 1.0.1
Werkzeug = 2.0.2
ZConfig = 2.9.3
asn1crypto = 1.3.0
atomicwrites = 1.4.0
......@@ -171,6 +169,7 @@ itsdangerous = 0.24
lock-file = 2.0
lxml = 4.6.3
meld3 = 1.0.2
mock = 3.0.5
more-itertools = 5.0.0
netaddr = 0.7.19
pathlib2 = 2.3.5
......@@ -190,13 +189,13 @@ setproctitle = 1.1.10
setuptools-dso = 1.7
rubygemsrecipe = 0.4.3
six = 1.12.0
slapos.cookbook = 1.0.214
slapos.core = 1.6.19
slapos.cookbook = 1.0.220
slapos.core = 1.7.2
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
slapos.libnetworkcache = 0.22
slapos.rebootstrap = 4.5
slapos.recipe.build = 0.50
slapos.recipe.build = 0.52
slapos.recipe.cmmi = 0.18
slapos.recipe.template = 4.6
slapos.toolbox = 0.126
......@@ -251,6 +250,11 @@ chardet = 3.0.4
urllib3 = 1.25.9
pkgconfig = 1.5.1
[versions:python2]
Werkzeug = 1.0.1
[networkcache]
download-cache-url = http://shacache.nxdcdn.com
download-dir-url = http://shadir.nxdcdn.com
......
......@@ -10,7 +10,6 @@ parts =
[supervisord-download-base]
recipe = slapos.recipe.build:download
mode = 0644
url = ${:_profile_base_location_}/${:_update_hash_filename_}
......
#!/usr/bin/env python2
#!/usr/bin/env python
"""
Suggested installation:
Helper script to update buildout.hash.cfg
Usage:
$ ./update-hash [BUILDOUT_HASH_CFG]
Automatic installation using husky, from the root of the repository:
$ npm install
Manual installation:
$ cp update-hash .git/hooks/
$ $EDITOR .git/hooks/pre-commit
......@@ -36,7 +44,6 @@ import hashlib
import os
import shutil
import sys
import tempfile
# Note: this is an intentionally very restrictive and primitive
# ConfigParser-ish parser.
......@@ -62,7 +69,11 @@ def main():
try:
outfile = os.fdopen(outfile_fd, 'w')
write = outfile.write
if sys.version_info <= (3,):
nextLine = iter(infile).next
else:
nextLine = iter(infile).__next__
while True:
try:
line = nextLine()
......@@ -80,7 +91,8 @@ def main():
os.path.join(
infile_dirname,
*hash_file_path.split('/')
)
),
'rb',
).read()
).hexdigest(),
eol,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment