Commit 57b92ffa authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

Conflicts:
	software/slapos-master/software.cfg
parents e0bc169e 70744c39
...@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c ...@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
[apache] [apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/ # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 2.4.27 version = 2.4.29
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 97b6bbfa83c866dbe20ef317e3afd108 md5sum = 0c599404ef6b69eee95bcd9fcd094407
pre-configure = pre-configure =
cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ && cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
......
From 8089fa02122fed0a6394eba14bbedcb1d18e2384 Mon Sep 17 00:00:00 2001
From: Kevin Atkinson <kevina@gnu.org>
Date: Thu, 29 Dec 2016 00:50:31 -0500
Subject: Compile Fixes for GCC 7.
Closes #519.
---
modules/filter/tex.cpp | 2 +-
prog/check_funs.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/filter/tex.cpp b/modules/filter/tex.cpp
index a979539..19ab63c 100644
--- a/modules/filter/tex.cpp
+++ b/modules/filter/tex.cpp
@@ -174,7 +174,7 @@ namespace {
if (c == '{') {
- if (top.in_what == Parm || top.in_what == Opt || top.do_check == '\0')
+ if (top.in_what == Parm || top.in_what == Opt || *top.do_check == '\0')
push_command(Parm);
top.in_what = Parm;
diff --git a/prog/check_funs.cpp b/prog/check_funs.cpp
index db54f3d..89ee09d 100644
--- a/prog/check_funs.cpp
+++ b/prog/check_funs.cpp
@@ -647,7 +647,7 @@ static void print_truncate(FILE * out, const char * word, int width) {
}
}
if (i == width-1) {
- if (word == '\0')
+ if (*word == '\0')
put(out,' ');
else if (word[len] == '\0')
put(out, word, len);
--
2.15.0.rc0
...@@ -5,6 +5,7 @@ parts = ...@@ -5,6 +5,7 @@ parts =
extends = extends =
../ncurses/buildout.cfg ../ncurses/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg ../perl/buildout.cfg
[aspell-dictionary-common] [aspell-dictionary-common]
...@@ -15,8 +16,11 @@ configure-command = ./configure --vars ASPELL=${aspell:location}/bin/aspell PREZ ...@@ -15,8 +16,11 @@ configure-command = ./configure --vars ASPELL=${aspell:location}/bin/aspell PREZ
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/aspell/aspell-0.60.6.1.tar.gz url = http://ftp.gnu.org/gnu/aspell/aspell-0.60.6.1.tar.gz
md5sum = e66a9c9af6a60dc46134fdacf6ce97d7 md5sum = e66a9c9af6a60dc46134fdacf6ce97d7
patch-options = -p1
patches =
${:_profile_base_location_}/Compile-Fixes-for-GCC-7.patch#0159a8841e1241153d2105d157fc8225
environment = environment =
PATH=${perl:location}/bin:%(PATH)s PATH=${patch:location}/bin:${perl:location}/bin:%(PATH)s
CPPFLAGS=-I${ncurses:location}/include CPPFLAGS=-I${ncurses:location}/include
LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
......
diff -N -u -r bazel.orig/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java bazel/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java
--- bazel.orig/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java 1980-01-01 00:00:00.000000000 +0900
+++ bazel/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java 2017-01-02 22:19:09.326924214 +0900
@@ -172,7 +172,7 @@
@Param(
name = "use_default_shell_env",
type = Boolean.class,
- defaultValue = "False",
+ defaultValue = "True",
named = true,
positional = false,
doc = "whether the action should use the built in shell environment or not"
diff -u -r a/compile.sh b/compile.sh
--- a/compile.sh 1980-01-01 00:00:00.000000000 +0900
+++ b/compile.sh 2017-10-24 10:57:21.469702614 +0900
@@ -49,6 +49,8 @@
# a chance of overriding this in case they want to do so.
: ${VERBOSE:=no}
+export LDFLAGS=$LDFLAGS
+
source scripts/bootstrap/buildenv.sh
mkdir -p output
diff -u -r a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java
--- a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java 1980-01-01 00:00:00.000000000 +0900
+++ b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java 2017-10-18 17:44:51.975049512 +0900
@@ -165,7 +165,7 @@
@Param(
name = "use_default_shell_env",
type = Boolean.class,
- defaultValue = "False",
+ defaultValue = "True",
named = true,
positional = false,
doc = "whether the action should use the built in shell environment or not."
# Description:
# The Bazel launcher.
package(
default_visibility = ["//visibility:public"],
)
WIN_LINK_OPTS = [
"-DEFAULTLIB:advapi32.lib", # GetUserNameW
"-DEFAULTLIB:ole32.lib", # CoTaskMemFree
"-DEFAULTLIB:shell32.lib", # SHGetKnownFolderPath
"-DEFAULTLIB:ws2_32.lib", # grpc
]
cc_library(
name = "blaze_util",
srcs = [
"blaze_util.cc",
"global_variables.h",
"startup_options.h",
] + select({
"//src:darwin": [
"blaze_util_darwin.cc",
"blaze_util_posix.cc",
],
"//src:darwin_x86_64": [
"blaze_util_darwin.cc",
"blaze_util_posix.cc",
],
"//src:freebsd": [
"blaze_util_freebsd.cc",
"blaze_util_posix.cc",
],
"//src:windows": [
"blaze_util_windows.cc",
],
"//src:windows_msys": [
"blaze_util_windows.cc",
],
"//src:windows_msvc": [
"blaze_util_windows.cc",
],
"//conditions:default": [
"blaze_util_linux.cc",
"blaze_util_posix.cc",
],
}),
hdrs = [
"blaze_util.h",
"blaze_util_platform.h",
],
linkopts = select({
"//src:darwin": [
"-framework CoreFoundation",
],
"//src:darwin_x86_64": [
"-framework CoreFoundation",
],
"//src:freebsd": [
],
"//src:windows": WIN_LINK_OPTS,
"//src:windows_msvc": WIN_LINK_OPTS,
"//conditions:default": [
"-lrt",
"{{ linkopts }}",
],
}),
deps = [
"//src/main/cpp/util",
"//src/main/cpp/util:blaze_exit_code",
] + select({
"//src:windows": ["//src/main/native/windows:lib-file"],
"//src:windows_msys": ["//src/main/native/windows:lib-file"],
"//src:windows_msvc": ["//src/main/native/windows:lib-file"],
"//conditions:default": [],
}),
)
cc_binary(
name = "client",
srcs = [
"blaze.cc",
"blaze.h",
"global_variables.cc",
"global_variables.h",
"main.cc",
] + select({
"//src:windows": ["//src/main/native/windows:resources.o"],
"//src:windows_msvc": ["//src/main/native/windows:resources.o"],
"//conditions:default": [],
}),
copts = select({
"//src:windows": ["/wd4018"],
"//src:windows_msvc": ["/wd4018"],
"//conditions:default": ["-Wno-sign-compare"],
}),
linkopts = select({
"//src:darwin": [
],
"//src:darwin_x86_64": [
],
"//src:freebsd": [
"-lprocstat",
"-lm",
],
"//src:windows_msvc": [
],
"//conditions:default": [
"-lrt",
"-ldl",
],
}),
visibility = ["//src:__pkg__"],
deps = [
":blaze_util",
":option_processor",
":startup_options",
":workspace_layout",
"//src/main/cpp/util",
"//src/main/cpp/util:errors",
"//src/main/cpp/util:logging",
"//src/main/cpp/util:strings",
"//src/main/protobuf:command_server_cc_proto",
"//third_party/ijar:zip",
],
)
cc_library(
name = "option_processor",
srcs = ["option_processor.cc"],
hdrs = [
"option_processor.h",
"option_processor-internal.h",
],
visibility = [
"//src:__pkg__",
"//src/test/cpp:__pkg__",
],
deps = [
":blaze_util",
":startup_options",
":workspace_layout",
"//src/main/cpp/util",
"//src/main/cpp/util:blaze_exit_code",
"//src/main/cpp/util:logging",
],
)
cc_library(
name = "startup_options",
srcs = ["startup_options.cc"],
hdrs = ["startup_options.h"],
visibility = [
"//src:__pkg__",
"//src/test/cpp:__pkg__",
],
deps = [
":blaze_util",
":workspace_layout",
"//src/main/cpp/util",
"//src/main/cpp/util:blaze_exit_code",
"//src/main/cpp/util:errors",
],
)
cc_library(
name = "workspace_layout",
srcs = ["workspace_layout.cc"],
hdrs = ["workspace_layout.h"],
visibility = [
"//src:__pkg__",
"//src/test/cpp:__pkg__",
],
deps = [
":blaze_util",
"//src/main/cpp/util",
],
)
filegroup(
name = "srcs",
srcs = glob(["**"]) + ["//src/main/cpp/util:srcs"],
visibility = ["//src:__pkg__"],
)
package(default_visibility = ["//src:__subpackages__"])
cc_library(
name = "logging",
srcs = ["logging.cc"],
hdrs = ["logging.h"],
)
cc_library(
name = "process-tools",
srcs = ["process-tools.cc"],
hdrs = ["process-tools.h"],
deps = [":logging"],
)
cc_binary(
name = "process-wrapper",
srcs = select({
"//src:windows": ["process-wrapper-windows.cc"],
"//src:windows_msvc": ["process-wrapper-windows.cc"],
"//conditions:default": [
"process-wrapper.cc",
"process-wrapper.h",
"process-wrapper-legacy.cc",
"process-wrapper-legacy.h",
"process-wrapper-options.cc",
"process-wrapper-options.h",
],
}),
linkopts = ["-lm", "{{ linkopts }}"],
deps = select({
"//src:windows": [],
"//src:windows_msvc": [],
"//conditions:default": [
":process-tools",
":logging",
],
}),
)
cc_binary(
name = "build-runfiles",
srcs = select({
"//src:windows": ["build-runfiles-windows.cc"],
"//src:windows_msvc": ["build-runfiles-windows.cc"],
"//conditions:default": ["build-runfiles.cc"],
}),
linkopts = ["{{ linkopts }}"],
)
cc_binary(
name = "linux-sandbox",
srcs = select({
"//src:darwin": ["dummy-sandbox.c"],
"//src:darwin_x86_64": ["dummy-sandbox.c"],
"//src:freebsd": ["dummy-sandbox.c"],
"//src:windows": ["dummy-sandbox.c"],
"//src:windows_msys": ["dummy-sandbox.c"],
"//src:windows_msvc": ["dummy-sandbox.c"],
"//conditions:default": [
"linux-sandbox.cc",
"linux-sandbox.h",
"linux-sandbox-options.cc",
"linux-sandbox-options.h",
"linux-sandbox-pid1.cc",
"linux-sandbox-pid1.h",
],
}),
linkopts = ["-lm", "{{ linkopts }}"],
deps = select({
"//src:darwin": [],
"//src:darwin_x86_64": [],
"//src:freebsd": [],
"//src:windows": [],
"//src:windows_msys": [],
"//src:windows_msvc": [],
"//conditions:default": [
":logging",
":process-tools",
],
}),
)
filegroup(
name = "jdk-support",
srcs = [
"jdk.BUILD",
],
)
exports_files([
"build_interface_so",
])
filegroup(
name = "srcs",
srcs = glob(["**"]),
visibility = ["//src:__pkg__"],
)
# Description:
# singlejar C++ implementation.
package(default_visibility = ["//src:__subpackages__"])
JAR_TOOL_PATH_COPT_TPL = "-DJAR_TOOL_PATH=\\\"external/local_jdk/bin/jar%s\\\""
JAR_TOOL_PATH_COPTS = select({
"//src:windows": [JAR_TOOL_PATH_COPT_TPL % ".exe"],
"//src:windows_msvc": [JAR_TOOL_PATH_COPT_TPL % ".exe"],
"//src:windows_msys": [JAR_TOOL_PATH_COPT_TPL % ".exe"],
"//conditions:default": [JAR_TOOL_PATH_COPT_TPL % ""],
})
filegroup(
name = "srcs",
srcs = glob(["**"]),
visibility = ["//src:__pkg__"],
)
filegroup(
name = "embedded_tools",
srcs = [
"BUILD",
"combiners.cc",
"combiners.h",
"diag.h",
"input_jar.cc",
"input_jar.h",
"mapped_file.h",
"options.cc",
"options.h",
"output_jar.cc",
"output_jar.h",
"singlejar_main.cc",
"token_stream.h",
"transient_bytes.h",
"zip_headers.h",
"zlib_interface.h",
],
visibility = ["//visibility:public"],
)
cc_binary(
name = "singlejar",
srcs = [
"singlejar_main.cc",
],
linkopts = select({
"//src:freebsd": ["-lm"],
"//conditions:default": ["{{ linkopts }}"],
}),
linkstatic = 1,
visibility = ["//visibility:public"],
deps = [
"options",
"output_jar",
"//third_party/zlib",
],
)
cc_test(
name = "combiners_test",
size = "large",
srcs = [
"combiners_test.cc",
":zip_headers",
":zlib_interface",
],
deps = [
":combiners",
":input_jar",
"//third_party:gtest",
"//third_party/zlib",
],
)
cc_test(
name = "input_jar_empty_jar_test",
srcs = [
"input_jar_empty_jar_test.cc",
],
data = [
"data/empty.zip",
],
deps = [
":input_jar",
":test_util",
"//src/main/cpp/util",
"//third_party:gtest",
],
)
cc_test(
name = "input_jar_preambled_test",
srcs = [
"input_jar_preambled_test.cc",
],
data = [
":test1",
],
deps = [
":input_jar",
":test_util",
"//src/main/cpp/util",
"//third_party:gtest",
],
)
cc_test(
name = "input_jar_scan_jartool_test",
size = "large",
srcs = [
"input_jar_scan_entries_test.h",
"input_jar_scan_jartool_test.cc",
],
copts = JAR_TOOL_PATH_COPTS,
data = [
"@local_jdk//:jar",
"@local_jdk//:jdk",
],
# Timing out, see https://github.com/bazelbuild/bazel/issues/1555
tags = ["manual"],
deps = [
":input_jar",
":test_util",
"//third_party:gtest",
],
)
cc_test(
name = "input_jar_scan_ziptool_test",
size = "large",
srcs = [
"input_jar_scan_entries_test.h",
"input_jar_scan_ziptool_test.cc",
],
# Timing out, see https://github.com/bazelbuild/bazel/issues/1555
tags = ["manual"],
deps = [
":input_jar",
":test_util",
"//third_party:gtest",
],
)
cc_test(
name = "input_jar_bad_jar_test",
srcs = [
"input_jar_bad_jar_test.cc",
],
deps = [
":input_jar",
":test_util",
"//third_party:gtest",
],
)
cc_test(
name = "options_test",
srcs = [
"options.h",
"options_test.cc",
],
deps = [
":options",
":test_util",
":token_stream",
"//src/main/cpp/util",
"//third_party:gtest",
],
)
sh_test(
name = "output_jar_bash_test",
srcs = ["output_jar_shell_test.sh"],
args = [
"src/test/shell",
"$(location :singlejar)",
],
data = [
":singlejar",
],
deps = ["//src/test/shell:bashunit"],
)
cc_test(
name = "output_jar_simple_test",
srcs = [
"output_jar_simple_test.cc",
],
copts = JAR_TOOL_PATH_COPTS,
data = [
":data1",
":data2",
":stored_jar",
":test1",
":test2",
"@local_jdk//:jar",
"@local_jdk//:jdk-default",
],
deps = [
":input_jar",
":options",
":output_jar",
":test_util",
"//src/main/cpp/util",
"//third_party:gtest",
],
)
cc_test(
name = "token_stream_test",
srcs = [
"token_stream_test.cc",
],
deps = [
":test_util",
":token_stream",
"//third_party:gtest",
],
)
cc_test(
name = "transient_bytes_test",
size = "large",
srcs = [
"transient_bytes_test.cc",
":transient_bytes",
":zlib_interface",
],
# Timing out, see https://github.com/bazelbuild/bazel/issues/1555
tags = ["manual"],
deps = [
":input_jar",
":test_util",
"//third_party:gtest",
"//third_party/zlib",
],
)
cc_test(
name = "zip_headers_test",
size = "small",
srcs = [
"zip_headers_test.cc",
":zip_headers",
],
deps = ["//third_party:gtest"],
)
cc_test(
name = "zlib_interface_test",
srcs = [
"zlib_interface_test.cc",
":zlib_interface",
],
deps = [
":test_util",
"//third_party:gtest",
"//third_party/zlib",
],
)
sh_test(
name = "zip64_test",
srcs = ["zip64_test.sh"],
args = [
"src/test/shell",
"$(location :singlejar)",
"$(location @local_jdk//:jar)",
],
data = [
":singlejar",
"//tools/defaults:jdk",
"@local_jdk//:jar",
],
deps = ["//src/test/shell:bashunit"],
)
cc_library(
name = "combiners",
srcs = [
"combiners.cc",
":transient_bytes",
":zip_headers",
],
hdrs = ["combiners.h"],
deps = ["//third_party/zlib"],
)
cc_library(
name = "input_jar",
srcs = [
"diag.h",
"input_jar.cc",
"mapped_file.h",
],
hdrs = [
"input_jar.h",
"zip_headers.h",
],
)
cc_library(
name = "options",
srcs = [
"diag.h",
"options.cc",
"options.h",
],
hdrs = ["options.h"],
deps = [
":token_stream",
],
)
cc_library(
name = "output_jar",
srcs = [
"diag.h",
"mapped_file.h",
"output_jar.cc",
"output_jar.h",
":zip_headers",
],
hdrs = ["output_jar.h"],
deps = [
":combiners",
":input_jar",
":options",
"//src/main/cpp/util",
"//third_party/zlib",
],
)
cc_library(
name = "test_util",
srcs = ["test_util.cc"],
hdrs = ["test_util.h"],
deps = [
"//src/main/cpp/util",
"//third_party:gtest",
],
)
cc_library(
name = "token_stream",
srcs = ["diag.h"],
hdrs = ["token_stream.h"],
)
filegroup(
name = "transient_bytes",
srcs = [
"diag.h",
"transient_bytes.h",
"zlib_interface.h",
":zip_headers",
],
)
filegroup(
name = "zip_headers",
srcs = ["zip_headers.h"],
)
filegroup(
name = "zlib_interface",
srcs = [
"diag.h",
"zlib_interface.h",
],
)
java_library(
name = "test1",
resources = [
"options.cc",
"zip_headers.h",
"zlib_interface.h",
],
)
java_library(
name = "test2",
resources = [
"token_stream.h",
"transient_bytes.h",
],
)
java_library(
name = "data1",
resources = [
"data/extra_file1",
"data/extra_file2",
],
)
java_library(
name = "data2",
resources = [
"data/extra_file1",
"data/extra_file3",
],
)
genrule(
name = "stored_jar",
srcs = [
"output_jar.cc",
"//tools/defaults:jdk",
],
outs = ["stored.jar"],
cmd = "$(location @local_jdk//:jar) -0cf \"$@\" $(location :output_jar.cc)",
tools = ["@local_jdk//:jar"],
)
...@@ -39,6 +39,21 @@ default_toolchain { ...@@ -39,6 +39,21 @@ default_toolchain {
toolchain_identifier: "local_linux" toolchain_identifier: "local_linux"
} }
default_toolchain {
cpu: "ppc"
toolchain_identifier: "local_linux"
}
default_toolchain {
cpu: "ppc64"
toolchain_identifier: "local_linux"
}
default_toolchain {
cpu: "ios_x86_64"
toolchain_identifier: "ios_x86_64"
}
toolchain { toolchain {
abi_version: "armeabi-v7a" abi_version: "armeabi-v7a"
abi_libc_version: "armeabi-v7a" abi_libc_version: "armeabi-v7a"
...@@ -90,8 +105,8 @@ toolchain { ...@@ -90,8 +105,8 @@ toolchain {
target_system_name: "local" target_system_name: "local"
toolchain_identifier: "local_linux" toolchain_identifier: "local_linux"
tool_path { name: "ar" path: "/usr/bin/ar" } tool_path { name: "ar" path: "{{ binutils_path }}/ar" }
tool_path { name: "compat-ld" path: "/usr/bin/ld" } tool_path { name: "compat-ld" path: "{{ binutils_path }}/ld" }
tool_path { name: "cpp" path: "{{ cpp_path }}" } tool_path { name: "cpp" path: "{{ cpp_path }}" }
tool_path { name: "dwp" path: "/usr/bin/dwp" } tool_path { name: "dwp" path: "/usr/bin/dwp" }
tool_path { name: "gcc" path: "{{ gcc_path }}" } tool_path { name: "gcc" path: "{{ gcc_path }}" }
...@@ -112,14 +127,14 @@ toolchain { ...@@ -112,14 +127,14 @@ toolchain {
# C(++) compiles invoke the compiler (as that is the one knowing where # C(++) compiles invoke the compiler (as that is the one knowing where
# to find libraries), but we provide LD so other rules can invoke the linker. # to find libraries), but we provide LD so other rules can invoke the linker.
tool_path { name: "ld" path: "/usr/bin/ld" } tool_path { name: "ld" path: "{{ binutils_path }}/ld" }
tool_path { name: "nm" path: "/usr/bin/nm" } tool_path { name: "nm" path: "{{ binutils_path }}/nm" }
tool_path { name: "objcopy" path: "/usr/bin/objcopy" } tool_path { name: "objcopy" path: "{{ binutils_path }}/objcopy" }
objcopy_embed_flag: "-I" objcopy_embed_flag: "-I"
objcopy_embed_flag: "binary" objcopy_embed_flag: "binary"
tool_path { name: "objdump" path: "/usr/bin/objdump" } tool_path { name: "objdump" path: "{{ binutils_path }}/objdump" }
tool_path { name: "strip" path: "/usr/bin/strip" } tool_path { name: "strip" path: "{{ binutils_path }}/strip" }
# Anticipated future default. # Anticipated future default.
unfiltered_cxx_flag: "-no-canonical-prefixes" unfiltered_cxx_flag: "-no-canonical-prefixes"
...@@ -160,9 +175,6 @@ toolchain { ...@@ -160,9 +175,6 @@ toolchain {
linker_flag: "-no-canonical-prefixes" linker_flag: "-no-canonical-prefixes"
# Have gcc return the exit code from ld. # Have gcc return the exit code from ld.
linker_flag: "-pass-exit-codes" linker_flag: "-pass-exit-codes"
# Stamp the binary with a unique identifier.
linker_flag: "-Wl,--build-id=md5"
linker_flag: "-Wl,--hash-style=gnu"
# Gold linker only? Can we enable this by default? # Gold linker only? Can we enable this by default?
# linker_flag: "-Wl,--warn-execstack" # linker_flag: "-Wl,--warn-execstack"
# linker_flag: "-Wl,--detect-odr-violations" # linker_flag: "-Wl,--detect-odr-violations"
...@@ -215,9 +227,6 @@ toolchain { ...@@ -215,9 +227,6 @@ toolchain {
tool_path { name: "dwp" path: "/usr/bin/dwp" } tool_path { name: "dwp" path: "/usr/bin/dwp" }
tool_path { name: "gcc" path: "osx_cc_wrapper.sh" } tool_path { name: "gcc" path: "osx_cc_wrapper.sh" }
cxx_flag: "-std=c++0x" cxx_flag: "-std=c++0x"
ar_flag: "-static"
ar_flag: "-s"
ar_flag: "-o"
linker_flag: "-lstdc++" linker_flag: "-lstdc++"
linker_flag: "-undefined" linker_flag: "-undefined"
linker_flag: "dynamic_lookup" linker_flag: "dynamic_lookup"
...@@ -378,8 +387,6 @@ toolchain { ...@@ -378,8 +387,6 @@ toolchain {
# Have gcc return the exit code from ld. # Have gcc return the exit code from ld.
#linker_flag: "-pass-exit-codes" #linker_flag: "-pass-exit-codes"
# Stamp the binary with a unique identifier. # Stamp the binary with a unique identifier.
#linker_flag: "-Wl,--build-id=md5"
linker_flag: "-Wl,--hash-style=gnu"
# Gold linker only? Can we enable this by default? # Gold linker only? Can we enable this by default?
# linker_flag: "-Wl,--warn-execstack" # linker_flag: "-Wl,--warn-execstack"
# linker_flag: "-Wl,--detect-odr-violations" # linker_flag: "-Wl,--detect-odr-violations"
...@@ -618,7 +625,6 @@ toolchain { ...@@ -618,7 +625,6 @@ toolchain {
compiler_flag: "-m64" compiler_flag: "-m64"
compiler_flag: "/D__inline__=__inline" compiler_flag: "/D__inline__=__inline"
# TODO(pcloudy): Review those flags below, they should be defined by cl.exe # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
compiler_flag: "/DOS_WINDOWS=OS_WINDOWS"
compiler_flag: "/DCOMPILER_MSVC" compiler_flag: "/DCOMPILER_MSVC"
# Don't pollute with GDI macros in windows.h. # Don't pollute with GDI macros in windows.h.
...@@ -664,46 +670,6 @@ toolchain { ...@@ -664,46 +670,6 @@ toolchain {
compiler_flag: "/wd4996" compiler_flag: "/wd4996"
linker_flag: "-m64" linker_flag: "-m64"
feature {
name: 'include_paths'
flag_set {
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-header-parsing'
action: 'c++-header-preprocessing'
action: 'c++-module-compile'
flag_group {
flag: '/I%{quote_include_paths}'
}
flag_group {
flag: '/I%{include_paths}'
}
flag_group {
flag: '/I%{system_include_paths}'
}
}
}
feature {
name: 'dependency_file'
flag_set {
action: 'assemble'
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-module-compile'
action: 'c++-header-preprocessing'
action: 'c++-header-parsing'
expand_if_all_available: 'dependency_file'
flag_group {
flag: '/DEPENDENCY_FILE'
flag: '%{dependency_file}'
}
}
}
# Stop passing -frandom-seed option # Stop passing -frandom-seed option
feature { feature {
name: 'random_seed' name: 'random_seed'
...@@ -745,6 +711,10 @@ toolchain { ...@@ -745,6 +711,10 @@ toolchain {
flag: '/Fi%{output_preprocess_file}' flag: '/Fi%{output_preprocess_file}'
} }
} }
implies: 'legacy_compile_flags'
implies: 'user_compile_flags'
implies: 'sysroot'
implies: 'unfiltered_compile_flags'
} }
action_config { action_config {
...@@ -778,6 +748,132 @@ toolchain { ...@@ -778,6 +748,132 @@ toolchain {
flag: '/Fi%{output_preprocess_file}' flag: '/Fi%{output_preprocess_file}'
} }
} }
implies: 'legacy_compile_flags'
implies: 'user_compile_flags'
implies: 'sysroot'
implies: 'unfiltered_compile_flags'
}
# TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
# not used in this crosstool
feature {
name: 'legacy_compile_flags'
flag_set {
expand_if_all_available: 'legacy_compile_flags'
action: 'assemble'
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-header-parsing'
action: 'c++-header-preprocessing'
action: 'c++-module-compile'
action: 'c++-module-codegen'
flag_group {
iterate_over: 'legacy_compile_flags'
flag: '%{legacy_compile_flags}'
}
}
}
feature {
name: 'include_paths'
flag_set {
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-header-parsing'
action: 'c++-header-preprocessing'
action: 'c++-module-compile'
flag_group {
iterate_over: 'quote_include_paths'
flag: '/I%{quote_include_paths}'
}
flag_group {
iterate_over: 'include_paths'
flag: '/I%{include_paths}'
}
flag_group {
iterate_over: 'system_include_paths'
flag: '/I%{system_include_paths}'
}
}
}
feature {
name: 'dependency_file'
flag_set {
action: 'assemble'
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-module-compile'
action: 'c++-header-preprocessing'
action: 'c++-header-parsing'
expand_if_all_available: 'dependency_file'
flag_group {
flag: '/DEPENDENCY_FILE'
flag: '%{dependency_file}'
}
}
}
feature {
name: 'user_compile_flags'
flag_set {
expand_if_all_available: 'user_compile_flags'
action: 'assemble'
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-header-parsing'
action: 'c++-header-preprocessing'
action: 'c++-module-compile'
action: 'c++-module-codegen'
flag_group {
iterate_over: 'user_compile_flags'
flag: '%{user_compile_flags}'
}
}
}
feature {
name: 'sysroot'
flag_set {
expand_if_all_available: 'sysroot'
action: 'assemble'
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-header-parsing'
action: 'c++-header-preprocessing'
action: 'c++-module-compile'
action: 'c++-module-codegen'
action: 'c++-link-executable'
action: 'c++-link-dynamic-library'
flag_group {
iterate_over: 'sysroot'
flag: '--sysroot=%{sysroot}'
}
}
}
feature {
name: 'unfiltered_compile_flags'
flag_set {
expand_if_all_available: 'unfiltered_compile_flags'
action: 'assemble'
action: 'preprocess-assemble'
action: 'c-compile'
action: 'c++-compile'
action: 'c++-header-parsing'
action: 'c++-header-preprocessing'
action: 'c++-module-compile'
action: 'c++-module-codegen'
flag_group {
iterate_over: 'unfiltered_compile_flags'
flag: '%{unfiltered_compile_flags}'
}
}
} }
compilation_mode_flags { compilation_mode_flags {
...@@ -805,3 +901,37 @@ toolchain { ...@@ -805,3 +901,37 @@ toolchain {
compiler_flag: "-Xcompilation-mode=opt" compiler_flag: "-Xcompilation-mode=opt"
} }
} }
# Stub crosstool to allow construction of apple crosstool configuration in a
# static configuration set.
# TODO(b/33804121): Remove this once dynamic configurations are in place.
toolchain {
toolchain_identifier: "ios_x86_64"
host_system_name: "x86_64-apple-macosx"
target_system_name: "x86_64-apple-ios"
target_cpu: "ios_x86_64"
target_libc: "ios"
compiler: "compiler"
abi_version: "local"
abi_libc_version: "local"
supports_gold_linker: false
supports_incremental_linker: false
supports_fission: false
supports_interface_shared_objects: false
supports_normalizing_ar: false
supports_start_end_lib: false
tool_path { name: "ar" path: "/bin/false" }
tool_path { name: "compat-ld" path: "/bin/false" }
tool_path { name: "cpp" path: "/bin/false" }
tool_path { name: "dwp" path: "/bin/false" }
tool_path { name: "gcc" path: "/bin/false" }
tool_path { name: "gcov" path: "/bin/false" }
tool_path { name: "ld" path: "/bin/false" }
tool_path { name: "nm" path: "/bin/false" }
tool_path { name: "objcopy" path: "/bin/false" }
tool_path { name: "objdump" path: "/bin/false" }
tool_path { name: "strip" path: "/bin/false" }
linking_mode_flags { mode: DYNAMIC }
}
build --verbose_failures
...@@ -24,36 +24,74 @@ template = ${:_profile_base_location_}/${:filename}.in ...@@ -24,36 +24,74 @@ template = ${:_profile_base_location_}/${:filename}.in
rendered = ${:location}/${:filename} rendered = ${:location}/${:filename}
cpp_path = ${gcc:location}/bin/cpp cpp_path = ${gcc:location}/bin/cpp
gcc_path = ${gcc:location}/bin/gcc gcc_path = ${gcc:location}/bin/gcc
binutils_path = ${binutils:location}/bin
gconv_path = ${gcc:location}/bin/gconv gconv_path = ${gcc:location}/bin/gconv
include_path = ${gcc:location}/include include_path = ${gcc:location}/include
gcc_lib64_path = ${gcc:location}/lib64 gcc_lib64_path = ${gcc:location}/lib64
context = context =
key cpp_path template-bazel-crosstool:cpp_path key cpp_path template-bazel-crosstool:cpp_path
key gcc_path template-bazel-crosstool:gcc_path key gcc_path template-bazel-crosstool:gcc_path
key binutils_path template-bazel-crosstool:binutils_path
key gconv_path template-bazel-crosstool:gconv_path key gconv_path template-bazel-crosstool:gconv_path
key include_path template-bazel-crosstool:include_path key include_path template-bazel-crosstool:include_path
key gcc_lib64_path template-bazel-crosstool:gcc_lib64_path key gcc_lib64_path template-bazel-crosstool:gcc_lib64_path
[template-bazel-src-main-cpp-build]
recipe = slapos.recipe.template:jinja2
location = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 640
filename = bazel_src_main_cpp_BUILD
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${:location}/${:filename}
linkopts = -Wl,-rpath,${gcc:location}/lib64
context =
key linkopts template-bazel-src-main-cpp-build:linkopts
[template-bazel-src-main-tools-build]
<= template-bazel-src-main-cpp-build
filename = bazel_src_main_tools_BUILD
[template-bazel-src-tools-singlejar-build]
<= template-bazel-src-main-cpp-build
filename = bazel_src_tools_singlejar_BUILD
[bazel] [bazel]
recipe = slapos.recipe.build recipe = slapos.recipe.build
url = https://github.com/bazelbuild/bazel/releases/download/0.4.3/bazel-0.4.3-dist.zip url = https://github.com/bazelbuild/bazel/releases/download/0.6.1/bazel-0.6.1-dist.zip
md5sum = cbd53f6f59915506da8998dab2098921 md5sum = 8c5c827e33d3ff74c263c1299810b485
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
patch-file-path = ${:_profile_base_location_}/bazel-0.4.3.patch patch-file-path = ${:_profile_base_location_}/bazel-0.6.1.patch
bazel-crosstool-modified-file-path = ${template-bazel-crosstool:rendered} bazel-crosstool-modified-file-path = ${template-bazel-crosstool:rendered}
bazel-src-main-cpp-build-path = ${template-bazel-src-main-cpp-build:rendered}
bazel-src-main-tools-build-path = ${template-bazel-src-main-tools-build:rendered}
bazel-src-tools-singlejar-build-path = ${template-bazel-src-tools-singlejar-build:rendered}
unzip-bin = ${unzip:location}/bin unzip-bin = ${unzip:location}/bin
zip-bin = ${zip:location}/bin zip-bin = ${zip:location}/bin
gcc-bin = ${gcc:location}/bin gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64 gcc-lib64 = ${gcc:location}/lib64
java_home = ${zulu:location} java_home = ${zulu:location}
bazelrc = ${:_profile_base_location_}/bazelrc
script = script =
extract_dir = self.extract(self.download(self.options['url'], self.options['md5sum'])) extract_dir = self.extract(self.download(self.options['url'], self.options['md5sum']))
crosstool_path = os.path.join(extract_dir, 'tools', 'cpp', 'CROSSTOOL') crosstool_path = os.path.join(extract_dir, 'tools', 'cpp', 'CROSSTOOL')
os.chmod(crosstool_path, 0644) os.chmod(crosstool_path, 0644)
shutil.copy(self.options['bazel-crosstool-modified-file-path'], shutil.copy(self.options['bazel-crosstool-modified-file-path'],
crosstool_path) crosstool_path)
target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java' src_main_cpp_build_path = os.path.join(extract_dir, 'src', 'main', 'cpp', 'BUILD')
os.chmod(src_main_cpp_build_path, 0644)
shutil.copy(self.options['bazel-src-main-cpp-build-path'],
src_main_cpp_build_path)
src_main_tools_build_path = os.path.join(extract_dir, 'src', 'main', 'tools', 'BUILD')
os.chmod(src_main_tools_build_path, 0644)
shutil.copy(self.options['bazel-src-main-tools-build-path'],
src_main_tools_build_path)
src_tools_singlejar_build_path = os.path.join(extract_dir, 'src', 'tools', 'singlejar', 'BUILD')
os.chmod(src_tools_singlejar_build_path, 0644)
shutil.copy(self.options['bazel-src-tools-singlejar-build-path'],
src_tools_singlejar_build_path)
target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java'
os.chmod(target_path, 0644) os.chmod(target_path, 0644)
call([self.buildout['bazel']['patch-binary'], '-p1', '-d', extract_dir, '-i', self.buildout['bazel']['patch-file-path']]) call([self.buildout['bazel']['patch-binary'], '-p1', '-d', extract_dir, '-i', self.buildout['bazel']['patch-file-path']])
path = ':'.join(( path = ':'.join((
...@@ -72,6 +110,8 @@ script = ...@@ -72,6 +110,8 @@ script =
'LDFLAGS':'-Wl,-rpath='+self.options['gcc-lib64'], 'LDFLAGS':'-Wl,-rpath='+self.options['gcc-lib64'],
'CC':self.options['gcc-bin']+'/gcc', 'CC':self.options['gcc-bin']+'/gcc',
'CXX':self.options['gcc-bin']+'/g++', 'CXX':self.options['gcc-bin']+'/g++',
'BAZELRC':self.options['bazelrc'],
'VERBOSE':'yes',
} }
bin_dir = os.path.join(self.options['location'], 'bin') bin_dir = os.path.join(self.options['location'], 'bin')
os.makedirs(bin_dir) os.makedirs(bin_dir)
......
...@@ -11,9 +11,9 @@ parts = ...@@ -11,9 +11,9 @@ parts =
[ca-certificates] [ca-certificates]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130+nmu1.tar.xz url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20170717.tar.xz
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
md5sum = a09e8b63126188fd0ed77f6fbaf5d35f md5sum = 55a6bb6b98afb16b3cde8e3ad1e262eb
patches = patches =
${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2 ${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
......
...@@ -40,7 +40,7 @@ interpreter = chainer-python ...@@ -40,7 +40,7 @@ interpreter = chainer-python
scripts = chainer-python scripts = chainer-python
[versions] [versions]
chainer = 1.22.0 chainer = 3.0.0
filelock = 2.0.7 filelock = 2.0.12
nose = 1.3.7 nose = 1.3.7
six = 1.10.0 six = 1.11.0
...@@ -6,14 +6,15 @@ extends = ...@@ -6,14 +6,15 @@ extends =
../openssl/buildout.cfg ../openssl/buildout.cfg
../perl/buildout.cfg ../perl/buildout.cfg
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
parts = parts =
curl curl
[curl] [curl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://curl.haxx.se/download/curl-7.54.1.tar.bz2 url = http://curl.haxx.se/download/curl-7.56.0.tar.xz
md5sum = 6b6eb722f512e7a24855ff084f54fe55 md5sum = 18ebc36c5dc9317d4a0b5db94a4e12ad
configure-options = configure-options =
--disable-static --disable-static
--disable-ldap --disable-ldap
...@@ -44,6 +45,6 @@ configure-options = ...@@ -44,6 +45,6 @@ configure-options =
--without-nghttp2 --without-nghttp2
environment = environment =
PATH=${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s PATH=${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
LDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib LDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
# ethtool - query or control network driver and hardware settings
# https://www.kernel.org/pub/software/network/ethtool
[buildout]
parts = ethtool
[ethtool]
recipe = slapos.recipe.cmmi
url = https://www.kernel.org/pub/software/network/ethtool/ethtool-4.11.tar.xz
md5sum = 16d38f4ebe23e44f96f7d8b38ed3652c
...@@ -19,7 +19,6 @@ configure-options = ...@@ -19,7 +19,6 @@ configure-options =
--without-libidn --without-libidn
--without-x --without-x
--with-drivers=FILES --with-drivers=FILES
--with-fontpath=${ghostscript-fonts:location}
# it seems that parallel build sometimes fails for ghostscript. # it seems that parallel build sometimes fails for ghostscript.
make-options = -j1 make-options = -j1
environment = environment =
...@@ -35,9 +34,3 @@ environment = ...@@ -35,9 +34,3 @@ environment =
<= ghostscript-common <= ghostscript-common
url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs920/ghostscript-9.20.tar.xz url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs920/ghostscript-9.20.tar.xz
md5sum = 8f3d383d48da22345937b66b01ab2960 md5sum = 8f3d383d48da22345937b66b01ab2960
[ghostscript-fonts]
recipe = slapos.recipe.build:download-unpacked
strip-top-level-dir = false
url = http://downloads.ghostscript.com/public/fonts/urw-base35-v1.10.zip
md5sum = 66e8bbd8228519d5dba82b9433a61bb0
# Go language - https://golang.org/ # Go language - https://golang.org/
[buildout] [buildout]
parts = golang extends =
../findutils/buildout.cfg
../git/buildout.cfg
[golang] parts = gowork
<= golang18
# ---- Go builds itself ----
[golang-common] [golang-common]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
...@@ -12,42 +14,107 @@ configure-command = : ...@@ -12,42 +14,107 @@ configure-command = :
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
make-binary = make-binary =
make-targets= cd src && ./all.bash && cp -alf .. ${:location} make-targets= cd src && ./all.bash && cp -alf .. ${:location}
# some testdata files have an issue with slapos.extension.strip.
post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
environment = environment =
GOROOT_FINAL=${:location} GOROOT_FINAL=${:location}
${:environment-extra} ${:environment-extra}
[golang14-repository]
<= git-repository
repository = https://github.com/golang/go.git
revision = go1.4.3-16-g4d5426a
[golang14] [golang14]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz path = ${golang14-repository:location}
md5sum = 76e42c8152e8560ded880a6d1d1f53cb
environment-extra = environment-extra =
[golang15] [golang18]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.5.4.src.tar.gz url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz
md5sum = a04d570515c46e4935c63605cbd3a04e md5sum = 64e9380e07bba907e26a00cf5fcbe77e
# go1.5 needs go1.4 to bootstrap # go1.8 needs go1.4 to bootstrap
environment-extra = environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} GOROOT_BOOTSTRAP=${golang14:location}
[golang19]
[golang16]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.6.3.src.tar.gz url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz
md5sum = bf3fce6ccaadd310159c9e874220e2a2 md5sum = 44105c865a1a810464df79233a05a568
# go1.6 needs go1.4 to bootstrap # go1.9 needs go1.4 to bootstrap
environment-extra = environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} GOROOT_BOOTSTRAP=${golang14:location}
[golang18]
<= golang-common
url = https://storage.googleapis.com/golang/go1.8.src.tar.gz
md5sum = 7743960c968760437b6e39093cfe6f67
# go1.8 needs go1.4 to bootstrap # ---- infrastructure to build Go workspaces / projects ----
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} # gowork is a top-level section representing workspace
#
# users should add `install` field to [gowork] to describe packages they want to
# be installed (+ automatically their dependencies are installed too). e.g.
#
# [gowork]
# install =
# lab.nexedi.com/kirr/neo/go/... \
# github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat
[gowork]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
bin = ${:directory}/bin
depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications)
golang = ${golang19:location}
# everything is done by dependent parts
recipe = plone.recipe.command
command = :
# env.sh for compiling and running go programs
[gowork]
env.sh = ${gowork-env.sh:output}
[gowork-env.sh]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/goenv.sh.in
output = ${gowork:directory}/env.sh
depends = ${gowork.mkdir:recipe}
md5sum = a9a265135931b3da53f4392870748264
[gowork.mkdir]
# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
# uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system
# libxml/libxslt or fail to bootstrap at all if those are not present.
recipe = plone.recipe.command
command = mkdir -p ${gowork:directory}
update-command = ${:command}
stop-on-error = true
# install go packages
# clients should put package list to install to gowork:install ("..." requests installing everything)
[gowork.goinstall]
recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
update-command = ${:command}
stop-on-error = true
[git-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
# a go package should:
# 1) <= go-git-package
# 2) provide go.importpath
# 3) provide repository (which is not the same as importpath in general case)
#
# the list of go packages for a go workspace state can be automatically
# generated with the help of go-pkg-snapshot tool.
[go-git-package]
<= git-repository
location = ${gowork:src}/${:go.importpath}
# env.sh for a Go workspace
# Usage: env.sh [/path/to/env.sh]
# ---- 8< ---- (buildout substitution here)
# PATH so that go & friends work out of the box
export PATH=${gowork:golang}/bin:${git:location}/bin:${buildout:bin-directory}:$PATH
X=${gowork:directory}
# ---- 8< ----
export GOPATH=$X:$GOPATH
export PATH=$X/bin:$PATH
export PS1="(`basename $X`) $PS1"
# strip trailing : from $GOPATH
GOPATH=$${GOPATH%:}
#!/bin/bash
# gowork-snapshot - find out installed go packages and produce buildout code to install them pinned
# FIXME currently works only with cwd=gowork/src
echo "# Code generated by gowork-snapshot; DO NOT EDIT."
# list installed go git repositories.
#
# this gives something like:
# github.com/cznic/strutil https://github.com/cznic/strutil 529a34b1c1
# golang.org/x/net https://go.googlesource.com/net 1087133bc4
# ...
gogit_list() {
find . -name .git | sort | \
while read repo; do
importpath=${repo#./}
importpath=${importpath%/.git}
echo -ne "${importpath}\t"
echo -ne "`git_upstream_url $repo`\t"
git -C $repo describe --long --always --abbrev=10
done
}
# git_upstream_url <repo> - show current branch upstream URL
git_upstream_url() {
repo=$1
head="`git -C $repo symbolic-ref --short HEAD`" # current branch - e.g. "t"
remote="`git -C $repo config --get branch.$head.remote`" # upstream name, e.g. "kirr"
url="`git -C $repo config --get remote.$remote.url`" # upstream URL
echo "$url"
}
# buildout_safe <name> - canonicalize name to be allowed to be used in buildout section name
# e.g. aaa/bbb -> aaa_bbb
#
# XXX can't use e.g. "go!lab.nexedi.com/kirr/neo" because buildout disallows
# "!" or "/" in section name references.
buildout_safe() {
# see _simple regex in slapos.buildout
echo -n "$1" | tr --complement -- "-a-zA-Z0-9 ._" _
}
echo
echo "# list of go git repositories to fetch"
echo "[gowork.goinstall]"
echo "depends_gitfetch ="
gogit_list | \
while read pkg _ _; do
echo " \${go_`buildout_safe $pkg`:recipe}"
done
echo
gogit_list | \
while read pkg url rev; do
echo
echo "[go_`buildout_safe $pkg`]"
echo "<= go-git-package"
echo "go.importpath = $pkg"
echo "repository = $url"
echo "revision = $rev"
done
...@@ -89,7 +89,7 @@ input = inline: ...@@ -89,7 +89,7 @@ input = inline:
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
path = ${helloweb-repository:location}/go/ path = ${helloweb-repository:location}/go/
go = ${golang18:location}/bin/go go = ${golang19:location}/bin/go
configure-command = : configure-command = :
make-binary = make-binary =
make-targets= cd ${:path} && make-targets= cd ${:path} &&
......
...@@ -54,7 +54,6 @@ configure-options = ...@@ -54,7 +54,6 @@ configure-options =
--without-wmf --without-wmf
--with-bzlib=${bzip2:location} --with-bzlib=${bzip2:location}
--with-zlib=${zlib:location} --with-zlib=${zlib:location}
--with-gs-font-dir=${ghostscript-fonts:location}
--with-frozenpaths --with-frozenpaths
patch-options = -p1 patch-options = -p1
patches = patches =
......
# ioping - simple disk I/O latency measuring tool
# https://github.com/koct9i/ioping
[buildout]
extends =
../git/buildout.cfg
parts =
ioping
[ioping-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
# NOTE we use a bit patched ioping version which shows not only avg latency but also its distribution
# repository = https://github.com/koct9i/ioping.git
repository = https://lab.nexedi.com/kirr/ioping.git
revision = v1.0-9-g34c97f7636
[ioping]
recipe = slapos.recipe.cmmi
path = ${ioping-repository:location}
configure-command = :
make-binary =
# XXX without vvv PREFIX=${:location} does not work
location= ${buildout:parts-directory}/${:_buildout_section_name_}
make-targets= make install PREFIX=${:location}
...@@ -25,7 +25,7 @@ interpreter = keras-python ...@@ -25,7 +25,7 @@ interpreter = keras-python
scripts = keras-python scripts = keras-python
[versions] [versions]
Keras = 2.0.1 Keras = 2.0.8
tensorflow = 1.0.1 tensorflow = 1.4.0rc1
h5py = 2.7.0rc2 h5py = 2.7.0rc2
Cython = 0.25.2 Cython = 0.25.2
# LMbench - Tools for Performance Analysis
# http://www.bitmover.com/lmbench/
[buildout]
extends =
../git/buildout.cfg
../golang/buildout.cfg
parts =
lmbench
[lmbench-repository]
<= git-repository
# NOTE we use a bit patched lmbench version with fixes to lat_tcp for errors not to go unnoticed and other addons
# repository = https://svn.code.sf.net/p/lmbench/code
repository = https://lab.nexedi.com/kirr/lmbench.git
revision = 9b108b6ff3
[lmbench]
recipe = slapos.recipe.cmmi
path = ${lmbench-repository:location}
configure-command = :
make-binary =
# XXX without vvv BASE=${:location} does not work
location= ${buildout:parts-directory}/${:_buildout_section_name_}
make-targets= cd ${lmbench-repository:location}/lmbench3/src
&& make install BASE=${:location}
&& bash -c ". ${gowork:env.sh} && go build -o ${:location}/bin/lat_tcp_go lat_tcp.go"
...@@ -35,5 +35,5 @@ environment = ...@@ -35,5 +35,5 @@ environment =
[postgresql92] [postgresql92]
<= postgresql-common <= postgresql-common
url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.22.tar.bz2 url = http://ftp.postgresql.org/pub/source/v9.2.23/postgresql-9.2.23.tar.bz2
md5sum = c5d3fb5229baf9e94ee2287980c55321 md5sum = c972e32b7f17dbc652d2462b7690d116
...@@ -3,14 +3,14 @@ parts = protobuf ...@@ -3,14 +3,14 @@ parts = protobuf
[protobuf] [protobuf]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-python-3.1.0.tar.gz url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
md5sum = 7a227a21379a2ea08cc5d7ba1fb1ba5b md5sum = 0820cc2e56d71aef8e99794fcbd184cd
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
[protobuf-cpp] [protobuf-cpp]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-cpp-3.1.0.tar.gz url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
md5sum = bd5e3eed635a8d32e2b99658633815ef md5sum = 6d59dad503bea5ad420fd09ddad84481
configure-command = configure-command =
./autogen.sh ./autogen.sh
./configure --prefix=${buildout:parts-directory}/${:_buildout_section_name_} ./configure --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
......
[buildout]
extends =
../../stack/slapos.cfg
../gcc/buildout.cfg
../cython/buildout.cfg
../scipy/buildout.cfg
parts =
python-cocoapi-build-install-egg
[python-cocoapi-repository]
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/cocodataset/cocoapi.git
tag = master
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/setup.py.patch ) || (rm -fr ${:location}; exit 1)
[python-cocoapi-build-interpreter]
recipe = zc.recipe.egg
initialization =
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
eggs =
setuptools
${cython:egg}
${scipy:egg}
${numpy:egg}
interpreter = python-cocoapi-build-interpreter
scripts = python-cocoapi-build-interpreter
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[python-cocoapi-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${python-cocoapi-repository:location}/PythonAPI
python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
gcc-location = ${gcc:location}
script =
os.makedirs(location)
workdir = self.options['workdir']
python_bin = self.options['python-bin']
gcc_location = self.options['gcc-location']
env = {'PATH':':'.join([gcc_location+'/bin',
os.environ['PATH']]),
'CC':gcc_location+'/bin/gcc',
'CXX':gcc_location+'/bin/g++',
}
call([python_bin, 'setup.py', 'build_ext'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[python-cocoapi-build-install-egg]
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
python-cocoapi-repository-path = ${python-cocoapi-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-python-cocoapi-build = ${python-cocoapi-build:location}
egg = pycocotools
script =
os.makedirs(location)
workdir = self.options['python-cocoapi-repository-path']+'/PythonAPI'
egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
diff --git a/PythonAPI/setup.py b/PythonAPI/setup.py
index eb3d508..d619b3c 100644
--- a/PythonAPI/setup.py
+++ b/PythonAPI/setup.py
@@ -1,4 +1,5 @@
-from distutils.core import setup
+#from distutils.core import setup
+from setuptools import setup
from Cython.Build import cythonize
from distutils.extension import Extension
import numpy as np
@@ -21,4 +22,4 @@ setup(name='pycocotools',
version='2.0',
ext_modules=
cythonize(ext_modules)
- )
\ No newline at end of file
+ )
[buildout]
extends =
../../stack/slapos.cfg
../gcc/buildout.cfg
../openblas/buildout.cfg
../cmake/buildout.cfg
../python-cffi/buildout.cfg
../python-PyYAML/buildout.cfg
../python-cocoapi/buildout.cfg
../pillow/buildout.cfg
../scipy/buildout.cfg
../matplotlib/buildout.cfg
../unzip/buildout.cfg
parts =
pytorch-egg
[pytorch-repository]
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/pytorch/pytorch
tag = master
commit = 4af66c43045a317b477918c503d105f565b4a66b
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --recursive --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:git-binary} checkout ${:commit}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/pytorch.4af66c4.patch ) || (rm -fr ${:location}; exit 1)
[pytorch-build-interpreter]
recipe = zc.recipe.egg
initialization =
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
import sys
sys.path.append('.')
sys.path.append('${pytorch-repository:location}/torch/lib/ATen')
eggs =
setuptools
${scipy:egg}
${numpy:egg}
${python-cffi:egg}
${python-PyYAML:egg}
interpreter = pytorch-build-interpreter
scripts = pytorch-build-interpreter
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pytorch-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${pytorch-repository:location}
python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
no-cuda = 0
cmake-bin = ${cmake:location}/bin
git-bin = ${git:location}/bin
binutils-location = ${binutils:location}
gcc-location = ${gcc:location}
openblas-location = ${openblas:location}
script =
os.makedirs(location)
workdir = self.options['workdir']
python_bin = self.options['python-bin']
binutils_location = self.options['binutils-location']
gcc_location = self.options['gcc-location']
openblas_location = self.options['openblas-location']
env = {'PYTHONPATH':workdir,
'PATH':':'.join([self.options['cmake-bin'],
self.options['git-bin'],
binutils_location+'/bin',
gcc_location+'/bin',
os.environ['PATH']]),
'CMAKE_INCLUDE_PATH':':'.join([gcc_location+'/include',
binutils_location+'/include',
openblas_location+'/include',
]),
'CMAKE_LIBRARY_PATH':':'.join([gcc_location+'/lib',
gcc_location+'/lib64',
binutils_location+'/lib',
openblas_location+'/lib',
]),
'CC':gcc_location+'/bin/gcc',
'CXX':gcc_location+'/bin/g++',
'PYTORCH_PYTHON':python_bin,
'SLAPOS_COMPILE_ARGS':' '.join(['-Wl,-rpath,'+gcc_location+'/lib',
'-Wl,-rpath,'+gcc_location+'/lib64',
'-Wl,-rpath,'+binutils_location+'/lib',
'-Wl,-rpath,'+openblas_location+'/lib',
])
}
if self.options.get('no-cuda') == '1':
env['NO_CUDA'] = '1'
import os.path
call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[pytorch-build-install-egg]
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
pytorch-repository-path = ${pytorch-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-pytorch-build = ${pytorch-build:location}
egg = torch
script =
os.makedirs(location)
workdir = self.options['pytorch-repository-path']
egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[pytorch-egg]
recipe = zc.recipe.egg
initialization =
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
eggs =
${scipy:egg}
${numpy:egg}
${python-PyYAML:egg}
${pytorch-build-install-egg:egg}
${pillow-python:egg}
${python-cocoapi-build-install-egg:egg}
${matplotlib:egg}
six
torchvision
interpreter = pytorch-python
scripts = pytorch-python
[versions]
torchvision = 0.1.6
diff --git a/setup.py b/setup.py
index 1d9a765..a50e9cb 100644
--- a/setup.py
+++ b/setup.py
@@ -90,7 +90,7 @@ def build_libs(libs):
assert lib in dep_libs, 'invalid lib: {}'.format(lib)
build_libs_cmd = ['bash', 'torch/lib/build_libs.sh']
my_env = os.environ.copy()
- my_env["PYTORCH_PYTHON"] = sys.executable
+ #my_env["PYTORCH_PYTHON"] = sys.executable
if WITH_SYSTEM_NCCL:
my_env["NCCL_ROOT_DIR"] = NCCL_ROOT_DIR
if WITH_CUDA:
diff --git a/torch/lib/build_libs.sh b/torch/lib/build_libs.sh
index af7020e..a91bd05 100755
--- a/torch/lib/build_libs.sh
+++ b/torch/lib/build_libs.sh
@@ -24,7 +24,7 @@ C_FLAGS=" -DTH_INDEX_BASE=0 -I$INSTALL_DIR/include \
-I$INSTALL_DIR/include/THS -I$INSTALL_DIR/include/THCS \
-I$INSTALL_DIR/include/THPP -I$INSTALL_DIR/include/THNN \
-I$INSTALL_DIR/include/THCUNN"
-LDFLAGS="-L$INSTALL_DIR/lib "
+LDFLAGS="-L$INSTALL_DIR/lib $SLAPOS_COMPILE_ARGS"
LD_POSTFIX=".so.1"
LD_POSTFIX_UNVERSIONED=".so"
if [[ $(uname) == 'Darwin' ]]; then
...@@ -4,6 +4,12 @@ ...@@ -4,6 +4,12 @@
# /opt/slapos folder, adapt this script as you please. # /opt/slapos folder, adapt this script as you please.
# #
# Be carefull to not run this script were the script is already installed. # Be carefull to not run this script were the script is already installed.
#
# Before run this script, ensure dependencies are installed, on debian, you can
# use the command bellow:
#
# apt-get install python gcc g++ make uml-utilities bridge-utils patch wget
#
# Use sudo or superuser and create slapos directory (you can pick a different directory) # Use sudo or superuser and create slapos directory (you can pick a different directory)
mkdir -p /opt/slapos/log/ mkdir -p /opt/slapos/log/
......
diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
index b386d59..f03b056 100755
--- a/tensorboard/pip_package/build_pip_package.sh
+++ b/tensorboard/pip_package/build_pip_package.sh
@@ -26,6 +26,7 @@ function main() {
DEST=$1
TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
RUNFILES="bazel-bin/tensorboard/pip_package/build_pip_package.runfiles/org_tensorflow_tensorboard"
+ WORKDIR=$(pwd)
echo $(date) : "=== Using tmpdir: ${TMPDIR}"
@@ -45,8 +46,8 @@ function main() {
rm -f MANIFEST
echo $(date) : "=== Building wheel"
echo $(pwd)
- python setup.py bdist_wheel >/dev/null
- python3 setup.py bdist_wheel >/dev/null
+ PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
+ #python3 setup.py bdist_wheel >/dev/null
mkdir -p ${DEST}
cp dist/* ${DEST}
popd
[buildout]
extends =
../zip/buildout.cfg
../bazel/buildout.cfg
parts =
slapos-cookbook-develop
slapos-cookbook
tensorboard-build-install-egg
[tensorboard-repository]
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/tensorflow/tensorboard
tag = 0.4
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1)
[tensorboard-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${tensorboard-repository:location}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel-bin = ${bazel:location}/bin
script =
os.makedirs(location)
workdir = self.options['workdir']
env = {'PATH':':'.join([self.options['gcc-bin'],
self.options['java_home_bin'],
self.options['bazel-bin'],
os.environ['PATH']]),
'COMPILER_PATH':':'.join([self.options['gcc-bin'],
os.environ.get('COMPILER_PATH') or '']),
'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
self.options['gcc-lib64'],
os.environ.get('LIBRARY_PATH') or '']),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_LIB_PATH':self.options['python27-lib'],
}
import os.path
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
call(bazel_command, cwd=workdir, env=env)
[tensorboard-build-install-egg]
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorboard-repository-path = ${tensorboard-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
egg = tensorflow-tensorboard
bazel-bin = ${bazel:location}/bin
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script =
os.makedirs(location)
workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc1-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'],
os.environ['PATH']]),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
}
call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions]
...@@ -5,6 +5,7 @@ extends = ...@@ -5,6 +5,7 @@ extends =
../zip/buildout.cfg ../zip/buildout.cfg
../bazel/buildout.cfg ../bazel/buildout.cfg
../protobuf-python/buildout.cfg ../protobuf-python/buildout.cfg
../tensorboard/buildout.cfg
parts = parts =
slapos-cookbook-develop slapos-cookbook-develop
slapos-cookbook slapos-cookbook
...@@ -26,11 +27,11 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -26,11 +27,11 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/tensorflow/tensorflow repository = https://github.com/tensorflow/tensorflow
tag = v1.0.1 tag = r1.4
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-v1.0.1.patch ) || (rm -fr ${:location}; exit 1) command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-r1.4.patch ) || (rm -fr ${:location}; exit 1)
[cuda] [cuda]
tf_need_cuda = 1 tf_need_cuda = 1
...@@ -72,6 +73,10 @@ script = ...@@ -72,6 +73,10 @@ script =
'TF_NEED_GCP':'0', 'TF_NEED_GCP':'0',
'TF_NEED_HDFS':'0', 'TF_NEED_HDFS':'0',
'TF_NEED_OPENCL':'0', 'TF_NEED_OPENCL':'0',
'TF_NEED_MKL':'0',
'TF_NEED_VERBS':'0',
'TF_CUDA_CLANG':'0',
'TF_NEED_MPI':'0',
'TF_NEED_CUDA':self.buildout['cuda']['tf_need_cuda'], 'TF_NEED_CUDA':self.buildout['cuda']['tf_need_cuda'],
##### FOR CUDA ##### ##### FOR CUDA #####
'GCC_HOST_COMPILER_PATH':os.path.join(self.options['gcc-bin'], 'gcc'), 'GCC_HOST_COMPILER_PATH':os.path.join(self.options['gcc-bin'], 'gcc'),
...@@ -98,18 +103,17 @@ recipe = slapos.recipe.build ...@@ -98,18 +103,17 @@ recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip unzip-binary = ${unzip:location}/bin/unzip
tensorflow-repository-path = ${tensorflow-repository:location} tensorflow-repository-path = ${tensorflow-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
need-tensorflow-build = ${tensorflow-build:location} need-tensorflow-build = ${tensorflow-build:location}
need-protobuf-python = ${protobuf-python:egg} need-protobuf-python = ${protobuf-python:egg}
egg = tensorflow egg = tensorflow
script = script =
os.makedirs(location) os.makedirs(location)
workdir = self.options['tensorflow-repository-path'] workdir = self.options['tensorflow-repository-path']
egg_name = 'tensorflow-1.0.1-py2.7-linux-x86_64.egg' egg_name = 'tensorflow-1.4.0rc1-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist') dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name) dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir) call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir]) call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions] [versions]
protobuf = 3.1.0.post1
wheel = 0.30.0a0
diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
index e2bee21..1df90c1 100755 index cbf06a9..226a423 100755
--- a/tensorflow/tools/pip_package/build_pip_package.sh --- a/tensorflow/tools/pip_package/build_pip_package.sh
+++ b/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -136,7 +136,7 @@ function main() { @@ -162,7 +162,7 @@ function main() {
pushd ${TMPDIR} pushd ${TMPDIR}
rm -f MANIFEST rm -f MANIFEST
echo $(date) : "=== Building wheel" echo $(date) : "=== Building wheel"
- "${PYTHON_BIN_PATH:-python}" setup.py bdist_wheel ${GPU_FLAG} >/dev/null - "${PYTHON_BIN_PATH:-python}" setup.py bdist_wheel ${PKG_NAME_FLAG} >/dev/null
+ "${PYTHON_BIN_PATH:-python}" setup.py bdist_egg ${GPU_FLAG} >/dev/null + "${PYTHON_BIN_PATH:-python}" setup.py bdist_egg ${PKG_NAME_FLAG} >/dev/null
mkdir -p ${DEST} mkdir -p ${DEST}
cp dist/* ${DEST} cp dist/* ${DEST}
popd popd
[buildout]
extends =
../libevent/buildout.cfg
../ncurses/buildout.cfg
parts +=
tmux
[tmux]
recipe = slapos.recipe.cmmi
url = https://github.com/tmux/tmux/releases/download/2.0/tmux-2.0.tar.gz
md5sum = 9fb6b443392c3978da5d599f1e814eaa
environment =
CFLAGS=-I${ncurses:location}/include -I${libevent:location}/include/
LDFLAGS=-L${ncurses:location}/lib/ -L${libevent:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/ -Wl,-rpath=${libevent:location}/lib/
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
...@@ -50,8 +50,8 @@ gitdb = 0.6.4 ...@@ -50,8 +50,8 @@ gitdb = 0.6.4
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -10,8 +10,8 @@ gitdb = 0.6.4 ...@@ -10,8 +10,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
pyasn1 = 0.2.3 pyasn1 = 0.2.3
......
[buildout]
extends =
../../stack/caddy/buildout.cfg
\ No newline at end of file
...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3 ...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
erp5.util = 0.4.49 erp5.util = 0.4.49
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12 ...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies # indirect dependancies
cp.recipe.cmd = 0.5 cp.recipe.cmd = 0.5
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
zope.exceptions = 4.0.7 zope.exceptions = 4.0.7
zope.testing = 4.1.3 zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0 zc.recipe.testrunner = 2.0.0
......
...@@ -17,6 +17,10 @@ ...@@ -17,6 +17,10 @@
"default": "erp5", "default": "erp5",
"type": "string" "type": "string"
}, },
"bt5": {
"description": "Business Template to install at automatic site creation. By default, all configurators are installed.",
"type": "string"
},
"timezone": { "timezone": {
"description": "Zope's timezone. Possible values are determined by host's libc, and typically come from a separate package (tzdata, ...)", "description": "Zope's timezone. Possible values are determined by host's libc, and typically come from a separate package (tzdata, ...)",
"default": "UTC", "default": "UTC",
......
...@@ -60,7 +60,7 @@ mode = 0644 ...@@ -60,7 +60,7 @@ mode = 0644
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.49 erp5.util = 0.4.49
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
ipython = 5.3.0 ipython = 5.3.0
apache-libcloud = 2.1.0 apache-libcloud = 2.1.0
gitdb2 = 2.0.2 gitdb2 = 2.0.2
......
...@@ -14,4 +14,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf ...@@ -14,4 +14,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -19,5 +19,5 @@ context = ...@@ -19,5 +19,5 @@ context =
[fluentd] [fluentd]
gems += gems +=
fluent-plugin-wendelin==0.1 fluent-plugin-wendelin==0.3
fluent-plugin-bin fluent-plugin-bin
...@@ -14,8 +14,19 @@ passwd = root1234 ...@@ -14,8 +14,19 @@ passwd = root1234
environment = environment =
GITLAB_ROOT_PASSWORD=${root-password:passwd} GITLAB_ROOT_PASSWORD=${root-password:passwd}
[backend-info] [service-postgresql]
# host = ${instance-parameter:ipv4-random} pgdata-directory = ${directory:srv}/pg
[gitlab-workhorse-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/glab-wh
[gitlab-workhorse]
srv = ${gitlab-workhorse-dir:srv}
socket = ${gitlab-workhorse:srv}/wh.socket
[unicorn]
socket = ${:srv}/unc.socket
[publish-instance-info] [publish-instance-info]
password = ${root-password:passwd} password = ${root-password:passwd}
......
...@@ -27,7 +27,7 @@ extends = ...@@ -27,7 +27,7 @@ extends =
parts = parts =
ruby2.1 ruby2.1
golang16 golang19
git git
postgresql92 postgresql92
redis28 redis28
...@@ -211,7 +211,7 @@ make-targets= cd ${git2go-repository:location} ...@@ -211,7 +211,7 @@ make-targets= cd ${git2go-repository:location}
&& cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin} && cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin}
environment = environment =
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang19:location}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gopath:directory} GOPATH=${gopath:directory}
[xnice-repository] [xnice-repository]
...@@ -245,7 +245,7 @@ configure-command = : ...@@ -245,7 +245,7 @@ configure-command = :
make-targets= ${:_buildout_section_name_} make-targets= ${:_buildout_section_name_}
environment = environment =
PATH=${golang16:location}/bin:%(PATH)s PATH=${golang19:location}/bin:%(PATH)s
############################### ###############################
...@@ -336,7 +336,7 @@ md5sum = 319d7dbe3ad9b260c1e292cfc0d13b11 ...@@ -336,7 +336,7 @@ md5sum = 319d7dbe3ad9b260c1e292cfc0d13b11
[instance-gitlab-test.cfg.in] [instance-gitlab-test.cfg.in]
<= download-file <= download-file
md5sum = cc8065104458af311c2ffa9ae20235a6 md5sum = a4ad76856db98e508af7e773d9ff78f9
[macrolib.cfg.in] [macrolib.cfg.in]
<= download-file <= download-file
...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3 ...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12 docutils = 0.12
plone.recipe.command = 1.1 plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001 rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
...@@ -72,7 +72,7 @@ async = 0.6.1 ...@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.4.2 slapos.recipe.template = 4.1
slapos.toolbox = 0.40.4 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
plone.recipe.command = 1.1 plone.recipe.command = 1.1
......
...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 ...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -113,5 +113,5 @@ mode = 0644 ...@@ -113,5 +113,5 @@ mode = 0644
[versions] [versions]
erp5.util = 0.4.49 erp5.util = 0.4.49
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
selenium = 2.53.1 selenium = 2.53.1
...@@ -85,7 +85,7 @@ pyzmq = 16.0.2 ...@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1 scikit-learn = 0.18.1
seaborn = 0.7.1 seaborn = 0.7.1
simplegeneric = 0.8.1 simplegeneric = 0.8.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
statsmodels = 0.8.0 statsmodels = 0.8.0
terminado = 0.6 terminado = 0.6
tornado = 4.4.2 tornado = 4.4.2
......
...@@ -5,13 +5,13 @@ extends = common.cfg ...@@ -5,13 +5,13 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1 websockify = 0.5.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
erp5.util = 0.4.49 erp5.util = 0.4.49
apache-libcloud = 1.1.0 apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
gitdb = 0.6.4 gitdb = 0.6.4
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
smmap = 0.9.0 smmap = 0.9.0
# websockify = 0.8.0 # websockify = 0.8.0
......
...@@ -17,7 +17,7 @@ parts += ...@@ -17,7 +17,7 @@ parts +=
versions = versions versions = versions
[versions] [versions]
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
[template-instance] [template-instance]
......
...@@ -116,12 +116,12 @@ mysqlclient = 1.3.12 ...@@ -116,12 +116,12 @@ mysqlclient = 1.3.12
persistent = 4.2.3 persistent = 4.2.3
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
smmap2 = 2.0.1 smmap2 = 2.0.1
transaction = 1.7.0 transaction = 1.7.0
zodbpickle = 0.6.0 zodbpickle = 0.6.0
zodbtools = 0.0.0.dev3 zodbtools = 0.0.0.dev4
# Required by: # Required by:
# slapos.toolbox==0.71 # slapos.toolbox==0.71
...@@ -153,3 +153,20 @@ paramiko = 2.1.2 ...@@ -153,3 +153,20 @@ paramiko = 2.1.2
# Required by: # Required by:
# slapos.toolbox==0.71 # slapos.toolbox==0.71
passlib = 1.7.1 passlib = 1.7.1
# Required by:
# zodbtools==0.0.0dev4
zodburi = 2.3.0
# Required by:
# zodburi==2.0
# ZEO 5 requires transaction >= 2
ZEO = 4.3.1
# Required by:
# zodburi==2.0
mock = 2.0.0
# Required by:
# ZEO==4.3.1
zdaemon = 4.2.0
...@@ -3,4 +3,14 @@ extends = software.cfg ...@@ -3,4 +3,14 @@ extends = software.cfg
[versions] [versions]
ZODB = 5.3.0 ZODB = 5.3.0
ZEO = 5.1.0
transaction = 2.1.2 transaction = 2.1.2
# Required by:
# ZEO==5.1.0
# trollius==2.1
futures = 3.1.1
# Required by:
# ZEO==5.1.0
trollius = 2.1
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_cznic_strutil:recipe}
${go_github.com_golang_glog:recipe}
${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe}
${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe}
${go_github.com_someonegg_gocontainer:recipe}
${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe}
${go_golang.org_x_tools:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe}
[go_github.com_cznic_strutil]
<= go-git-package
go.importpath = github.com/cznic/strutil
repository = https://github.com/cznic/strutil
revision = 529a34b1c1
[go_github.com_golang_glog]
<= go-git-package
go.importpath = github.com/golang/glog
repository = https://github.com/golang/glog
revision = 23def4e6c1
[go_github.com_kisielk_og-rek]
<= go-git-package
go.importpath = github.com/kisielk/og-rek
repository = https://github.com/kisielk/og-rek
revision = dd41cde712
[go_github.com_kylelemons_godebug]
<= go-git-package
go.importpath = github.com/kylelemons/godebug
repository = https://github.com/kylelemons/godebug
revision = d65d576e93
[go_github.com_pkg_errors]
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors
revision = v0.8.0-6-g2b3a18b5f0
[go_github.com_pkg_profile]
<= go-git-package
go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile
revision = v1.2.1-0-g5b67d42886
[go_github.com_someonegg_gocontainer]
<= go-git-package
go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer
revision = fc2c7e84b5
[go_golang.org_x_net]
<= go-git-package
go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net
revision = 1087133bc4
[go_golang.org_x_perf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf
revision = 4469e6ce8c
[go_golang.org_x_sync]
<= go-git-package
go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync
revision = 8e0aa688b6
[go_golang.org_x_tools]
<= go-git-package
go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools
revision = 9bd2f44268
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 2578d58311
[go_lab.nexedi.com_kirr_neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.8-1326-g4d0cd89484
# TODO instance which runs this test periodically automatically and ingests results to ERP5
# env.sh for putting everything needed to run neotest on path
. ${gowork:env.sh}
PATH="${ethtool:location}/sbin:$PATH"
PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH"
PATH="${mariadb:location}/bin:$PATH"
PATH="${mariadb:location}/scripts:$PATH"
PATH="${python2.7:location}/bin:$PATH"
# add all eggs and develop-eggs to py path
#
# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
# TODO(kirr) try doing so
PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH # for .egg-link to be found by setuptools
for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
done
for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
export PYTHONPATH=$egg:$PYTHONPATH
done
export PATH
export PYTHONPATH
export PS1="(neotest-`basename ${buildout:directory}`) $PS1"
#!/bin/bash -e
# neotest wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/neotest "$@"
# neotest SR with main components taken from latest git versions
[buildout]
extends = software.cfg
[go_lab.nexedi.com_kirr_go123]
branch = master
revision=
[go_lab.nexedi.com_kirr_neo]
branch = t
revision=
[lmbench-repository]
branch = x/kirr
revision=
[ioping-repository]
branch = x/hist
revision=
# NEO test software-release
#
# This software-release prepares environment so that neotest NEO
# testing/benchmarking can be run in a SlapOS WebRunner.
[buildout]
extends =
../../stack/slapos.cfg
../neoppod/software-common.cfg
../../component/golang/buildout.cfg
gowork.cfg
../../component/wendelin.core/buildout.cfg
../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg
../../component/lmbench/buildout.cfg
parts =
gowork
lmbench
ioping
ethtool
neoppod-develop
neoppod
wendelin.core-dev
eggs
neotest-env.sh
neotest
# for instance
slapos-deps-eggs
slapos-cookbook
# instance.cfg
# go packages to install (+ automatically their dependencies)
[gowork]
install =
lab.nexedi.com/kirr/neo/go/... \
github.com/pkg/profile \
golang.org/x/perf/cmd/benchstat
# dev-install neo from go checkout
[neoppod-develop]
setup = ${go_lab.nexedi.com_kirr_neo:location}
# neotest-env.sh + neotest wrapper
[buildout-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:_buildout_section_name_}.in
output = ${buildout:directory}/${:_buildout_section_name_}
[neotest-env.sh]
<= buildout-template
md5sum = 6e2203c7a86e8a3e8e56b8086115f5d4
[neotest]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = fb3b4109128c1db1739ef5bb6abd1d94
# instance (TODO)
[instance.cfg]
<= buildout-template
md5sum = 410e1b2d72829824b28cc0299adb472e
# eggs:
[eggs]
recipe = zc.recipe.egg
eggs =
# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
ZODB3
# also for wc
numpy
# to install not only wendelin.core modules but also scripts
wendelin.core
# for ZEO scripts (runzeo)
ZEO
# wendelin.core: latest not yet released
[wendelin.core-repository]
revision= v0.11-4-g38fbc83ceb
# ping eggs versions
[versions]
pyasn1 = 0.3.7
ZODB3 = 3.11.0
numpy = 1.13.3
zope.testing = 4.6.2
# Required by:
# ZEO==4.3.1
# ZODB==4.4.5
# zodburi==2.3.0
ZConfig = 3.2.0
# Required by:
# ZEO==4.3.1
# ZODB==4.4.5
zc.lockfile = 1.2.1
# Required by:
# slapos.toolbox==0.73
erp5.util = 0.4.49
...@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in ...@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
...@@ -65,4 +65,4 @@ mode = 0644 ...@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -110,8 +110,8 @@ gitdb = 0.6.4 ...@@ -110,8 +110,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -47,4 +47,4 @@ mode = 0644 ...@@ -47,4 +47,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -11,4 +11,4 @@ extends = common.cfg ...@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6 Pygments = 1.6
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10 collective.recipe.template = 1.10
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -108,13 +108,17 @@ SSLProxyEngine On ...@@ -108,13 +108,17 @@ SSLProxyEngine On
# As backend is trusting REMOTE_USER header unset it always # As backend is trusting REMOTE_USER header unset it always
RequestHeader unset REMOTE_USER RequestHeader unset REMOTE_USER
RequestHeader unset SSL_CLIENT_SERIAL
{% if parameter_dict['ca-cert'] -%} {% if parameter_dict['ca-cert'] -%}
SSLVerifyClient require SSLVerifyClient optional
RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
SSLCACertificateFile {{ parameter_dict['ca-cert'] }} SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
{% if parameter_dict['crl'] -%}
SSLCARevocationCheck chain SSLCARevocationCheck chain
SSLCARevocationFile {{ parameter_dict['crl'] }} SSLCARevocationFile {{ parameter_dict['crl'] }}
{%- endif %} {%- endif %}
{%- endif %}
ErrorLog "{{ parameter_dict['error-log'] }}" ErrorLog "{{ parameter_dict['error-log'] }}"
# Default apache log format with request time in microsecond at the end # Default apache log format with request time in microsecond at the end
...@@ -133,8 +137,10 @@ RewriteEngine On ...@@ -133,8 +137,10 @@ RewriteEngine On
Listen {{ ip }}:{{ port }} Listen {{ ip }}:{{ port }}
{% endfor -%} {% endfor -%}
<VirtualHost *:{{ port }}> <VirtualHost *:{{ port }}>
{% if enable_authentication -%} SSLEngine on
{% if enable_authentication and parameter_dict['shared-ca-cert'] and parameter_dict['shared-crl'] -%}
SSLVerifyClient require SSLVerifyClient require
# Custom block we use for now different parameters.
RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
SSLCACertificateFile {{ parameter_dict['shared-ca-cert'] }} SSLCACertificateFile {{ parameter_dict['shared-ca-cert'] }}
SSLCARevocationPath {{ parameter_dict['shared-crl'] }} SSLCARevocationPath {{ parameter_dict['shared-crl'] }}
...@@ -145,8 +151,6 @@ Listen {{ ip }}:{{ port }} ...@@ -145,8 +151,6 @@ Listen {{ ip }}:{{ port }}
ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-error.log" ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-error.log"
CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-access.log" combined CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-access.log" combined
{% endif -%} {% endif -%}
SSLEngine on
RewriteRule ^/(.*) {{ backend }}/$1 [L,P] RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
</VirtualHost> </VirtualHost>
{% endfor -%} {% endfor -%}
...@@ -15,16 +15,12 @@ ...@@ -15,16 +15,12 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = aa10af616493b97d2b630c46d869a9d7 md5sum = 95f7341ff490cb70df61fad35d08bc30
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = c5ce18fa4d4be9b9a2d789f3bbd37840 md5sum = c76c4b36c351b7383dad7bed04e3bc90
[template-apache-backend-conf] [template-apache-backend-conf]
filename = apache-backend.conf.in filename = apache-backend.conf.in
md5sum = ea77222f440bb72fee4939fe1b72976e md5sum = 991bfcc4b6abae2ec59b69d71013e8ad
[template-create-erp5-site-real]
filename = instance-create-erp5-site-real.cfg.in
md5sum = 86a2b244341218cd0c4b6d398c61ee20
{% set part_list = [] -%} {% set part_list = [] -%}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %} {% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %}
{% set caucase_url = slapparameter_dict.get('caucase-url', '') -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set shared_ca_path = slapparameter_dict['shared-certificate-authority-path'] -%} {% set shared_ca_path = slapparameter_dict['shared-certificate-authority-path'] -%}
...@@ -37,6 +38,56 @@ context = key content {{content_section_name}}:content ...@@ -37,6 +38,56 @@ context = key content {{content_section_name}}:content
mode = {{ mode }} mode = {{ mode }}
{%- endmacro %} {%- endmacro %}
[certificate-request-base]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/request-instance-certificate
parameters-extra = true
command-line = {{ parameter_dict['bin-directory'] }}/caucase-cliweb
--crt-file ${apache-conf-ssl:cert}
--key-file ${apache-conf-ssl:key}
--crl-file ${apache-conf-ssl:crl}
--ca-url {{ caucase_url }}
--ca-crt-file ${apache-conf-ssl:ca-cert}
{% macro request_cert(name, common_name) -%}
{% set get_crl_periodicity = slapparameter_dict.get('crl-update-periodicity', 'daily') -%}
[{{ section(name ~ '-certificate-request') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/request-{{ name }}-certificate
command-line =
${certificate-request-base:wrapper-path}
--cn {{ common_name }}
--request
[{{ section(name ~ '-renew-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = {{ name }}-certificate-auto-renew
time = weekly
# 2592000 = 30*24*60*60 equivalent to one month in seconds
command = ${certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${apache-graceful:output}"
[{{ section(name ~ '-download-crl') }}]
# download the crl for the first time
recipe = plone.recipe.command
command =
if [ ! -s "${apache-conf-ssl:crl}" ]; then
${certificate-request-base:wrapper-path} --update-crl
fi
update-command = ${:command}
stop-on-error = true
[{{ section(name ~ '-update-crl-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = {{ name }}-update-crl
time = {{ get_crl_periodicity }}
# XXX - Update crl call apache graceful restart, it's not recommended to check crl too often, Apache
# has an issue with reload and can be frozen and stop responding. Default periodicity time = daily
command = ${certificate-request-base:wrapper-path} --update-crl --on-crl-update="${apache-graceful:output}"
{%- endmacro %}
{% if use_ipv6 -%} {% if use_ipv6 -%}
[zope-tunnel-base] [zope-tunnel-base]
recipe = slapos.cookbook:ipv4toipv6 recipe = slapos.cookbook:ipv4toipv6
...@@ -82,6 +133,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} ...@@ -82,6 +133,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
-#} -#}
{% do zope_family_address_list[0][0] -%} {% do zope_family_address_list[0][0] -%}
{% set haproxy_port = next_port() -%} {% set haproxy_port = next_port() -%}
{% set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
{% do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%} {% do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
{% if has_webdav -%} {% if has_webdav -%}
{% set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#} {% set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
...@@ -90,8 +142,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} ...@@ -90,8 +142,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set internal_scheme = 'http' -%} {% set internal_scheme = 'http' -%}
{% set external_scheme = 'https' -%} {% set external_scheme = 'https' -%}
{% endif -%} {% endif -%}
{% set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%} {% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name] -%}
{% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%} {% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
{% endfor -%} {% endfor -%}
...@@ -125,9 +176,9 @@ crl = ${directory:apache-conf}/crl.pem ...@@ -125,9 +176,9 @@ crl = ${directory:apache-conf}/crl.pem
backend-list = {{ dumps(apache_dict.values()) }} backend-list = {{ dumps(apache_dict.values()) }}
ip-list = {{ dumps(apache_ip_list) }} ip-list = {{ dumps(apache_ip_list) }}
pid-file = ${directory:run}/apache.pid pid-file = ${directory:run}/apache.pid
log-dir = ${directory:log}
error-log = ${directory:log}/apache-error.log error-log = ${directory:log}/apache-error.log
access-log = ${directory:log}/apache-access.log access-log = ${directory:log}/apache-access.log
log-dir = ${directory:log}
# Apache 2.4's default value (60 seconds) can be a bit too short # Apache 2.4's default value (60 seconds) can be a bit too short
timeout = 300 timeout = 300
# Basic SSL server configuration # Basic SSL server configuration
...@@ -144,8 +195,6 @@ shared-ca-cert = {{ shared_ca_path }}/cacert.pem ...@@ -144,8 +195,6 @@ shared-ca-cert = {{ shared_ca_path }}/cacert.pem
shared-crl = {{ shared_ca_path }}/crl shared-crl = {{ shared_ca_path }}/crl
{%- endif %} {%- endif %}
[apache-conf] [apache-conf]
< = jinja2-template-base < = jinja2-template-base
template = {{ parameter_dict['template-apache-conf'] }} template = {{ parameter_dict['template-apache-conf'] }}
...@@ -156,6 +205,18 @@ context = section parameter_dict apache-conf-parameter-dict ...@@ -156,6 +205,18 @@ context = section parameter_dict apache-conf-parameter-dict
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/apache wrapper-path = ${directory:services}/apache
command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
wait-for-files =
${apache-conf-ssl:cert}
${apache-conf-ssl:key}
[apache-graceful]
recipe = collective.recipe.template
input = inline:
#!/bin/sh
kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
output = ${directory:bin}/apache-httpd-graceful
mode = 700
[{{ section('apache-promise') }}] [{{ section('apache-promise') }}]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
...@@ -164,7 +225,7 @@ path = ${directory:promise}/apache ...@@ -164,7 +225,7 @@ path = ${directory:promise}/apache
hostname = {{ ipv4 }} hostname = {{ ipv4 }}
port = {{ apache_dict.values()[0][0] }} port = {{ apache_dict.values()[0][0] }}
[publish] [{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%} {% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %} {{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
...@@ -178,6 +239,11 @@ key = ${apache-ssl-key:rendered} ...@@ -178,6 +239,11 @@ key = ${apache-ssl-key:rendered}
cert = ${apache-ssl-cert:rendered} cert = ${apache-ssl-cert:rendered}
{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }} {{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }} {{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
{% elif caucase_url -%}
key = ${apache-conf-ssl:key}
cert = ${apache-conf-ssl:cert}
{{ request_cert('erp5', 'instance.apache@erp5') }}
{% else %} {% else %}
recipe = plone.recipe.command recipe = plone.recipe.command
command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}" command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
...@@ -191,12 +257,15 @@ cert = ${apache-ssl-ca:rendered} ...@@ -191,12 +257,15 @@ cert = ${apache-ssl-ca:rendered}
crl = ${apache-ssl-crl:rendered} crl = ${apache-ssl-crl:rendered}
{{ simplefile('apache-ssl-ca', '${apache-conf-ssl:ca-cert}', ssl_parameter_dict['ca-cert']) }} {{ simplefile('apache-ssl-ca', '${apache-conf-ssl:ca-cert}', ssl_parameter_dict['ca-cert']) }}
{{ simplefile('apache-ssl-crl', '${apache-conf-ssl:crl}', ssl_parameter_dict['crl']) }} {{ simplefile('apache-ssl-crl', '${apache-conf-ssl:crl}', ssl_parameter_dict['crl']) }}
{% elif caucase_url -%}
cert = ${apache-conf-ssl:ca-cert}
crl = ${apache-conf-ssl:crl}
{% else %} {% else %}
cert = cert =
crl = crl =
{%- endif %} {%- endif %}
{% set apache_service_log_list = {} -%} {% set apache_service_log_list = {} -%}
{% for family_name, (_, _, _, authentication) in apache_dict.items() -%} {% for family_name, (_, _, _, authentication) in apache_dict.items() -%}
{% if authentication -%} {% if authentication -%}
...@@ -211,7 +280,7 @@ post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bi ...@@ -211,7 +280,7 @@ post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bi
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
[logrotate-apache] [{{ section('logrotate-apache') }}]
< = logrotate-entry-base < = logrotate-entry-base
name = apache name = apache
log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log} log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
...@@ -222,7 +291,7 @@ recipe = slapos.cookbook:mkdirectory ...@@ -222,7 +291,7 @@ recipe = slapos.cookbook:mkdirectory
apache-conf = ${:etc}/apache apache-conf = ${:etc}/apache
bin = ${buildout:directory}/bin bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc etc = ${buildout:directory}/etc
promise = ${directory:etc}/promise promise = ${:etc}/promise
services = ${:etc}/run services = ${:etc}/run
var = ${buildout:directory}/var var = ${buildout:directory}/var
run = ${:var}/run run = ${:var}/run
...@@ -235,39 +304,33 @@ newcerts = ${:ca-dir}/newcerts ...@@ -235,39 +304,33 @@ newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl crl = ${:ca-dir}/crl
apachedex = ${monitor-directory:private}/apachedex apachedex = ${monitor-directory:private}/apachedex
[monitor-generate-apachedex-report] [{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command} wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}" command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
command = apachedex_every_3_hour command = apachedex_every_23_hour
[apachedex-parameters] [apachedex-parameters]
default_parameter =
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12 # which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log} apache-log-list = ${apache-conf-parameter-dict:access-log}
default = ${monitor-directory:etc}/apdex_default configuration = {{ slapparameter_dict['apachedex-configuration'] }}
base-list = ${monitor-directory:etc}/apdex_base_list promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list [{{ section('monitor-promise-apachedex-result') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promise}/check-apachedex-result
command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }} monitor-httpd-port = {{ next_port() }}
monitor-title = Balancer monitor monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }} password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
file apachedex-default ${apachedex-parameters:default}
file apachedex-base-list ${apachedex-parameters:base-list}
file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
[buildout] [buildout]
extends = extends =
{{ logrotate_cfg }} {{ logrotate_cfg }}
{{ parameter_dict['template-monitor'] }} {{ parameter_dict['template-monitor'] }}
parts += parts +=
publish
logrotate-apache
monitor-generate-apachedex-report
{{ part_list | join('\n ') }} {{ part_list | join('\n ') }}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
services = ${:etc}/run
promise = ${:etc}/promise
[erp5-bootstrap]
recipe = slapos.cookbook:erp5.bootstrap
runner-path = ${directory:services}/erp5-bootstrap
{# Note: a random domain name will be picked if several point to the same IP -#}
{% set reverse_hosts = {} -%}
{% for x, y in publish['hosts-dict'].iteritems() -%}
{% do reverse_hosts.__setitem__(y, x) -%}
{% endfor -%}
{# XXX: Expect the first database to be the one to use for catalog. -#}
{% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%}
mysql-url = {{ dumps(urlparse.urlunparse(mysql_parsed[:1] + (mysql_parsed.username + ":" + mysql_parsed.password + "@" + reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) + ':' ~ mysql_parsed.port, ) + mysql_parsed[2:])) }}
{# Pick the first http[s] family found, they should be all equivalent anyway. -#}
{# Don't pick the https[s] configurated with ssl-authenticat=true. By convention, this family name contain 'service'. -#}
{% set family_list = [] -%}
{% for key, value in publish.items() -%}
{% if key.startswith('family-') and value.startswith('http') and not 'service' in key -%}
{% do family_list.append(value.split('://', 1)) -%}
{% endif -%}
{% endfor -%}
zope-url = {{ dumps(family_list[0][0] + '://' + publish['inituser-login'] + ':' + publish_early['inituser-password'] + '@' + family_list[0][1] + '/' + publish['site-id']) }}
[promise-erp5-site]
recipe = slapos.cookbook:check_url_available
url = ${erp5-bootstrap:zope-url}
path = ${directory:promise}/erp5-site
dash_path = {{ parameter_dict['dash-location'] }}/bin/dash
curl_path = {{ parameter_dict['curl-location'] }}/bin/curl
[buildout]
parts = promise-erp5-site
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
{% import "root_common" as root_common with context %} {% import "root_common" as root_common with context -%}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%} {% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%} {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%} {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
...@@ -9,7 +9,9 @@ ...@@ -9,7 +9,9 @@
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%} {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%} {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
[request-common] [request-common]
<= request-common-base <= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
...@@ -52,6 +54,14 @@ config-name = {{ name }} ...@@ -52,6 +54,14 @@ config-name = {{ name }}
connection-url = smtp://127.0.0.2:0/ connection-url = smtp://127.0.0.2:0/
{%- endif %} {%- endif %}
{% if caucase_url -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
[request-caucase]
connection-http-url = {{ caucase_url }}
{%- else %}
{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891, 'auto-sign-csr-amount': 2}, {'http-url': True, 'https-url': False}) }}
{% endif -%}
{# ZODB -#} {# ZODB -#}
{% set zodb_dict = {} -%} {% set zodb_dict = {} -%}
{% set storage_dict = {} -%} {% set storage_dict = {} -%}
...@@ -95,16 +105,16 @@ recipe = slapos.cookbook:publish-early ...@@ -95,16 +105,16 @@ recipe = slapos.cookbook:publish-early
{%- if neo %} {%- if neo %}
neo-cluster gen-neo-cluster:name neo-cluster gen-neo-cluster:name
{%- if neo[0] %} {%- if neo[0] %}
neo-cluster = {{ neo[0] }} neo-cluster = {{ dumps(neo[0]) }}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %} {%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %} {%- if inituser_password %}
inituser-password = {{ inituser_password }} inituser-password = {{ dumps(inituser_password) }}
{%- endif %} {%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%} {%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %} {%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ deadlock_debugger_password }} deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
{%- endif %} {%- endif %}
[gen-password] [gen-password]
...@@ -133,12 +143,13 @@ return = ...@@ -133,12 +143,13 @@ return =
zope-address-list zope-address-list
hosts-dict hosts-dict
monitor-base-url monitor-base-url
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator' -%} {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc slapos_configurator' -%}
{% if has_jupyter -%} {% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% endif -%} {% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-caucase-url = ${request-caucase:connection-http-url}
config-cloudooo-url = ${request-cloudooo:connection-url} config-cloudooo-url = ${request-cloudooo:connection-url}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
...@@ -171,23 +182,22 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} ...@@ -171,23 +182,22 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
software-type = zope software-type = zope
{% set zope_family_dict = {} -%} {% set zope_family_dict = {} -%}
{% set jupyter_zope_family_default = [] -%}
{% set zope_backend_path_dict = {} -%} {% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%} {% set ssl_authentication_dict = {} -%}
{% set jupyter_zope_family_default = [] -%}
{% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%} {% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
{% set partition_name = 'zope-' ~ custom_name -%} {% set partition_name = 'zope-' ~ custom_name -%}
{% set section_name = 'request-' ~ partition_name -%} {% set section_name = 'request-' ~ partition_name -%}
{% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
{% set zope_family = zope_parameter_dict.get('family', 'default') -%} {% set zope_family = zope_parameter_dict.get('family', 'default') -%}
{% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
{# # default jupyter zope family is first zope family. -#} {# # default jupyter zope family is first zope family. -#}
{# # use list.append() to update it, because in jinja2 set changes only local scope. -#} {# # use list.append() to update it, because in jinja2 set changes only local scope. -#}
{% if not jupyter_zope_family_default -%} {% if not jupyter_zope_family_default -%}
{% do jupyter_zope_family_default.append(zope_family) -%} {% do jupyter_zope_family_default.append(zope_family) -%}
{% endif -%} {% endif -%}
{% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%} {% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
{% do zope_backend_path_dict.setdefault(zope_parameter_dict.get('family', 'default'), backend_path) -%} {% do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
{% do ssl_authentication_dict.setdefault(zope_parameter_dict.get('family', 'default'), zope_parameter_dict.get('ssl-authentication', False)) -%} {% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
[{{ section_name }}] [{{ section_name }}]
<= request-zope-base <= request-zope-base
name = {{ partition_name }} name = {{ partition_name }}
...@@ -201,7 +211,6 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque ...@@ -201,7 +211,6 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }} config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
config-name = {{ partition_name }}
{% endfor -%} {% endfor -%}
{# if not explicitly configured, connect jupyter to first zope family, which -#} {# if not explicitly configured, connect jupyter to first zope family, which -#}
...@@ -211,7 +220,6 @@ config-name = {{ partition_name }} ...@@ -211,7 +220,6 @@ config-name = {{ partition_name }}
{% endif -%} {% endif -%}
{# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#} {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
{% set zope_address_list_id_dict = {} -%}
{% set zope_family_parameter_dict = {} -%} {% set zope_family_parameter_dict = {} -%}
{% for family_name, zope_section_id_list in zope_family_dict.items() -%} {% for family_name, zope_section_id_list in zope_family_dict.items() -%}
{% for zope_section_id in zope_section_id_list -%} {% for zope_section_id in zope_section_id_list -%}
...@@ -300,6 +308,7 @@ return = ...@@ -300,6 +308,7 @@ return =
{{ family }}-v6 {{ family }}-v6
{% endfor -%} {% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }} config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%} {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
...@@ -307,16 +316,17 @@ config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list} ...@@ -307,16 +316,17 @@ config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}
{% endfor -%} {% endfor -%}
# XXX: should those really be same for all families ? # XXX: should those really be same for all families ?
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }} config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
config-backend-path = {{ dumps(balancer_dict.get('apache-backend-path', '/') % {'site-id': site_id}) }}
config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }} config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-shared-certificate-authority-path = ${directory:ca-dir}
config-monitor-passwd = ${monitor-htpasswd:passwd} config-monitor-passwd = ${monitor-htpasswd:passwd}
config-name = ${:name} config-name = ${:name}
config-caucase-url = ${request-caucase:connection-http-url}
config-crl-update-periodicity = {{ crl_update_period }}
config-shared-certificate-authority-path = ${directory:ca-dir}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration', config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }} '--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
...@@ -365,10 +375,10 @@ monitor-httpd-port = 8386 ...@@ -365,10 +375,10 @@ monitor-httpd-port = 8386
[buildout] [buildout]
extends = {{ template_monitor }} extends = {{ template_monitor }}
parts += parts +=
apache-certificate-authority apache-certificate-authority
fix-ca-folder fix-ca-folder
publish
monitor-base monitor-base
[monitor-conf-parameters] [monitor-conf-parameters]
......
...@@ -58,11 +58,6 @@ filename = instance-balancer.cfg.in ...@@ -58,11 +58,6 @@ filename = instance-balancer.cfg.in
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
filename = apache-backend.conf.in filename = apache-backend.conf.in
[template-create-erp5-site-real]
< = download-base-part
filename = instance-create-erp5-site-real.cfg.in
# Test Suite: SLAPOS-MASTER-MASTER ran at 2017/07/04 08:41:0.615347 UTC # Test Suite: SLAPOS-MASTER-MASTER ran at 2017/07/04 08:41:0.615347 UTC
# 6 failures, 1 errors, 1316 total, status: FAIL # 6 failures, 1 errors, 1316 total, status: FAIL
......
...@@ -97,4 +97,4 @@ mode = 640 ...@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3 Pygments = 2.1.3
collective.recipe.template = 1.10 collective.recipe.template = 1.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab ...@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
[template-runner] [template-runner]
filename = instance-runner.cfg filename = instance-runner.cfg
md5sum = 8f49df215a1596efcec94e2cca009711 md5sum = baf70a9f2f433ea0f2f5139700e53b1f
[template-runner-import-script] [template-runner-import-script]
filename = template/runner-import.sh.jinja2 filename = template/runner-import.sh.jinja2
...@@ -31,7 +31,7 @@ md5sum = 5cfa49bcf20612844e1c50a85740d0b3 ...@@ -31,7 +31,7 @@ md5sum = 5cfa49bcf20612844e1c50a85740d0b3
[template-runner-export-script] [template-runner-export-script]
filename = template/runner-export.sh.jinja2 filename = template/runner-export.sh.jinja2
md5sum = 78b2454951e6dbe124612eafebcf5690 md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
[instance-runner-export] [instance-runner-export]
filename = instance-runner-export.cfg.in filename = instance-runner-export.cfg.in
...@@ -63,11 +63,7 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62 ...@@ -63,11 +63,7 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62
[template-bash-profile] [template-bash-profile]
filename = template/bash_profile.in filename = template/bash_profile.in
md5sum = 1c88cbca0c1e705eeb4b544ef4616097 md5sum = 712ca70488051f97e7a7b11a02a06bb1
[template-bashrc]
filename = template/bashrc.in
md5sum = d0b9060ec96d4d5b188e36c84240ea7d
[template-supervisord] [template-supervisord]
filename = template/supervisord.conf.in filename = template/supervisord.conf.in
......
...@@ -13,6 +13,7 @@ extends = ...@@ -13,6 +13,7 @@ extends =
../../component/nano/buildout.cfg ../../component/nano/buildout.cfg
../../component/nginx/buildout.cfg ../../component/nginx/buildout.cfg
../../component/openssh/buildout.cfg ../../component/openssh/buildout.cfg
../../component/mosh/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../component/python-2.7/buildout.cfg ../../component/python-2.7/buildout.cfg
...@@ -137,15 +138,10 @@ filename = resilient_software_release_information.py.in ...@@ -137,15 +138,10 @@ filename = resilient_software_release_information.py.in
< = template-download-base < = template-download-base
filename = slapos-slapuser-script.in filename = slapos-slapuser-script.in
[template-bashrc]
< = template-download-base
filename = bashrc.in
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
${pycurl:egg} ${pycurl:egg}
collective.recipe.environment
collective.recipe.template collective.recipe.template
cns.recipe.symlink cns.recipe.symlink
erp5.util erp5.util
......
...@@ -167,7 +167,7 @@ minishell_cwd_file = $${directory:etc}/.minishell-cwd ...@@ -167,7 +167,7 @@ minishell_cwd_file = $${directory:etc}/.minishell-cwd
minishell_history_file = $${directory:etc}/.minishell_history minishell_history_file = $${directory:etc}/.minishell_history
software_info_json = $${runnerdirectory:home}/software_info.json software_info_json = $${runnerdirectory:home}/software_info.json
instance_info_json = $${runnerdirectory:home}/instance_info.json instance_info_json = $${runnerdirectory:home}/instance_info.json
path = $${shell:path} path = $${shell-environment:path}
instance_name = $${slap-parameter:instance-name} instance_name = $${slap-parameter:instance-name}
default_repository = $${slap-parameter:slapos-repository} default_repository = $${slap-parameter:slapos-repository}
...@@ -201,7 +201,8 @@ recipe = slapos.cookbook:wrapper ...@@ -201,7 +201,8 @@ recipe = slapos.cookbook:wrapper
arguments = --server_url=$${slap-connection:server-url} --key_file=$${slap-connection:key-file} --cert_file=$${slap-connection:cert-file} --computer_id=$${slap-connection:computer-id} --partition_id=$${slap-connection:partition-id} arguments = --server_url=$${slap-connection:server-url} --key_file=$${slap-connection:key-file} --cert_file=$${slap-connection:cert-file} --computer_id=$${slap-connection:computer-id} --partition_id=$${slap-connection:partition-id}
command-line = ${buildout:directory}/bin/slaprunnertest $${:arguments} command-line = ${buildout:directory}/bin/slaprunnertest $${:arguments}
wrapper-path = $${directory:bin}/runTestSuite wrapper-path = $${directory:bin}/runTestSuite
environment = RUNNER_CONFIG=$${slapos-cfg:rendered} environment = PATH=$${shell-environment:path}
RUNNER_CONFIG=$${slapos-cfg:rendered}
parameters-extra = true parameters-extra = true
# Deploy openssh-server # Deploy openssh-server
...@@ -226,7 +227,7 @@ template = inline: ...@@ -226,7 +227,7 @@ template = inline:
PasswordAuthentication no PasswordAuthentication no
PubkeyAuthentication yes PubkeyAuthentication yes
AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys
ForceCommand if [ -z "$SSH_ORIGINAL_COMMAND" ]; then ${bash:location}/bin/bash -l; else eval "$SSH_ORIGINAL_COMMAND"; fi ForceCommand if [ -z "$SSH_ORIGINAL_COMMAND" ]; then $${shell-environment:shell} -l; else SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
Subsystem sftp ${openssh:location}/libexec/sftp-server Subsystem sftp ${openssh:location}/libexec/sftp-server
[runner-sshd-raw-server] [runner-sshd-raw-server]
...@@ -424,7 +425,7 @@ recipe = slapos.cookbook:wrapper ...@@ -424,7 +425,7 @@ recipe = slapos.cookbook:wrapper
command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --timeout 200 --threads 3 --log-level error --preload command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --timeout 200 --threads 3 --log-level error --preload
error-log-file = gunicorn-error.log error-log-file = gunicorn-error.log
wrapper-path = $${gunicorn:bin_launcher} wrapper-path = $${gunicorn:bin_launcher}
environment = PATH=$${environ:PATH}:${git:location}/bin/ environment = PATH=$${shell-environment:path}
RUNNER_CONFIG=$${slaprunner:slapos.cfg} RUNNER_CONFIG=$${slaprunner:slapos.cfg}
LANG=en_GB.UTF-8 LANG=en_GB.UTF-8
...@@ -640,18 +641,11 @@ template = inline: ...@@ -640,18 +641,11 @@ template = inline:
--disable-ssl \ --disable-ssl \
--disable-ssl-menu \ --disable-ssl-menu \
--unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \ --unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
--service "/:$(id -u):$(id -g):HOME:$${shell:wrapper} -l" --service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
[shell] [shell-environment]
recipe = slapos.cookbook:shell
wrapper = $${directory:bin}/bash
shell = ${bash:location}/bin/bash shell = ${bash:location}/bin/bash
home = $${buildout:directory} path = ${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin:${mosh:location}/bin:${bash:location}/bin:$${buildout:directory}/bin/:/usr/bin:/bin/
path = $${environ:PATH}:/usr/bin:/bin/:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${tig:location}/bin
ps1 = "\\w> "
[environ]
recipe = collective.recipe.environment
[prepare-software] [prepare-software]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -712,16 +706,10 @@ recipe = slapos.recipe.template:jinja2 ...@@ -712,16 +706,10 @@ recipe = slapos.recipe.template:jinja2
template = ${template-bash-profile:location}/${template-bash-profile:filename} template = ${template-bash-profile:location}/${template-bash-profile:filename}
rendered = $${buildout:directory}/.bash_profile rendered = $${buildout:directory}/.bash_profile
context = context =
raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin raw path $${shell-environment:path}
raw shell $${shell-environment:shell}
key workdir runnerdirectory:home key workdir runnerdirectory:home
[bashrc]
recipe = slapos.recipe.template:jinja2
template = ${template-bashrc:location}/${template-bashrc:filename}
rendered = $${buildout:directory}/.bashrc
context =
raw bin_folder ${buildout:bin-directory}
#--------------------------- #---------------------------
#-- #--
#-- supervisord managing slaprunner automation features #-- supervisord managing slaprunner automation features
...@@ -734,7 +722,7 @@ exitcodes = 0 ...@@ -734,7 +722,7 @@ exitcodes = 0
logfile = $${directory:log}/supervisord.log logfile = $${directory:log}/supervisord.log
no_logfile = NONE no_logfile = NONE
numprocs = 1 numprocs = 1
path = $${shell:path} path = $${shell-environment:path}
pidfile = $${directory:run}/supervisord.pid pidfile = $${directory:run}/supervisord.pid
ip = $${slaprunner:ipv4} ip = $${slaprunner:ipv4}
server = $${:ip}:$${:port} server = $${:ip}:$${:port}
......
...@@ -10,14 +10,13 @@ extends = common.cfg ...@@ -10,14 +10,13 @@ extends = common.cfg
Flask-Auth = 0.85 Flask-Auth = 0.85
apache-libcloud = 1.2.1 apache-libcloud = 1.2.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
futures = 3.0.5 futures = 3.0.5
gitdb = 0.6.4 gitdb = 0.6.4
gunicorn = 19.4.5 gunicorn = 19.7.1
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
cd {{ workdir }} cd {{ workdir }}
export PATH={{- path }} export PATH={{- path }}
export SHELL={{- shell }}
export PS1="$ " export PS1="$ "
if [ -f "$HOME/.bashrc" ] ; then if [ -f "$HOME/.bashrc" ] ; then
...@@ -29,13 +30,4 @@ alias l='ls -CF' ...@@ -29,13 +30,4 @@ alias l='ls -CF'
echo "Welcome to SlapOS slaprunner shell" echo "Welcome to SlapOS slaprunner shell"
echo echo
# TODO: how to run slapos ?
#echo "You can use the following slapos CLI commands:"
#echo "slapos node software"
#echo "slapos node instance"
#echo "slapos request"
#echo ""
# XXX for now we can use:
#echo ~/bin/supervisorctl -c ~/etc/supervisorctl.conf
#echo ~/bin/supervisorctl -c ~/srv/runner/instance/etc/supervisorctl.conf
export PATH={{ bin_folder }}:$PATH
...@@ -6,6 +6,7 @@ umask 077 ...@@ -6,6 +6,7 @@ umask 077
# Exit on any error, to prevent inconsistent backup # Exit on any error, to prevent inconsistent backup
# Error on unset variable expansion # Error on unset variable expansion
set -eu set -eu
set -o pipefail
# Redirect output to log # Redirect output to log
exec > >(tee -ai '{{ output_log_file }}') exec > >(tee -ai '{{ output_log_file }}')
...@@ -22,18 +23,10 @@ rsync () { ...@@ -22,18 +23,10 @@ rsync () {
# Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653 # Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
IGNOREEXIT=24 IGNOREEXIT=24
IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
set -o pipefail
set -x set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>1 | (egrep -v "$IGNOREOUT" || true) '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>&1 | (egrep -v "$IGNOREOUT" || true) || [ $? = "$IGNOREEXIT" ]
set +x set +x
ret=$?
if [[ $ret == $IGNOREEXIT ]]; then
ret=0
fi
exit $ret
} }
( (
......
...@@ -59,7 +59,7 @@ eggs = collective.recipe.template ...@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.28 slapos.recipe.build = 0.28
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test. # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout] [buildout]
......
...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb ...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640 mode = 640
[versions] [versions]
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
...@@ -2,13 +2,16 @@ ...@@ -2,13 +2,16 @@
extends = extends =
../../component/keras/buildout.cfg ../../component/keras/buildout.cfg
../../component/chainer/buildout.cfg ../../component/chainer/buildout.cfg
../../component/pytorch/buildout.cfg
software.cfg software.cfg
parts += parts +=
keras-egg keras-egg
chainer-egg chainer-egg
pytorch-egg
[eggs] [eggs]
eggs += eggs +=
${tensorboard-build-install-egg:egg}
${tensorflow-build-install-egg:egg} ${tensorflow-build-install-egg:egg}
${protobuf-python:egg} ${protobuf-python:egg}
${h5py:egg} ${h5py:egg}
...@@ -19,6 +22,9 @@ eggs += ...@@ -19,6 +22,9 @@ eggs +=
filelock filelock
nose nose
${chainer:egg} ${chainer:egg}
${pytorch-build-install-egg:egg}
${python-cocoapi-build-install-egg:egg}
torchvision
[cuda] [cuda]
# If you use cuda, please adjust paramters for your environment # If you use cuda, please adjust paramters for your environment
......
[$${caddy-configuration:ipv6}]:$${caddy-configuration:port}
log $${caddy-configuration:access_log}
root $${directory:public_html}
\ No newline at end of file
[buildout]
extends =
../../component/golang/buildout.cfg
../../stack/slapos.cfg
gowork.cfg
parts =
gowork
slapos-cookbook
instance-profile
template-caddy
caddy
[gowork]
install =
github.com/mholt/caddy
[instance-profile]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = ae83c35995ce14432ded78ca30cab61b
output =${buildout:directory}/instance.cfg
filename = instance.cfg
mode = 0644
[template-caddyfile]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/Caddyfile.in
md5sum = 616f9c7cb788e1f1d7cee2093a7dc6ef
output = ${buildout:directory}/Caddyfile.in
mode = 0644
[template-caddy]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-caddy.cfg.in
md5sum = 045022e5698badca5dbdd6b95518f8e1
output = ${buildout:directory}/instance-caddy.cfg.in
mode = 0644
[template-public-html]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/index.html
md5sum = b5794ac8b10ed90173ad566e6e324b35
output = ${buildout:directory}/index.html
mode = 0644
[caddy]
recipe = slapos.recipe.cmmi
path = ${go_github.com_mholt_caddy:location}
go = ${gowork:golang}/bin/go
configure-command = :
make-targets =
make-binary = cd ${:path}/caddy && ${:go} install -v
environment =
PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gowork:directory}
output = ${gowork:bin}/caddy
\ No newline at end of file
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_mholt_caddy:recipe}
[go_github.com_mholt_caddy]
<= go-git-package
go.importpath = github.com/mholt/caddy
repository = https://github.com/mholt/caddy.git
revision = v0.10.10-0-gc4dfbb9956
<h1>Welcome</h1>
\ No newline at end of file
[buildout]
parts =
caddy-service
caddy-configuration
public-html
publish-connection-information
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
service = $${:etc}/service
public_html = $${buildout:directory}/public_html
run = $${:var}/run
log = $${:var}/log
#www = $${:srv}/www
#ssl = $${:etc}/ssl
#################################
# caddy service
#################################
[caddy-service]
recipe = slapos.cookbook:wrapper
command-line = ${caddy:output} -conf $${caddy-configuration:output}
wrapper-path = $${directory:service}/caddy
output = $${:wrapper-path}
[caddy-configuration]
recipe = slapos.recipe.template
url = ${template-caddyfile:output}
output = $${directory:etc}/Caddyfile
mode = 0600
access_log = $${directory:log}/caddy-access.log
error_log = $${directory:log}/caddy-error.log
ipv6 = $${slap-network-information:global-ipv6}
local_ip = $${slap-network-information:local-ipv4}
port = 9443
[public-html]
recipe = slapos.recipe.template
url = ${template-public-html:output}
output = $${directory:public_html}/index.html
mode = 0600
[publish-connection-information]
recipe = slapos.cookbook:publish
url = http://[$${caddy-configuration:ipv6}]:$${caddy-configuration:port}
\ No newline at end of file
#############################
#
# Deploy caddy instance
#
#############################
[buildout]
parts =
switch-softwaretype
# publish-connection-parameter
# Define egg directories to be the one from Software Release
# (/opt/slapgrid/...)
# Always the same.
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:caddy}
caddy = $${dynamic-template-caddy:rendered}
[dynamic-template-caddy]
recipe = slapos.recipe.template:jinja2
template = ${template-caddy:output}
rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
filename = instance-caddy.cfg
[slap-connection]
computer-id = {slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[instance-parameter]
# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
# We use the slapconfiguration recipe with a few parameters (partition id,
# computer id, certificate, etc).
# It will then authenticate to SlapOS Master and fetch the instance parameters.
# The parameters are accessible from {instance-parameter:configuration.name-of-parameter}
# Always the same. Just copy/paste.
# See docstring of slapos.cookbook:slapconfiguration for more information.
recipe = slapos.cookbook:slapconfiguration
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
...@@ -94,8 +94,8 @@ caucase = 0.1.4 ...@@ -94,8 +94,8 @@ caucase = 0.1.4
futures = 3.1.1 futures = 3.1.1
gitdb2 = 2.0.2 gitdb2 = 2.0.2
gunicorn = 19.7.1 gunicorn = 19.7.1
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
slapos.toolbox = 0.72 slapos.toolbox = 0.73
smmap2 = 2.0.3 smmap2 = 2.0.3
# Required by: # Required by:
......
...@@ -28,7 +28,7 @@ md5sum = a317d2f948cd3d16c860d05cc07ecf42 ...@@ -28,7 +28,7 @@ md5sum = a317d2f948cd3d16c860d05cc07ecf42
[template-caucase] [template-caucase]
filename = instance-caucase.cfg.jinja2.in filename = instance-caucase.cfg.jinja2.in
md5sum = 7db9e8bf23cf4689e7986c381b94d2cb md5sum = d31d4c9855d653d3d74c6133a7d85530
[instance-caucase] [instance-caucase]
filename = instance.cfg.in filename = instance.cfg.in
......
...@@ -157,7 +157,7 @@ mode = 700 ...@@ -157,7 +157,7 @@ mode = 700
[caucase-gunicorn] [caucase-gunicorn]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
socket = ${directory:ca-dir}/ca.flaskserver.sock socket = ${directory:ca-dir}/ng.sock
command-line = {{ gunicorn_bin }} caucase.wsgi:app -b unix:${:socket} -e CA_CONFIGURATION_FILE=${caucase-conf:output} --error-logfile ${:log-file} --pid ${:pid-file} --capture-output --timeout 60 --threads 2 --log-level error --preload command-line = {{ gunicorn_bin }} caucase.wsgi:app -b unix:${:socket} -e CA_CONFIGURATION_FILE=${caucase-conf:output} --error-logfile ${:log-file} --pid ${:pid-file} --capture-output --timeout 60 --threads 2 --log-level error --preload
log-file = ${directory:log}/ca-gunicorn-error.log log-file = ${directory:log}/ca-gunicorn-error.log
pid-file = ${directory:run}/ca-gunicorn.pid pid-file = ${directory:run}/ca-gunicorn.pid
......
...@@ -89,7 +89,7 @@ PasteScript = 2.0.2 ...@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7 WSGIUtils = 0.7
python-magic = 0.4.6 python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 3.0 slapos.recipe.template = 4.1
# Required by: # Required by:
# PasteScript==2.0 # PasteScript==2.0
......
...@@ -167,6 +167,10 @@ stop-on-error = true ...@@ -167,6 +167,10 @@ stop-on-error = true
update-command = ${:command} update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
[mariadb-start-clone-from-backup]
<= download-base
mode = 755
[mariadb-resiliency-after-import-script] [mariadb-resiliency-after-import-script]
<= download-base <= download-base
mode = 755 mode = 755
...@@ -276,6 +280,7 @@ context = ...@@ -276,6 +280,7 @@ context =
key mariadb_location mariadb:location key mariadb_location mariadb:location
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key mariadb_slow_query_report_script mariadb-slow-query-report-script:target key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
key matplotlibrc_location matplotlibrc:location key matplotlibrc_location matplotlibrc:location
key mesa_location mesa:location key mesa_location mesa:location
key onlyoffice_x2t_location onlyoffice-x2t:location key onlyoffice_x2t_location onlyoffice-x2t:location
...@@ -683,8 +688,6 @@ PyXML = 0.8.5 ...@@ -683,8 +688,6 @@ PyXML = 0.8.5
Pympler = 0.4.3 Pympler = 0.4.3
StructuredText = 2.11.1 StructuredText = 2.11.1
WSGIUtils = 0.7 WSGIUtils = 0.7
# ZEO 5 requires transaction >= 2
ZEO = 4.3.1
ZODB3 = 3.11.0 ZODB3 = 3.11.0
# astroid 1.4.1 breaks testDynamicClassGeneration # astroid 1.4.1 breaks testDynamicClassGeneration
astroid = 1.3.8 astroid = 1.3.8
......
...@@ -15,15 +15,19 @@ ...@@ -15,15 +15,19 @@
# not need these here). # not need these here).
[mariadb-resiliency-after-import-script] [mariadb-resiliency-after-import-script]
filename = instance-mariadb-resiliency-after-import-script.sh.in filename = instance-mariadb-resiliency-after-import-script.sh.in
md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49 md5sum = c1f1083bf6c911a0e65dcb841fba327d
[mariadb-slow-query-report-script] [mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in filename = mysql-querydigest.sh.in
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
[mariadb-start-clone-from-backup]
filename = instance-mariadb-start-clone-from-backup.sh.in
md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb] [template-mariadb]
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = 99450bb6426770bd0c27f9dbec9ca542 md5sum = 8ea5033142f450a2e90431817771cb44
[template-kumofs] [template-kumofs]
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
...@@ -67,7 +71,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0 ...@@ -67,7 +71,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = f95174eee2de45d1e3ccb1d5f88355f1 md5sum = 47d09a83d44f38d3ea62743f004e866b
[monitor-template-dummy] [monitor-template-dummy]
filename = dummy.cfg filename = dummy.cfg
...@@ -75,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e ...@@ -75,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 909c8eb4f1f2f2d58ad982cec67228bc md5sum = 33c8f97afe58058d428108fb933523ac
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -87,7 +91,7 @@ md5sum = 27d26c6380883cf3bd7b2f003f7888d8 ...@@ -87,7 +91,7 @@ md5sum = 27d26c6380883cf3bd7b2f003f7888d8
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = c80648d5918ae865bc02cb8fccfc56e3 md5sum = f64c568f1365eb1164f12f48fede9a99
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
......
...@@ -288,7 +288,7 @@ apachedex = ${monitor-directory:private}/apachedex ...@@ -288,7 +288,7 @@ apachedex = ${monitor-directory:private}/apachedex
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command} wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}" command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
command = apachedex_every_3_hour command = apachedex_every_23_hour
[apachedex-parameters] [apachedex-parameters]
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
...@@ -299,9 +299,8 @@ promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }} ...@@ -299,9 +299,8 @@ promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
[{{ section('monitor-promise-apachedex-result') }}] [{{ section('monitor-promise-apachedex-result') }}]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
recipe =
wrapper-path = ${directory:promise}/check-apachedex-result wrapper-path = ${directory:promise}/check-apachedex-result
command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_file "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}" command-line = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
......
...@@ -143,7 +143,7 @@ return = ...@@ -143,7 +143,7 @@ return =
zope-address-list zope-address-list
hosts-dict hosts-dict
monitor-base-url monitor-base-url
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%} {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
{% if has_jupyter -%} {% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% endif -%} {% endif -%}
......
...@@ -15,7 +15,6 @@ set -e ...@@ -15,7 +15,6 @@ set -e
mysql_executable='{{ mysql_executable }}' mysql_executable='{{ mysql_executable }}'
mariadb_data_directory='{{ mariadb_data_directory }}' mariadb_data_directory='{{ mariadb_data_directory }}'
mariadb_backup_directory='{{ mariadb_backup_directory }}' mariadb_backup_directory='{{ mariadb_backup_directory }}'
instance_directory='{{ instance_directory }}'
pid_file='{{ pid_file }}' pid_file='{{ pid_file }}'
binlog_path='{{ binlog_path }}' binlog_path='{{ binlog_path }}'
server_executable='{{ server_executable }}' server_executable='{{ server_executable }}'
...@@ -71,7 +70,7 @@ fi ...@@ -71,7 +70,7 @@ fi
echo "Importing data..." echo "Importing data..."
# Use latest dump XXX can contain funny characters # Use latest dump XXX can contain funny characters
dump=$(ls -r "$mariadb_backup_directory" | head -1) dump=$(ls -r "$mariadb_backup_directory" | head -1)
zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock" zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root
RESTORE_EXIT_CODE=$? RESTORE_EXIT_CODE=$?
if [ $RESTORE_EXIT_CODE -eq 0 ]; then if [ $RESTORE_EXIT_CODE -eq 0 ]; then
......
#!{{ dash }}
set -eu
if [ $# -ne 7 ]; then
echo "Bootstrap a mariadb instance from available backup data so it replicates from given master."
echo " $0 <BACKUP FILE> <MASTER HOST> <MASTER PORT> <MASTER USER> <MASTER SSL CA FILE> <MASTER SSL CERT FILE> <MASTER SSL KEY FILE>"
exit 1
fi
BACKUP=$1
MASTER_HOST=$2
MASTER_PORT=$3
MASTER_USER=$4
MASTER_SSL_CA=$5
MASTER_SSL_CERT=$6
MASTER_SSL_KEY=$7
CLIENT='{{ client }}'
DATA_DIRECTORY='{{ data_directory }}'
PID_FILE='{{ pid_file }}'
SERVER='{{ server }}'
UPDATE='{{ update }}'
SOCKET='{{ socket }}'
# Make sure mariadb is not already running
if [ -e "$PID_FILE" ]; then
PID=$(cat "$PID_FILE")
if [ $? -ne 0 ]; then
echo "Cannot read Mariadb pidfile, assuming running. Aborting."
exit 1
fi
if kill -0 "$PID"; then
echo "Mariadb is already running with pid $PID. Aborting."
exit 1
fi
fi
BACKUP_HEAD="$(zcat "$BACKUP" | head -n 100)"
SQL_CHANGE_MASTER=$(echo "$BACKUP_HEAD" | grep "^--\s*CHANGE MASTER TO " | sed "s/^--\s*//")
if [ -z "$SQL_CHANGE_MASTER" ]; then
echo "'CHANGE MASTER TO' statement not found in given backup file."
echo "Is replication enabled on future master ?"
exit 1
fi
SQL_SET_GTID="$(echo "$BACKUP_HEAD" | grep "^--\s*SET GLOBAL gtid_slave_pos=" | sed "s/^--\s*//")"
if [ -z "$SQL_SET_GTID" ]; then
echo "Info: GTID not found in backup, it will not be enabled."
MASTER_USE_GTID=0
else
echo "Info: GTID found in backup, it will be enabled."
MASTER_USE_GTID=1
fi
echo "EXISTING DATABASE CONTENT WILL BE DESTROYED"
echo "You have 5 seconds to interrupt this script..."
if sleep 5; then
echo "Expired, proceeding"
else
echo "Interrupted, aborting"
exit 1
fi
echo "Emptying data directory..."
find "$DATA_DIRECTORY" -mindepth 1 -delete
echo -n "Starting mariadb for backup restoration"
"$SERVER" --innodb-flush-method=nosync --skip-innodb-doublewrite --innodb-flush-log-at-trx-commit=0 --sync-frm=0 --slow-query-log=0 --skip-log-bin &
PID=$!
trap "kill $PID; wait; exit 1" EXIT
while true; do
if [ ! -e "/proc/$PID" ]; then
trap EXIT
echo "Service exited, check logs"
wait
exit 1
fi
test -e "$SOCKET" && break
echo -n .
sleep 0.5
done
"$UPDATE"
echo "Importing $BACKUP ..."
zcat "$BACKUP" | "$CLIENT" -u root
echo "Configuring server as slave..."
if [ "$MASTER_USE_GTID" -eq 1 ]; then
MASTER_USE_GTID_SQL="current_pos"
else
MASTER_USE_GTID_SQL="NO"
fi
"$CLIENT" -u root -e "
CHANGE MASTER TO
MASTER_HOST='$MASTER_HOST',
MASTER_USER='$MASTER_USER',
MASTER_PORT=$MASTER_PORT,
MASTER_SSL=1,
MASTER_SSL_CA='$MASTER_SSL_CA',
MASTER_SSL_CERT='$MASTER_SSL_CERT',
MASTER_SSL_KEY='$MASTER_SSL_KEY',
MASTER_SSL_VERIFY_SERVER_CERT=1,
MASTER_USE_GTID=$MASTER_USE_GTID_SQL;
"
"$CLIENT" -u root -e "$SQL_CHANGE_MASTER"
test "$MASTER_USE_GTID" -eq 1 && "$CLIENT" -u root -e "$SQL_SET_GTID"
"$CLIENT" -u root -e "START SLAVE;"
echo "Stopping mariadb..."
trap EXIT
kill $PID
wait
echo "Done. Start mariadb normally. You may use 'show slave status' SQL command to monitor progress."
...@@ -252,6 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl ...@@ -252,6 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var var = ${buildout:directory}/var
log = ${:var}/log log = ${:var}/log
run = ${:var}/run run = ${:var}/run
slowquery = ${monitor-directory:private}/slowquerydex
[{{ section('resiliency-exclude-file') }}] [{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency # Generate rdiff exclude file in case of resiliency
...@@ -262,7 +263,20 @@ rendered = ${directory:srv}/exporter.exclude ...@@ -262,7 +263,20 @@ rendered = ${directory:srv}/exporter.exclude
[dash] [dash]
dash = {{ dumps(dash) }} dash = {{ dumps(dash) }}
[resiliency-after-import-script] [{{ section('start-clone-from-backup') }}]
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-start-clone-from-backup'] }}
rendered = ${directory:bin}/start-clone-from-backup
context =
key dash dash:dash
key client binary-wrap-mysql:wrapper-path
key data_directory directory:mariadb-data
key pid_file my-cnf-parameters:pid-file
key server mysqld:rendered
key update update-mysql:output
key socket my-cnf-parameters:socket
[{{ section('resiliency-after-import-script') }}]
# Generate after import script used by importer instance of webrunner # Generate after import script used by importer instance of webrunner
< = jinja2-template-executable < = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }} template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
...@@ -272,7 +286,6 @@ context = ...@@ -272,7 +286,6 @@ context =
key mysql_executable binary-wrap-mysql:wrapper-path key mysql_executable binary-wrap-mysql:wrapper-path
key mariadb_data_directory directory:mariadb-data key mariadb_data_directory directory:mariadb-data
key mariadb_backup_directory directory:mariadb-backup-full key mariadb_backup_directory directory:mariadb-backup-full
key instance_directory buildout:directory
key pid_file my-cnf-parameters:pid-file key pid_file my-cnf-parameters:pid-file
key binlog_path my-cnf-parameters:binlog-path key binlog_path my-cnf-parameters:binlog-path
key server_executable mysqld:rendered key server_executable mysqld:rendered
...@@ -288,7 +301,7 @@ context = ...@@ -288,7 +301,7 @@ context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path} raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder monitor-directory:private key output_folder directory:slowquery
[slow-query-digest-parameters] [slow-query-digest-parameters]
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }} max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
...@@ -296,9 +309,8 @@ slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }} ...@@ -296,9 +309,8 @@ slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
[{{ section('monitor-promise-slowquery-result') }}] [{{ section('monitor-promise-slowquery-result') }}]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
recipe =
wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result wrapper-path = ${directory:promise}/check-slow-query-pt-digest-result
command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_file "${monitor-directory:private}/slowquery_digest.txt" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}" command-line = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}"
[{{ section('promise') }}] [{{ section('promise') }}]
......
...@@ -99,10 +99,6 @@ bin-directory = {{ bin_directory }} ...@@ -99,10 +99,6 @@ bin-directory = {{ bin_directory }}
apachedex-location = {{ bin_directory }}/apachedex apachedex-location = {{ bin_directory }}/apachedex
run-apachedex-location = {{ bin_directory }}/runApacheDex run-apachedex-location = {{ bin_directory }}/runApacheDex
promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result
6tunnel = {{ sixtunnel_location }}
curl-location = {{ curl_location }}
dash = {{ dash_location }}
bash = {{ bash_location }}
template-haproxy-cfg = {{ template_haproxy_cfg }} template-haproxy-cfg = {{ template_haproxy_cfg }}
template-apache-conf = {{ template_apache_conf }} template-apache-conf = {{ template_apache_conf }}
template-monitor = {{ dumps(template_monitor) }} template-monitor = {{ dumps(template_monitor) }}
...@@ -183,6 +179,7 @@ link-binary = {{ dumps(mariadb_link_binary) }} ...@@ -183,6 +179,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
bin-directory = {{ bin_directory }} bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }} mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }} mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
mariadb-start-clone-from-backup = {{ mariadb_start_clone_from_backup }}
promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
percona-tools-location = {{ percona_toolkit_location }} percona-tools-location = {{ percona_toolkit_location }}
template-monitor = {{ template_monitor }} template-monitor = {{ template_monitor }}
......
...@@ -131,5 +131,5 @@ depends = ...@@ -131,5 +131,5 @@ depends =
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.72 slapos.toolbox = 0.73
...@@ -53,6 +53,9 @@ allow-hosts += ...@@ -53,6 +53,9 @@ allow-hosts +=
www.dabeaz.com www.dabeaz.com
www.owlfish.com www.owlfish.com
# Use an https index
index = https://pypi.python.org/simple/
# XXX: Workaround of SlapOS limitation # XXX: Workaround of SlapOS limitation
# Unzippig of eggs is required, as SlapOS do not yet provide nicely working # Unzippig of eggs is required, as SlapOS do not yet provide nicely working
# development / fast switching environment for whole software # development / fast switching environment for whole software
...@@ -116,7 +119,7 @@ click = 6.7 ...@@ -116,7 +119,7 @@ click = 6.7
cliff = 2.4.0 cliff = 2.4.0
cmd2 = 0.7.0 cmd2 = 0.7.0
collective.recipe.template = 2.0 collective.recipe.template = 2.0
cryptography = 2.0.3 cryptography = 2.1.1
decorator = 4.0.11 decorator = 4.0.11
idna = 2.2 idna = 2.2
inotifyx = 0.2.2 inotifyx = 0.2.2
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment