Commit 02e5acf0 authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Add import script

parent 051e0cf9
......@@ -15,15 +15,15 @@
[instance-theia]
filename = instance-theia.cfg.jinja.in
md5sum = ded3b813c6f04a267a4da10d362196c0
md5sum = b21731996dc8e17b71ab616cada3e446
[instance]
filename = instance.cfg.in
md5sum = dbb8f3edad591b3715f3f49f52035f79
md5sum = c4f5f13fd2c898b21a0f70efe27676ae
[instance-import]
filename = instance-import.cfg.in
md5sum = 0398a33f33dc99679da0728b2cfbcbf9
md5sum = 4ba329e17bb196626c06c3be7b108163
[instance-export]
filename = instance-export.cfg.in
......@@ -37,6 +37,10 @@ md5sum = f3958f0f4aa3f9c0414b1872fd2236fb
filename = theia-export-script.jinja
md5sum = 56e139e3cce8ab034487e13fa78c5fb8
[theia-import-script]
filename = theia-import-script.jinja
md5sum = 8339aa7d56fa379b0c1b7568daca19fd
[yarn.lock]
filename = yarn.lock
md5sum = e9c0f6cc380b53425f521a167c2fdf66
......
......@@ -22,10 +22,39 @@ name = Theia Import Frontend
# is notified that the backup files have just been pushed to it.
# All it expects is that a script be available in importer:wrapper.
[importer]
recipe = slapos.cookbook:wrapper
command-line = echo "Import Not Implemented Yet"
wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
wrapper = $${:wrapper-path}
wrapper = $${theia-import-script:rendered}
[theia-import-script]
recipe = slapos.recipe.template:jinja2
template = ${software-info:theia-import-script}
rendered = $${directory:bin}/theia-import-script
mode = 700
restore-exit-code-file = $${directory:srv}/$${:restore-exit-code-file-basename}
restore-exit-code-file-basename = importer-exit-code-file
restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-basename}
restore-error-message-file-basename = importer-error-message-file
resilient-log-basename = resilient.log
context =
import sys sys
import easy_install zc.buildout.easy_install
key ipv4 slapos-standalone-config:ipv4
key ipv6 slapos-standalone-config:ipv6
key proxy_port slapos-standalone-config:port
raw instance_folder $${directory:runner}/instance
section directory directory
raw supervisorctl ${software-info:supervisorctl}
raw supervisord_conf $${directory:runner}/etc/supervisord.conf
raw slapos ${software-info:slapos}
raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
raw output_log_file $${directory:var}/log/$${:resilient-log-basename}
raw shell_binary ${software-info:bash}
raw sqlite3_binary ${software-info:sqlite3}
raw rsync_binary ${software-info:rsync}
raw restore_exit_code_file $${:restore-exit-code-file}
raw restore_error_message_file $${:restore-error-message-file}
# Resilient connection parameters of import instance are published
# through the resilient stack.
......
......@@ -291,7 +291,7 @@ template =
. ${gowork:env.sh}
# reset PS1 from gowork
export PS1='$ '
export HOME=$${buildout:directory}
export HOME=$${directory:home}
export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
......@@ -399,6 +399,7 @@ ip = $${instance-parameter:ipv4-random}
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = $${slapos-standalone-port:port}
local-software-release-root = $${directory:home}
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
......@@ -427,6 +428,7 @@ template =
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:local-software-release-root} \
$${slapos-standalone-config:computer-id} \
--sr='$${instance-parameter:configuration.embedded-sr}' \
--srtype='$${instance-parameter:configuration.embedded-sr-type}' \
......
......@@ -9,11 +9,15 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[software-info]
slapos = ${buildout:bin-directory}/slapos
python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
python = ${python:location}/bin/python
rsync = ${rsync:location}/bin/rsync
sqlite3 = ${sqlite3:location}/bin/sqlite3
bash = ${bash:location}/bin/bash
supervisorctl = ${buildout:bin-directory}/supervisorctl
theia-export-script = ${theia-export-script:output}
theia-import-script = ${theia-import-script:output}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
......
......@@ -34,11 +34,48 @@ common-parts =
rdiff-backup
parts =
# >>>>>>>>>>>>
# Use unreleased versions of slapos.core and slapos.toolbox
slapos.core-dev
slapos.toolbox-dev
# <<<<<<<<<<<<
${:common-parts}
# default for slapos-standalone
shared-part-list =
# >>>>>>>>>>>>
# Use unreleased versions of slapos.core and slapos.toolbox
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/xavier_thompson/slapos.core.git
branch = slapgrid_install_only_on_proxy_root_path
git-executable = ${git:location}/bin/git
develop = true
[slapos.core-dev]
recipe = zc.recipe.egg:develop
egg = slapos.core
setup = ${slapos.core-repository:location}
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
branch = 0.118
git-executable = ${git:location}/bin/git
develop = true
[slapos.toolbox-dev]
recipe = zc.recipe.egg:develop
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location}
depends = ${slapos-toolbox-dependencies:eggs}
[versions]
slapos.core =
slapos.toolbox =
# <<<<<<<<<<<<
# We keep the gcc part in sync with the one from erp5 software, so that when we install
# erp5 inside theia's slapos parts can be shared.
[gcc]
......@@ -76,6 +113,7 @@ initialization =
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
parser.add_argument('local_software_release_root')
parser.add_argument('computer_id')
parser.add_argument('--sr')
parser.add_argument('--srtype')
......@@ -109,6 +147,8 @@ initialization =
software_root="%s/software" % args.base_directory,
instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root=args.local_software_release_root,
)
standalone.start()
partition_count = 20
......@@ -365,3 +405,6 @@ output = ${buildout:directory}/instance.cfg
[theia-export-script]
<= download-base
[theia-import-script]
<= download-base
#!{{ shell_binary }}
LC_ALL=C
export LC_ALL
umask 077
# Exit on any error, to prevent inconsistent backup
# Error on unset variable expansion
set -eu
# Redirect output to log
exec > >(tee -ai '{{ output_log_file }}')
exec 2>&1
echo -e "\n\n$0 run at : $(date)"
srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
ERROR_MESSAGE=""
fail_with_exit_code () {
echo 1 > $RESTORE_EXIT_CODE_FILE
echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
exit 1
}
trap fail_with_exit_code ERR
log_message () {
ERROR_MESSAGE=$1
echo -e $1
}
# Delete the error message file, to not keep it even after a successful build
rm "$RESTORE_ERROR_MESSAGE_FILE" || true
rsync () {
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
set +x
}
SQLITE3="{{ sqlite3_binary }}"
log_message "Stop Slapproxy..."
{{ supervisorctl }} -c {{ supervisord_conf }} stop slapos-proxy
log_message "Restoring WebRunner content..."
(
# XXX: code duplication with runner-export.sh.jinja2
path=$srv_directory
backup_path=$backup_directory/srv
cd "$backup_path"
if [ -d runner/instance ]; then
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
exclude=$({{ sys.executable }} - "$path/runner" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
try:
os.chdir(partition)
except OSError as e:
if e.errno != errno.ENOTDIR:
raise
continue
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "Restoring partitions"
echo "$exclude" | rsync --exclude-from=- runner/instance "$path/runner"
fi
test -d project && echo "Restoring project folder" && rsync project "$path"
test -d frontend-static/public && echo "Restoring public folder" && rsync frontend-static/public "$path/frontend-static"
test -f runner/var/proxy.db.dump && echo "Restoring proxy" && rm $path/runner/var/proxy.db && $SQLITE3 $path/runner/var/proxy.db ".read runner/var/proxy.db.dump"
)
log_message "Restoring WebRunner config (etc directory)..."
(
cd "$backup_directory"/etc/
# Hidden files are related to the webrunner's internals
cp -r .??* "$etc_directory"
)
# Invoke arbitrary script to perform specific restoration
# procedure.
runner_import_restore=$srv_directory/runner-import-restore
if [ -x "$runner_import_restore" ]; then
log_message "Running $runner_import_restore..."
"$srv_directory/runner-import-restore"
fi
# If no "etc/.project" neither "srv/runner/var/proxy.db", we can safely assume
# that there is no instance deployed on runner0
if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/var/proxy.db" ]; then
log_message "No Software Requested... Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
log_message "Updating slapproxy database..."
# Restart slapproxy to load new slapos.cfg configuration
log_message "Start Slapproxy with new configuration..."
{{ supervisorctl }} -c {{ supervisord_conf }} start slapos-proxy
HOME='{{ directory["home"] }}'
# XXX Hardcoded
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export MAKEFLAGS=-j4
SLAPOS='{{ slapos }}'
DATABASE="$HOME/srv/runner/var/proxy.db"
db_query () {
# Try opening locked tables for 5 seconds to prevent "database is locked" error
"$SQLITE3" "$DATABASE" <<EOF
.timeout 5000
$@
EOF
}
# If slapproxy database is empty then no software release was opened
if [ ! -s "$DATABASE" ]; then
log_message "Slapproxy database empty, no Software Requested... Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
# Slap proxy table contain version number, find the table name dynamically.
# This is known to work with version 11 or 12 of tables, but it will probably
# work with earlier versions as well.
DB_PARTITION_TABLE=$(db_query ".table partition__")
DB_PARTITION_NETWORK_TABLE=$(db_query ".table partition\_network__")
DB_SOFTWARE_TABLE=$(db_query ".table software__")
# XXX HACK: Change slapproxy database to get correct IPs
IPV4='{{ ipv4 }}'
IPV6='{{ ipv6 }}'
db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV4' where netmask='255.255.255.255';"
db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV6' where netmask='ffff:ffff:ffff::';"
MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
log_message "Removing old supervisord service description files..."
# XXX: Path hardcoded in slapos.core
rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
SLAPOSCFG='{{ slapos_cfg }}'
SLAPGRIDSRLOG='{{ slapos_node_software_log }}'
SLAPGRIDCPLOG='{{ slapos_node_instance_log }}'
contain_software_release=0
SOFTWARE_RELEASES_COUNT=$(db_query "SELECT count(1) FROM $DB_SOFTWARE_TABLE WHERE url != '';")
if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then
contain_software_release=1
fi
if [ $contain_software_release -eq 0 ]; then
log_message "No Software Release were deployed, so skip to continue..."
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
log_message "Building newest Software Release..."
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
(tail -n 200 "$SLAPGRIDSRLOG" && false)
contain_instance=0
for folder in $srv_directory/runner/instance/slappart*/; do
if [ -f "$folder/buildout.cfg" ]; then
contain_instance=1
fi
done
# If instance do not contains template.cfg it means the user contains no instance.
# so it is safer to assume that he is using slaprunner for develop buildout rather them slapos.
if [ $contain_instance -eq 0 ]; then
log_message "None Instance were deployed with this software release, so skip to continue..."
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
# Remove defined scripts to force buildout to recreate them to have updated paths
rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
log_message "Fixing Instances as needed after import..."
# XXX hardcoded
# Using '--no-supervisord' to avoid starting any services
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --no-supervisord --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" ||
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --no-supervisord --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" ||
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --no-supervisord --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" ||
(tail -n 200 "$SLAPGRIDCPLOG" && false)
# Invoke defined scripts for each partition inside of slaprunner
log_message "Invoke custom import scripts defined by each instances..."
for partition in "$srv_directory"/runner/instance/slappart*/
do
script=$partition/srv/runner-import-restore
if [ -x "$script" ]; then
log_message "Running custom instance script : $script..."
"$script"
fi
done
# XXX HACK: Remove stale timestamps after modifying the instances state
log_message "Removing stale timestamps..."
rm -f "$srv_directory"/runner/instance/slappart*/.timestamp
# Write exit code to an arbitrary file that will be checked by promise/monitor
log_message "Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment