software/theia: Add import script

software/theia/buildout.hash.cfg

@@ -15,15 +15,15 @@
 
 [instance-theia]
 filename = instance-theia.cfg.jinja.in
-md5sum = ded3b813c6f04a267a4da10d362196c0
+md5sum = b21731996dc8e17b71ab616cada3e446
 
 [instance]
 filename = instance.cfg.in
-md5sum = dbb8f3edad591b3715f3f49f52035f79
+md5sum = c4f5f13fd2c898b21a0f70efe27676ae
 
 [instance-import]
 filename = instance-import.cfg.in
-md5sum = 0398a33f33dc99679da0728b2cfbcbf9
+md5sum = 4ba329e17bb196626c06c3be7b108163
 
 [instance-export]
 filename = instance-export.cfg.in
@@ -37,6 +37,10 @@ md5sum = f3958f0f4aa3f9c0414b1872fd2236fb
 filename = theia-export-script.jinja
 md5sum = 56e139e3cce8ab034487e13fa78c5fb8
 
+[theia-import-script]
+filename = theia-import-script.jinja
+md5sum = 8339aa7d56fa379b0c1b7568daca19fd
+
 [yarn.lock]
 filename = yarn.lock
 md5sum = e9c0f6cc380b53425f521a167c2fdf66

software/theia/instance-import.cfg.in

@@ -22,10 +22,39 @@ name = Theia Import Frontend
 # is notified that the backup files have just been pushed to it.
 # All it expects is that a script be available in importer:wrapper.
 [importer]
-recipe = slapos.cookbook:wrapper
-command-line = echo "Import Not Implemented Yet"
-wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
-wrapper = $${:wrapper-path}
+wrapper = $${theia-import-script:rendered}
+
+[theia-import-script]
+recipe = slapos.recipe.template:jinja2
+template = ${software-info:theia-import-script}
+rendered = $${directory:bin}/theia-import-script
+mode = 700
+restore-exit-code-file = $${directory:srv}/$${:restore-exit-code-file-basename}
+restore-exit-code-file-basename = importer-exit-code-file
+restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-basename}
+restore-error-message-file-basename = importer-error-message-file
+resilient-log-basename = resilient.log
+context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
+  key ipv4 slapos-standalone-config:ipv4
+  key ipv6 slapos-standalone-config:ipv6
+  key proxy_port slapos-standalone-config:port
+  raw instance_folder $${directory:runner}/instance
+  section directory directory
+  raw supervisorctl ${software-info:supervisorctl}
+  raw supervisord_conf $${directory:runner}/etc/supervisord.conf
+  raw slapos ${software-info:slapos}
+  raw slapos_cfg $${directory:runner}/etc/slapos.cfg
+  raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
+  raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
+  raw  output_log_file $${directory:var}/log/$${:resilient-log-basename}
+  raw  shell_binary ${software-info:bash}
+  raw  sqlite3_binary ${software-info:sqlite3}
+  raw  rsync_binary ${software-info:rsync}
+  raw  restore_exit_code_file $${:restore-exit-code-file}
+  raw  restore_error_message_file $${:restore-error-message-file}
+
 
 # Resilient connection parameters of import instance are published
 # through the resilient stack.

software/theia/instance-theia.cfg.jinja.in

@@ -291,7 +291,7 @@ template =
   . ${gowork:env.sh}
   # reset PS1 from gowork
   export PS1='$ '
-  export HOME=$${buildout:directory}
+  export HOME=$${directory:home}
   export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
 
 
@@ -399,6 +399,7 @@ ip = $${instance-parameter:ipv4-random}
 ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
 port = $${slapos-standalone-port:port}
+local-software-release-root = $${directory:home}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
 
@@ -427,6 +428,7 @@ template =
       $${slapos-standalone-config:ipv4} \
       $${slapos-standalone-config:ipv6} \
       $${slapos-standalone-config:port} \
+      $${slapos-standalone-config:local-software-release-root} \
       $${slapos-standalone-config:computer-id} \
       --sr='$${instance-parameter:configuration.embedded-sr}' \
       --srtype='$${instance-parameter:configuration.embedded-sr-type}' \

software/theia/instance.cfg.in

@@ -9,11 +9,15 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
 [software-info]
+slapos = ${buildout:bin-directory}/slapos
 python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
 python = ${python:location}/bin/python
 rsync = ${rsync:location}/bin/rsync
 sqlite3 = ${sqlite3:location}/bin/sqlite3
+bash = ${bash:location}/bin/bash
+supervisorctl = ${buildout:bin-directory}/supervisorctl
 theia-export-script = ${theia-export-script:output}
+theia-import-script = ${theia-import-script:output}
 
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration

software/theia/software.cfg

@@ -34,11 +34,48 @@ common-parts =
   rdiff-backup
 
 parts =
+# >>>>>>>>>>>>
+# Use unreleased versions of slapos.core and slapos.toolbox
+  slapos.core-dev
+  slapos.toolbox-dev
+# <<<<<<<<<<<<
   ${:common-parts}
 
 # default for slapos-standalone
 shared-part-list =
 
+# >>>>>>>>>>>>
+# Use unreleased versions of slapos.core and slapos.toolbox
+[slapos.core-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/xavier_thompson/slapos.core.git
+branch = slapgrid_install_only_on_proxy_root_path
+git-executable = ${git:location}/bin/git
+develop = true
+
+[slapos.core-dev]
+recipe = zc.recipe.egg:develop
+egg = slapos.core
+setup = ${slapos.core-repository:location}
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
+branch = 0.118
+git-executable = ${git:location}/bin/git
+develop = true
+
+[slapos.toolbox-dev]
+recipe = zc.recipe.egg:develop
+egg = slapos.toolbox
+setup = ${slapos.toolbox-repository:location}
+depends = ${slapos-toolbox-dependencies:eggs}
+
+[versions]
+slapos.core =
+slapos.toolbox =
+# <<<<<<<<<<<<
+
 # We keep the gcc part in sync with the one from erp5 software, so that when we install
 # erp5 inside theia's slapos parts can be shared.
 [gcc]
@@ -76,6 +113,7 @@ initialization =
   parser.add_argument('ipv4')
   parser.add_argument('ipv6')
   parser.add_argument('server_port', type=int)
+  parser.add_argument('local_software_release_root')
   parser.add_argument('computer_id')
   parser.add_argument('--sr')
   parser.add_argument('--srtype')
@@ -109,6 +147,8 @@ initialization =
       software_root="%s/software" % args.base_directory,
       instance_root="%s/instance" % args.base_directory,
       partition_forward_configuration=partition_forward_configuration,
+      slapos_bin="${buildout:bin-directory}/slapos",
+      local_software_release_root=args.local_software_release_root,
   )
   standalone.start()
   partition_count = 20
@@ -365,3 +405,6 @@ output = ${buildout:directory}/instance.cfg
 
 [theia-export-script]
 <= download-base
+
+[theia-import-script]
+<= download-base

software/theia/theia-import-script.jinja

+#!{{ shell_binary }}
+LC_ALL=C
+export LC_ALL
+umask 077
+
+# Exit on any error, to prevent inconsistent backup
+# Error on unset variable expansion
+set -eu
+
+# Redirect output to log
+exec > >(tee -ai '{{ output_log_file }}')
+exec 2>&1
+
+echo -e "\n\n$0 run at : $(date)"
+
+srv_directory='{{ directory["srv"] }}'
+backup_directory='{{ directory["backup"] }}'
+etc_directory='{{ directory["etc"] }}'
+
+RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
+RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
+ERROR_MESSAGE=""
+
+fail_with_exit_code () {
+  echo 1 > $RESTORE_EXIT_CODE_FILE
+  echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
+  exit 1
+}
+trap fail_with_exit_code ERR
+
+log_message () {
+    ERROR_MESSAGE=$1
+    echo -e $1
+}
+# Delete the error message file, to not keep it even after a successful build
+rm "$RESTORE_ERROR_MESSAGE_FILE" || true
+
+rsync () {
+  set -x
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
+  set +x
+}
+
+SQLITE3="{{ sqlite3_binary }}"
+
+log_message "Stop Slapproxy..."
+{{ supervisorctl }} -c {{ supervisord_conf }} stop slapos-proxy
+
+log_message "Restoring WebRunner content..."
+(
+  # XXX: code duplication with runner-export.sh.jinja2
+  path=$srv_directory
+  backup_path=$backup_directory/srv
+  cd "$backup_path"
+
+  if [ -d runner/instance ]; then
+    # Concatenate the exclude file of each partition of webrunner
+    # to create a global exclude file.
+    # Also, ignore all buildout-managed files.
+    exclude=$({{ sys.executable }} - "$path/runner" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            try:
+              os.chdir(partition)
+            except OSError as e:
+              if e.errno != errno.ENOTDIR:
+                raise
+              continue
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "Restoring partitions"
+    echo "$exclude" | rsync --exclude-from=- runner/instance "$path/runner"
+  fi
+
+  test -d project && echo "Restoring project folder" && rsync project "$path"
+  test -d frontend-static/public && echo "Restoring public folder" && rsync frontend-static/public "$path/frontend-static"
+  test -f runner/var/proxy.db.dump && echo "Restoring proxy" && rm $path/runner/var/proxy.db && $SQLITE3 $path/runner/var/proxy.db ".read runner/var/proxy.db.dump"
+)
+
+log_message "Restoring WebRunner config (etc directory)..."
+(
+  cd "$backup_directory"/etc/
+  # Hidden files are related to the webrunner's internals
+  cp -r .??* "$etc_directory"
+)
+
+# Invoke arbitrary script to perform specific restoration
+# procedure.
+runner_import_restore=$srv_directory/runner-import-restore
+if [ -x "$runner_import_restore" ]; then
+  log_message "Running $runner_import_restore..."
+  "$srv_directory/runner-import-restore"
+fi
+
+# If no "etc/.project" neither "srv/runner/var/proxy.db", we can safely assume
+# that there is no instance deployed on runner0
+if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/var/proxy.db" ]; then
+  log_message "No Software Requested... Writing status file... End"
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+log_message "Updating slapproxy database..."
+
+# Restart slapproxy to load new slapos.cfg configuration
+log_message "Start Slapproxy with new configuration..."
+{{ supervisorctl }} -c {{ supervisord_conf }} start slapos-proxy
+
+HOME='{{ directory["home"] }}'
+# XXX Hardcoded
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export MAKEFLAGS=-j4
+SLAPOS='{{ slapos }}'
+DATABASE="$HOME/srv/runner/var/proxy.db"
+db_query () {
+  # Try opening locked tables for 5 seconds to prevent "database is locked" error
+"$SQLITE3" "$DATABASE" <<EOF
+.timeout 5000
+$@
+EOF
+}
+# If slapproxy database is empty then no software release was opened
+if [ ! -s "$DATABASE" ]; then
+  log_message "Slapproxy database empty, no Software Requested... Writing status file... End"
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+# Slap proxy table contain version number, find the table name dynamically.
+# This is known to work with version 11 or 12 of tables, but it will probably
+# work with earlier versions as well.
+DB_PARTITION_TABLE=$(db_query ".table partition__")
+DB_PARTITION_NETWORK_TABLE=$(db_query ".table partition\_network__")
+DB_SOFTWARE_TABLE=$(db_query ".table software__")
+
+# XXX HACK: Change slapproxy database to get correct IPs
+IPV4='{{ ipv4 }}'
+IPV6='{{ ipv6 }}'
+db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV4' where netmask='255.255.255.255';"
+db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV6' where netmask='ffff:ffff:ffff::';"
+
+MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
+
+log_message "Removing old supervisord service description files..."
+# XXX: Path hardcoded in slapos.core
+rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
+
+SLAPOSCFG='{{ slapos_cfg }}'
+SLAPGRIDSRLOG='{{ slapos_node_software_log }}'
+SLAPGRIDCPLOG='{{ slapos_node_instance_log }}'
+
+contain_software_release=0
+
+SOFTWARE_RELEASES_COUNT=$(db_query "SELECT count(1) FROM $DB_SOFTWARE_TABLE WHERE url != '';")
+if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then
+  contain_software_release=1
+fi
+
+if [ $contain_software_release -eq 0 ]; then
+  log_message "No Software Release were deployed, so skip to continue..."
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+log_message "Building newest Software Release..."
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+(tail -n 200 "$SLAPGRIDSRLOG" && false)
+
+contain_instance=0
+for folder in $srv_directory/runner/instance/slappart*/; do
+  if [ -f "$folder/buildout.cfg" ]; then
+    contain_instance=1
+  fi
+done
+
+# If instance do not contains template.cfg it means the user contains no instance.
+# so it is safer to assume that he is using slaprunner for develop buildout rather them slapos.
+if [ $contain_instance -eq 0 ]; then
+  log_message "None Instance were deployed with this software release, so skip to continue..."
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+# Remove defined scripts to force buildout to recreate them to have updated paths
+rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
+
+log_message "Fixing Instances as needed after import..."
+# XXX hardcoded
+# Using '--no-supervisord' to avoid starting any services
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --no-supervisord --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --no-supervisord --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --no-supervisord --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" ||
+(tail -n 200 "$SLAPGRIDCPLOG" && false)
+
+# Invoke defined scripts for each partition inside of slaprunner
+log_message "Invoke custom import scripts defined by each instances..."
+for partition in "$srv_directory"/runner/instance/slappart*/
+do
+  script=$partition/srv/runner-import-restore
+  if [ -x "$script" ]; then
+    log_message "Running custom instance script : $script..."
+    "$script"
+  fi
+done
+
+# XXX HACK: Remove stale timestamps after modifying the instances state
+log_message "Removing stale timestamps..."
+rm -f "$srv_directory"/runner/instance/slappart*/.timestamp
+
+# Write exit code to an arbitrary file that will be checked by promise/monitor
+log_message "Writing status file... End"
+echo 0 > $RESTORE_EXIT_CODE_FILE
+exit 0