slaprunner: partial rewrite of import/export scrips in Python

Further rewrite should be done but at least there's no awk code anymore.

component/gawk/buildout.cfg

-[buildout]
-parts = gawk
-
-[gawk]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gawk/gawk-4.1.4.tar.gz
-md5sum = f20c94ca51b6ebfc9bffb90f95c8ffbb

software/slaprunner/buildout.hash.cfg

@@ -15,7 +15,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = e89d25c1bb7bf507bd96973d444934a6
+md5sum = 78e4c796b15a9c5b9b41c79a3c995ee7
 
 [template-runner]
 filename = instance-runner.cfg
@@ -23,19 +23,19 @@ md5sum = c76c66c331f1f0d9f416c56ead53e804
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = feee1dc29d160e0d796dab0ba767745f
+md5sum = 40e839e76d03fabefb1859ea72383529
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 70931f214b3f7c3992e29c150a30d3cf
+md5sum = 5d49f4fe6de7bd02db0067d0aeac6de4
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = b921435ab5a8c408026e74d9a40eea44
+md5sum = 02023f89b622c49aee42130f839846a7
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 546b368baaed84bd2d3be7d6b68a5030
+md5sum = 22a5dfe9bfc8533f8eab851e0996a6d7
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/common.cfg

@@ -2,12 +2,10 @@
 extends =
   buildout.hash.cfg
   ../../component/bash/buildout.cfg
-  ../../component/coreutils/buildout.cfg
   ../../component/busybox/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
-  ../../component/gawk/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/tig/buildout.cfg
   ../../component/logrotate/buildout.cfg

software/slaprunner/instance-runner-export.cfg.in

@@ -62,11 +62,6 @@ ip = ${slaprunner:ipv4}
 [supervisord]
 port = ${supervisord-free-port:port}
 
-[exporter-configuration]
-coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
-gawk-location = {{ dumps(parameter_dict['gawk-location']) }}
-backup_wait_time = {{ dumps(parameter_dict['backup_wait_time']) }}
-
 [exporter]
 recipe = slapos.recipe.template:jinja2
 template = {{ exporter_script_path }}
@@ -75,8 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
 wrapper = ${:rendered}
 mode = 700
 context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
   section directory directory
-  section parameter_dict exporter-configuration
+  raw  backup_wait_time {{ backup_wait_time }}
   raw  output_log_file ${directory:log}/resilient.log
   raw  shell_binary {{ bash_executable_location }}
   raw  rsync_binary {{ rsync_executable_location }}

software/slaprunner/instance-runner-import.cfg.in

@@ -72,10 +72,6 @@ maximum = 22241
 [cron-entry-prepare-software]
 recipe =
 
-[importer-configuration]
-coreutils-location = ${coreutils:location}
-gawk-location = ${gawk:location}
-
 [importer]
 recipe = slapos.recipe.template:jinja2
 template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
@@ -89,12 +85,13 @@ restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-ba
 restore-error-message-file-basename = importer-error-message-file
 resilient-log-basename = resilient.log
 context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
   key backend_url slaprunner:access-url
   key ipv4 slaprunner:ipv4
   key ipv6 slaprunner:ipv6
   key proxy_port slaprunner:proxy_port
   key instance_folder slaprunner:instance_root
-  section parameter_dict importer-configuration
   section directory directory
   section supervisord supervisord
   raw  output_log_file $${directory:log}/$${:resilient-log-basename}

software/slaprunner/instance.cfg

@@ -47,11 +47,6 @@ import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 mode = 0644
 
-[template-runner-export-configuration]
-coreutils-location = ${coreutils:location}
-gawk-location = ${gawk:location}
-backup_wait_time = ${exporter-default-configuration:backup_wait_time}
-
 [template-runner-export]
 recipe = slapos.recipe.template:jinja2
 template = ${instance-runner-export:target}
@@ -60,7 +55,7 @@ mode = 640
 context =
   key pbsready_export_template_path template-pbsready-export:rendered
   key template_runner_path instance-base-runner:rendered
-  section parameter_dict template-runner-export-configuration
+  raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
   raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
   raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
   raw buildout_executable_location ${buildout:executable}

software/slaprunner/template/runner-export.sh.jinja2

@@ -24,15 +24,8 @@ rsync () {
   set +x
 }
 
-relativise () {
-  while IFS= read -r line; do
-    if [ ! -z "$line" ]; then
-      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
-    fi
-  done
-}
-
 (
+  # XXX: code duplication with runner-import.sh.jinja2
   path=$srv_directory/runner
   backup_path=$backup_directory/runner/
   cd "$path"
@@ -41,31 +34,46 @@ relativise () {
     # Concatenate the exclude file of each partition of webrunner
     # to create a global exclude file.
     # Also, ignore all buildout-managed files.
-    (
-      echo "*.sock"
-      echo "*.socket"
-      echo "*.pid"
-      echo ".installed*.cfg"
-      for partition in "$path"/instance/slappart*; do
-        # So "relativise" can handle relative paths (which are expected to be relative to partition).
-        cd "$partition"
-        exclude_file=srv/exporter.exclude
-        if [ -r "$exclude_file" ]; then
-          relativise "$path" < "$exclude_file"
-        fi
-        for installed in .installed*.cfg; do
-          if [ -r "$installed" ]; then
-            # Print every line from each __buildout_installed__ found.
-            '{{ parameter_dict["gawk-location"] }}/bin/gawk' '
-              BEGIN { do_print = 0 }
-              match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
-              /^\S/ { do_print = 0; next }
-              match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
-            ' "$installed" | relativise "$path"
-          fi
-        done
-      done
-    ) | rsync --exclude-from=- instance "$backup_path"
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            os.chdir(partition)
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
   fi
 
   test -d project  && rsync project "$backup_path"
@@ -116,6 +124,6 @@ if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
   exit 0
 fi
 echo "ERROR: Some backups are not consistent, exporter should be re-run."
-echo "Let's sleep {{ parameter_dict['backup_wait_time'] }} minutes, to let the backup end..."
-sleep {{ parameter_dict["backup_wait_time"] }}m
+echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
+sleep {{ backup_wait_time }}m
 exit 1

software/slaprunner/template/runner-import.sh.jinja2

@@ -41,16 +41,9 @@ rsync () {
   set +x
 }
 
-relativise () {
-  while IFS= read -r line; do
-    if [ ! -z "$line" ]; then
-      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
-    fi
-  done
-}
-
 log_message "Restoring WebRunner content..."
 (
+  # XXX: code duplication with runner-export.sh.jinja2
   path=$srv_directory/runner
   backup_path=$backup_directory/runner/
   cd "$backup_path"
@@ -59,31 +52,46 @@ log_message "Restoring WebRunner content..."
     # Concatenate the exclude file of each partition of webrunner
     # to create a global exclude file.
     # Also, ignore all buildout-managed files.
-    (
-      echo "*.sock"
-      echo "*.socket"
-      echo "*.pid"
-      echo ".installed*.cfg"
-      for partition in "$path"/instance/slappart*; do
-        # So "relativise" can handle relative paths (which are expected to be relative to partition).
-        cd "$partition"
-        exclude_file=srv/exporter.exclude
-        if [ -r "$exclude_file" ]; then
-          relativise "$path" < "$exclude_file"
-        fi
-        for installed in .installed*.cfg; do
-          if [ -r "$installed" ]; then
-            # Print every line from each __buildout_installed__ found.
-            '{{ parameter_dict["gawk-location"] }}/bin/gawk' '
-              BEGIN { do_print = 0 }
-              match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
-              /^\S/ { do_print = 0; next }
-              match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
-            ' "$installed" | relativise "$path"
-          fi
-        done
-      done
-    ) | rsync --exclude-from=- instance "$path"
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            os.chdir(partition)
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$path"
   fi
 
   test -d project  && rsync project "$path"