Update Release Candidate

component/golang/buildout.cfg

@@ -39,9 +39,17 @@ environment =
   ${:environment-extra}
 patch-options = -p1
 
-[golang-common-pre-1.21]
+[golang-common-pre-1.25]
 <= golang-common
 
+# TestAmbientCaps and TestAmbientCapsUserns tests fail in the absence of a
+# "nobody" user
+patches +=
+  https://github.com/golang/go/commit/1959703dddffef61760d5d4fdedc4df8d2d73438.patch?full_index=1#ba1b2de74195880a29bb7913588358e5
+
+[golang-common-pre-1.21]
+<= golang-common-pre-1.25
+
 # TestChown currently fails in a user-namespace
 # https://github.com/golang/go/issues/42525
 # the patches apply to 1.21 > go >= 1.12
@@ -108,7 +116,7 @@ environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
 [golang1.21]
-<= golang-common
+<= golang-common-pre-1.25
 url = https://go.dev/dl/go1.21.13.src.tar.gz
 md5sum = 3454362fc56df5020920e43fee2776ab
 # go1.21 requires go1.17.13 to bootstrap (see https://go.dev/blog/rebuild)
@@ -125,7 +133,7 @@ patches +=
   ${:_profile_base_location_}/crypto-tls-fix-Config.Time-in-tests-using-expired-ce-go-1-21.patch#8e30a06c854a9654e5b789c887453d64
 
 [golang1.23]
-<= golang-common
+<= golang-common-pre-1.25
 url = https://go.dev/dl/go1.23.5.src.tar.gz
 md5sum = e13bea63175a402f3bdac29a048cc8b6
 # go1.23 requires go1.20.6 to bootstrap (see https://go.dev/blog/rebuild)

component/openssh/buildout.cfg

@@ -16,9 +16,9 @@ parts =
 [openssh]
 recipe = slapos.recipe.cmmi
 shared = true
-md5sum = bc04ff77796758c0b37bd0bc9314cd3f
+md5sum = f617b95fe278bfea8d004589c7a68a85
 location = @@LOCATION@@
-url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-9.8p1.tar.gz
+url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-9.9p2.tar.gz
 patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
 patches =

component/pygolang/test.cfg

@@ -5,6 +5,8 @@ extends =
     ../../stack/nxdtest.cfg
     ../pytest/buildout.cfg
     ../numpy/buildout.cfg
+#   pygolang tests need zodbpickle
+    ../ZODB/buildout.cfg
     buildout.cfg
 
 parts =

component/sqlite3/buildout.cfg

@@ -20,6 +20,11 @@ configure-options =
   --with-readline-inc=-I${readline:location}/include
 post-install =
   gcc -I%(location)s/include -fPIC -shared ext/misc/cksumvfs.c -o %(location)s/lib/cksumvfs.so
+  set %(location)s/bin/sqlite3-cksumvfs
+  cat <<TEMPLATE >$1
+  %(cksumvfs)s
+  TEMPLATE
+  chmod +x $1
 # Increase MAX_VARIABLE_NUMBER like many os. For example:
 # https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/sqlite
 # NEO needs SQLITE_ENABLE_UPDATE_DELETE_LIMIT to drop partitions.
@@ -27,3 +32,18 @@ environment =
   CPPFLAGS=-I${zlib:location}/include -DSQLITE_MAX_VARIABLE_NUMBER=250000 -DSQLITE_ENABLE_RTREE=1
   LDFLAGS=-Wl,-rpath=@@LOCATION@@/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
   PATH=${tcl:location}/bin:${xz-utils:location}/bin:%(PATH)s
+cksumvfs =
+  #!/bin/sh -e
+  if [ \$# = 0 ]; then
+    x=
+  else
+    [ "\$${1%%-*}" ]; [ -f "\$1" ] # options not supported
+    x=".open \\"\$(printf %s "\$1" |sed 's/[\\\\"]/\\\\\\0/g')\\""
+    shift
+  fi
+  y=/proc/self/fd/3
+  [ ! -e \$y ]
+  exec @@LOCATION@@/bin/sqlite3 -init \$y '' "\$@" 3<<EOF
+  .load cksumvfs.so
+  \$x
+  EOF

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.397'
+version = '1.0.401'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -166,6 +166,10 @@ setup(name=name,
           'zero-knowledge.read = slapos.recipe.zero_knowledge:ReadRecipe',
           'zero-knowledge.write = slapos.recipe.zero_knowledge:WriteRecipe'
         ],
+        'zc.buildout.uninstall': [
+          'publish_failsafe = slapos.recipe.publish:RecipeFailsafe.uninstall',
+          'publish.serialised_failsafe = slapos.recipe.publish:SerialisedFailsafe.uninstall',
+        ]
       },
       extras_require=extras_require,
       test_suite='slapos.test',

slapos/recipe/librecipe/__init__.py

@@ -264,7 +264,12 @@ class BaseSlapRecipe:
     self.request = self.computer_partition.request
     self.setConnectionDict = self.computer_partition.setConnectionDict
     self._createDefaultDirectoryStructure()
-    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+
+    # Lazy load of the instance parameter dict
+    # to prevent an http calls if not needed
+    @property
+    def parameter_dict(self):
+      return self.computer_partition.getInstanceParameterDict()
 
     # call children part of install
     path_list = self._install()

slapos/recipe/librecipe/genericslap.py

@@ -66,7 +66,12 @@ class GenericSlapRecipe(GenericBaseRecipe):
 
     self.request = self.computer_partition.request
     self.setConnectionDict = self.computer_partition.setConnectionDict
-    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+
+    # Lazy load of the instance parameter dict
+    # to prevent an http calls if not needed
+    @property
+    def parameter_dict(self):
+      return self.computer_partition.getInstanceParameterDict()
 
     # call children part of install
     path_list = self._install()

slapos/recipe/publish.py

@@ -71,9 +71,9 @@ class Serialised(Recipe):
 class Failsafe(object):
   def _setConnectionDict(self, publish_dict, slave_reference):
     error_status_file = self.options.get('-error-status-file')
-    if error_status_file:
-      self.return_list = [error_status_file]
-    else:
+    # Note: We can't put -error-status-file in return list as by default it is
+    #       not present, and buildout wants the section to have it, so it
+    #       Uninstalls/Installs the part instead of just Updating it
     self.return_list = []
     try:
       super(Failsafe, self)._setConnectionDict(publish_dict, slave_reference)
@@ -86,6 +86,19 @@ class Failsafe(object):
         if os.path.exists(error_status_file):
           os.unlink(error_status_file)
 
+  def update(self):
+    error_status_file = self.options.get('-error-status-file')
+    if error_status_file is not None:
+      if os.path.exists(error_status_file):
+        # last run failed, so need to reinstall
+        self.install()
+
+  def uninstall(name, options):
+    error_status_file = options.get('-error-status-file')
+    if error_status_file is not None:
+      if os.path.exists(error_status_file):
+        os.unlink(error_status_file)
+
 
 class RecipeFailsafe(Failsafe, Recipe):
   pass

software/gitlab/buildout.hash.cfg

@@ -58,7 +58,7 @@ md5sum = 35bb9f1d8f4fd6675bd768d8a7e1253c
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
-md5sum = 330e1b13f793303e069ec8b7b8c882ea
+md5sum = 550c96c4652b6d93b6b423b4f0c2ab19
 
 [macrolib.cfg.in]
 _update_hash_filename_ = macrolib.cfg.in

software/gitlab/gowork.cfg

@@ -26,7 +26,7 @@ revision      = v0.8.0-12-g816c908556
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/git-backup
 repository    = https://lab.nexedi.com/kirr/git-backup.git
-revision      = 3327aedfbe903b210366105b8a4f9f85a313a0a3
+revision      = 7b24f69c0d27686603ee5a10a3b7c475b8e6a4fb
 
 [go_lab.nexedi.com_kirr_go123]
 <= go-git-package

software/gitlab/instance-gitlab-export.cfg.in

@@ -46,6 +46,7 @@ input = inline: gitlab-shell-work*
   gitlab-work*
   var/log/**
   var/backup/**
+  var/gitlab/**
   var/repositories*
   var/repositories/**
   srv/postgresql/**
@@ -55,6 +56,7 @@ input = inline: gitlab-shell-work*
   etc/service/postgres-start
   srv/redis/**
   srv/puma/puma.socket
+  .cache/**
   .cache
 output = ${directory:srv}/exporter.exclude
 

software/kvm/README.rst

@@ -198,3 +198,21 @@ The failure observed to confirm the situation can be found in
   ValueError: external-disk problems: conflicts with external-disk-number = XX, conflicts with already configured disks amount XX in /srv/slapgrid/slappartNN/etc/.data-disk-amount
 
 Where ``XX`` is the previously used ``external-disk-number`` and ``NN`` is the partition.
+
+Fixing qmpbackup dirty bitmap
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It can happen that bin/exporter will fail with:
+
+CRITICAL - main: Error executing backup: Bitmap 'qmpbackup-virtio0-NNN' is inconsistent and cannot be used
+
+In such case it is required to:
+
+ * stop the kvm
+ * use qemu-img info virtual.qcow2 to find the bitmap value, it shall be qmpbackup-virtio0-NNN
+ * remove the bitmap value with qemu-img bitmap --remove virtual.qcow2 qmpbackup-virtio0-NNN
+ * start back the kvm
+ * re-rexecute the backup with bin/exporter
+ * remove FULL-*partial from backup destination
+
+Such situation might happen when more than one exporter is running in the same time.

software/neoppod/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = d8a62f4a2b68fe97146871d07a500443
+md5sum = 14322bc623dd84b27f3c0a4bf75bec3f
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-neo.cfg.in

@@ -88,6 +88,12 @@ command = mysql
 {%   endif -%}
 {%   set query_string = urllib.urlencode(extra_dict) -%}
 
+[{{ section('sqlite3-cksumvfs') }}]
+recipe = slapos.recipe.build
+location = ${directory:bin}/sqlite3
+source = {{sqlite3_location}}/bin/${:_buildout_section_name_}
+install = import os; os.symlink(options['source'], location)
+
 {% else -%}
 {% do assert(not storage_count) -%}
 

software/rapid-cdn/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 4a53b09ba4cbf15e32a5088cc81db5ea
 
 [profile-master]
 filename = instance-master.cfg.in
-md5sum = 653e8249ee71c3fa71b172fa3d2a4207
+md5sum = 78c556911944339f0c03c8416db8dbaa
 
 [profile-slave-list]
 filename = instance-slave-list.cfg.in
@@ -34,7 +34,7 @@ md5sum = 8d6d2280ad47b431c433495bf9809adc
 
 [profile-master-publish-slave-information]
 filename = instance-master-publish-slave-information.cfg.in
-md5sum = cba4d995962f7fbeae3f61c9372c4181
+md5sum = 91f4e406ef66fb6c3fac5d4af06ee0ca
 
 [template-frontend-haproxy-configuration]
 _update_hash_filename_ = templates/frontend-haproxy.cfg.in

software/rapid-cdn/instance-master-publish-slave-information.cfg.in

-{% set part_list = [] %}
-{% set slave_information_dict = {} %}
+{%- set part_list = [] %}
+{%- set slave_information_dict = {} %}
 
 # regroup slave information from all frontends
-{% for frontend, slave_list_raw in slave_information.items() %}
-{%   if slave_list_raw %}
-{%     set slave_list = json_module.loads(slave_list_raw) %}
-{%   else %}
-{%    set slave_list = [] %}
-{%   endif %}
-{%   for slave_dict in slave_list %}
-{%     set slave_reference = slave_dict.pop('slave-reference') %}
-{%     set log_access_url = slave_dict.pop('log-access', '') %}
-{%     set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
-{%     do current_slave_dict.update(slave_dict) %}
-{%     set log_access_list = current_slave_dict.get('log-access-urls', []) %}
-{%     do log_access_list.append( frontend + ': ' + log_access_url) %}
-{%     do current_slave_dict.__setitem__(
+{%- for frontend, slave_list_raw in slave_information.items() %}
+{%-   if slave_list_raw %}
+{%-     set slave_list = json_module.loads(slave_list_raw) %}
+{%-   else %}
+{%-    set slave_list = [] %}
+{%-   endif %}
+{%-   for slave_dict in slave_list %}
+{%-     set slave_reference = slave_dict.pop('slave-reference') %}
+{%-     set log_access_url = slave_dict.pop('log-access', '') %}
+{%-     set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
+{%-     do current_slave_dict.update(slave_dict) %}
+{%-     set log_access_list = current_slave_dict.get('log-access-urls', []) %}
+{%-     do log_access_list.append( frontend + ': ' + log_access_url) %}
+{%-     do current_slave_dict.__setitem__(
             'log-access-urls',
             log_access_list
             ) %}
-{%     do current_slave_dict.__setitem__(
+{%-     do current_slave_dict.__setitem__(
             'replication_number',
             current_slave_dict.get('replication_number', 0) + 1
             ) %}
-{%     do slave_information_dict.__setitem__(slave_reference, current_slave_dict) %}
-{%   endfor %}
-{% endfor %}
+{%-     do slave_information_dict.__setitem__(slave_reference, current_slave_dict) %}
+{%-   endfor %}
+{%- endfor %}
 
-{% for slave_reference, rejected_info_list in rejected_slave_information['rejected-slave-dict'].items() %}
-{%   if slave_reference not in slave_information_dict %}
-{%     do slave_information_dict.__setitem__(slave_reference, {}) %}
-{%   endif %}
-{%   do slave_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %}
-{% endfor %}
+{%- for slave_reference, rejected_info_list in rejected_slave_information['rejected-slave-dict'].items() %}
+{%-   if slave_reference not in slave_information_dict %}
+{%-     do slave_information_dict.__setitem__(slave_reference, {}) %}
+{%-   endif %}
+{%-   do slave_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %}
+{%- endfor %}
 
-{% for slave_reference, warning_info_list in warning_slave_information['warning-slave-dict'].items() %}
-{%   if slave_reference not in slave_information_dict %}
-{%     do slave_information_dict.__setitem__(slave_reference, {}) %}
-{%   endif %}
-{%   do slave_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %}
-{% endfor %}
+{%- for slave_reference, warning_info_list in warning_slave_information['warning-slave-dict'].items() %}
+{%-   if slave_reference not in slave_information_dict %}
+{%-     do slave_information_dict.__setitem__(slave_reference, {}) %}
+{%-   endif %}
+{%-   do slave_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %}
+{%- endfor %}
 
-{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).items() %}
-{%   if slave_reference not in rejected_slave_information['rejected-slave-dict'] %}
-{%     if slave_reference not in slave_information_dict %}
-{%       do slave_information_dict.__setitem__(slave_reference, {}) %}
-{%     endif %}
-{%     do slave_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
-{%     do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
-{%     do slave_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
-{%   endif %}
-{% endfor %}
+{%- for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).items() %}
+{%-   if slave_reference not in rejected_slave_information['rejected-slave-dict'] %}
+{%-     if slave_reference not in slave_information_dict %}
+{%-       do slave_information_dict.__setitem__(slave_reference, {}) %}
+{%-     endif %}
+{%-     do slave_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
+{%-     do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
+{%-     do slave_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
+{%-   endif %}
+{%- endfor %}
 
 [common-frontend-information]
-{% for frontend_key, frontend_value in frontend_information.items() %}
+{%- for frontend_key, frontend_value in frontend_information.items() %}
 {{ frontend_key }} = {{ frontend_value }}
-{% endfor %}
+{%- endfor %}
 
 # Publish information for each slave
-{% set active_slave_instance_list = json_module.loads(active_slave_instance_dict['active-slave-instance-list']) %}
-{% for slave_reference, slave_information in slave_information_dict.items() %}
+{%- set active_slave_instance_list = json_module.loads(active_slave_instance_dict['active-slave-instance-list']) %}
+{%- for slave_reference, slave_information in slave_information_dict.items() %}
 {#   Filter out destroyed, so not existing anymore, slaves #}
 {#   Note: This functionality is not yet covered by tests, please modify with care #}
-{%   if slave_reference in active_slave_instance_list %}
-{%     set publish_section_title = 'publish-%s' % slave_reference %}
-{%     do part_list.append(publish_section_title) %}
+{%-   if slave_reference in active_slave_instance_list %}
+{%-     set publish_section_title = 'publish-%s' % slave_reference %}
+{%-     do part_list.append(publish_section_title) %}
 [{{ publish_section_title }}]
 <= common-frontend-information
 recipe = slapos.cookbook:publish_failsafe
 -error-status-file = {{ publish_failsafe_error }}/{{ slave_reference }}-error.status
 -slave-reference = {{ slave_reference }}
-{%     set log_access_url = slave_information.pop('log-access-urls', None) %}
-{%     if log_access_url %}
+{%-     set log_access_url = slave_information.pop('log-access-urls', None) %}
+{%-     if log_access_url %}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
 log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
-{%     endif %}
-{%     for key, value in slave_information.items() %}
+{%-     endif %}
+{%-     for key, value in slave_information.items() %}
 {{ key }} = {{ dumps(value) }}
-{%     endfor %}
-{%   endif %}
-{% endfor %}
+{%-     endfor %}
+{%-   endif %}
+{%- endfor %}
 
 [buildout]
 extends = {{ profile_common }}
 parts =
 {% for part in part_list %}
-{{ '  %s' % part }}
+{{- '  %s' % part }}
 {% endfor %}

software/rapidspace-vps/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = eb69f3c2a26948eb7e413a891589608d
 
 [software.json]
 filename = software.cfg.json
-md5sum = 701a7f6e72c1effb2c9b3edacb971851
+md5sum = 41ffc9987429f3d1751c7a63a7276ad8
 
 [instance.json]
 filename = ../kvm/boot-image-input-schema.json

stack/slapos.cfg

@@ -417,7 +417,7 @@ ZConfig = 3.6.1
 zdaemon = 4.2.0
 zeroconf = 0.62.0:whl
 zipp = 3.12.0:whl
-zodburi = 2.5.0
+zodburi = 3.0.0
 zope.event = 5.0
 zope.exceptions = 5.2
 zope.interface = 7.1.1
@@ -501,6 +501,7 @@ websocket-client = 0.59.0
 Werkzeug = 1.0.1
 widgetsnbextension = 2.0.0
 zipp = 1.2.0:whl
+zodburi = 2.5.0
 zope.event = 4.6.0
 zope.exceptions = 4.6
 zope.interface = 5.5.2