Update Release Candidate

software/cython-test/software.cfg → component/cythonplus/buildout.cfg

 [buildout]
 extends =
-  ../../stack/slapos.cfg
-  ../../component/numpy/buildout.cfg
+  ../defaults.cfg
+  ../git/buildout.cfg
+  ../numpy/buildout.cfg
 
-parts =
-  slapos-cookbook
-  template
+parts = cythonplus_env.sh
+
+[gcc]
+min_version = 8.4
 
 [python]
 part = python3
 
-[template]
-recipe = slapos.recipe.template:jinja2
-rendered = ${buildout:directory}/template.cfg
-template =
-  inline:[buildout]
-  eggs-directory = ${buildout:eggs-directory}
-  develop-eggs-directory = ${buildout:develop-eggs-directory}
-  parts = runTestSuite
-
-  [slap-configuration]
-  recipe = slapos.cookbook:slapconfiguration.serialised
-  computer = $${slap-connection:computer-id}
-  partition = $${slap-connection:partition-id}
-  url = $${slap-connection:server-url}
-  key = $${slap-connection:key-file}
-  cert = $${slap-connection:cert-file}
-
-  [directory]
-  recipe = slapos.cookbook:mkdirectory
-  bin = $${buildout:directory}/bin
-  tmp = $${buildout:directory}/tmp
-
-  [runTestSuite]
-  recipe = slapos.recipe.template:jinja2
-  rendered = $${directory:bin}/$${:_buildout_section_name_}
-  template = ${runTestSuite.in:target}
-  mode = 0755
-  context =
-    key tmpdir directory:tmp
-    key slapparameter_dict slap-configuration:configuration
-    key cython_repository cython-repository:location
-    raw runTestSuite_py ${runTestSuite_py:bin-directory}/${runTestSuite_py:interpreter}
-    raw cython_env_sh ${cython_env.sh:rendered}
-
-  [cython-repository]
-  recipe = slapos.recipe.build:gitclone
-  repository = ${cython-repository:location}
-  git-executable = ${git:location}/bin/git
-  shared = true
-
-[cython-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/cython.git
-git-executable = ${git:location}/bin/git
-sparse-checkout = /.gitignore
-
-[runTestSuite.in]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_buildout_section_name_}
-md5sum = 02094e80cde9631081077fc96b401065
-
-[runTestSuite_py]
-recipe = zc.recipe.egg
-eggs = erp5.util
-interpreter = ${:_buildout_section_name_}
-
+# Dependencies for the Cython+ test suite
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -73,7 +20,13 @@ eggs =
   coverage
   pycodestyle
 
-[cython_env.sh]
+[cythonplus-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/cython.git
+revision = cythonplus-0.1
+git-executable = ${git:location}/bin/git
+
+[cythonplus_env.sh]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/${:_buildout_section_name_}
 template =
@@ -81,25 +34,25 @@ template =
   {% if 'part' in gcc -%}
   {%   set path = path + ':' + gcc.prefix + '/bin' -%}
   {% endif -%}
-  export PATH={{ path }}:$PATH
+  export PATH={{ path }}$${PATH:+:$PATH}
   export PYTHON={{ python }}
+  export PYTHONPATH={{ cythonplus_repository }}$${PYTHONPATH:+:$PYTHONPATH}
   export PYTHONPATH={{ ':'.join(easy_install.working_set(eggs['eggs'].split(), [
       eggs['develop-eggs-directory'],
       eggs['eggs-directory'],
     ]).entries) }}$${PYTHONPATH:+:$PYTHONPATH}
-  # EmbedTest needs libintl.
+  {# Set path to libintl needed for cython EmbedTest #}
   export LD_RUN_PATH={{ gettext }}/lib$${LD_RUN_PATH:+:$LD_RUN_PATH}
   export LIBRARY_PATH={{ gettext }}/lib$${LIBRARY_PATH:+:$LIBRARY_PATH}
-  {##}
 context =
   section eggs eggs
   section gcc gcc
+  key cythonplus_repository cythonplus-repository:location
   key gettext gettext:location
   key python python:executable
   import os os
   import easy_install zc.buildout.easy_install
 
 [versions]
-slapos.recipe.template = 4.4
 coverage = 4.5.1
 pycodestyle = 2.5.0

component/garbage-collector/buildout.cfg

@@ -7,8 +7,8 @@ parts = garbage-collector
 [garbage-collector]
 recipe = slapos.recipe.cmmi
 shared = true
-md5sum = 6f77f9fff5fb5bf96adfc1e93cd035b6
-url = http://www.hboehm.info/gc/gc_source/gc-7.2g.tar.gz
+md5sum = 67a5093e2f9f381bd550aa891d00b54b
+url = http://www.hboehm.info/gc/gc_source/gc-8.0.4.tar.gz
 configure-options =
   --enable-cplusplus
   --disable-gcj-support

component/gcc/buildout.cfg

@@ -71,6 +71,11 @@ patches =
 version = 8.4.0
 md5sum = bb815a8e3b7be43c4a26fa89dbbd9795
 
+[gcc-10.2]
+<= gcc-common
+version = 10.2.0
+md5sum = e9fd9b1789155ad09bcf3ae747596b50
+
 [gcc-minimal]
 <= gcc-5.5
 configure-options =

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.17.tar.gz
-md5sum = 786a967c73cc1455c938d42fbe333bfe
+url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.20.tar.gz
+md5sum = 9f85ea9e6fd7d49a11cdc4c6269e10dd
 configure-command = true
 # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
 # otherwise use "generic".

component/jupyter-py2/README.md

+This directory is only temporary (until ERP5 is Python3) and will not actively
+be maintained.
+
+It has to be removed once ERP5 uses `software/jupyter` or do not use jupyter at
+all.

component/jupyter-py2/buildout.cfg

+[buildout]
+extends = 
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../openssl/buildout.cfg
+  ../jupyter/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+parts +=
+  slapos-cookbook
+  jupyter
+  jupyter-notebook-initialized-scripts
+  instance-jupyter-notebook
+
+[gcc]
+# Always build GCC for Fortran (see openblas).
+max_version = 0
+
+[jupyter]
+python_executable = ${buildout:bin-directory}/${:interpreter}
+
+[download-file-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+mode = 0644
+
+[jupyter-notebook-config]
+<= download-file-base
+
+[jupyter-set-password]
+<= download-file-base
+
+[erp5-kernel]
+<= download-file-base
+
+[kernel-json]
+<= download-file-base
+
+[custom-js]
+<= download-file-base
+
+[instance-jupyter-notebook]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/template.cfg
+mode = 0644
+context =
+  key bin_directory buildout:bin-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key openssl_output openssl-output:openssl
+  key python_executable jupyter:python_executable
+  key jupyter_config_location jupyter-notebook-config:location
+  key jupyter_config_filename jupyter-notebook-config:filename
+  key jupyter_set_password_location jupyter-set-password:location
+  key jupyter_set_password_filename jupyter-set-password:filename
+  key erp5_kernel_location erp5-kernel:location
+  key erp5_kernel_filename erp5-kernel:filename
+  key kernel_json_location kernel-json:location
+  key kernel_json_filename kernel-json:filename
+  key custom_js_location custom-js:location
+  key custom_js_filename custom-js:filename
+  key monitor_template_rendered buildout:directory
+
+[versions]
+Pygments = 2.2.0
+astor = 0.5
+backports-abc = 0.5
+backports.functools-lru-cache = 1.6.1
+backports.shutil-get-terminal-size = 1.0.0
+cycler = 0.10.0
+ipykernel = 4.5.2
+ipython = 5.3.0
+ipython-genutils = 0.1.0
+ipywidgets = 6.0.0
+jupyter-client = 5.0.0
+jupyter-core = 4.3.0
+jupyterlab = 0.26.3
+jupyterlab-launcher = 0.3.1
+matplotlib = 2.1.2
+mistune = 0.7.3
+nbformat = 4.3.0
+notebook = 4.4.1
+pandas = 0.19.2
+plone.recipe.command = 1.1
+prompt-toolkit = 1.0.13
+ptyprocess = 0.5.1
+pyzmq = 16.0.2
+scikit-learn = 0.18.1
+seaborn = 0.7.1
+simplegeneric = 0.8.1
+statsmodels = 0.8.0
+terminado = 0.6
+tornado = 4.4.2
+traitlets = 4.3.2
+widgetsnbextension = 2.0.0
+# numpy >= 1.13.1 is required for numpy.core.multiarray
+numpy = 1.13.1
+
+
+# Required by:
+# tornado==4.4.2
+certifi = 2020.6.20
+
+# Required by:
+# notebook==4.3.2
+# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
+nbconvert = 4.1.0
+
+# Required by:
+# ipython==5.3.0
+pathlib2 = 2.2.1
+
+# Required by:
+# statsmodels==0.8.0
+patsy = 0.4.1
+
+# Required by:
+# ipython==5.3.0
+pexpect = 4.2.1
+
+# Required by:
+# ipython==5.3.0
+pickleshare = 0.7.4
+
+# Required by:
+# matplotlib==2.1.2
+# pandas==0.19.2
+python-dateutil = 2.6.0
+
+# Required by:
+# pathlib2==2.2.1
+scandir = 1.5
+
+# Required by:
+# statsmodels==0.8.0
+scipy = 0.19.0
+
+# Required by:
+# tornado==4.4.2
+singledispatch = 3.4.0.3
+
+# Required by:
+# prompt-toolkit==1.0.13
+wcwidth = 0.1.7
+
+jupyter = 1.0.0
+jupyter-console = 5.1.0
+
+# Required by:
+# jupyter==1.0.0
+qtconsole = 4.3.0
+
+et-xmlfile = 1.0.1
+h5py = 2.7.1
+mpmath = 1.0.0
+openpyxl = 2.5.2
+sympy = 1.1.1
+xlrd = 1.1.0
+
+# Required by:
+# openpyxl==2.5.2
+jdcal = 1.4

component/jupyter-py2/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-jupyter-notebook]
+filename = instance.cfg.in
+md5sum = 1d5fe6cc4e48672ae7be1c223794a932
+
+[jupyter-notebook-config]
+filename = jupyter_notebook_config.py.jinja
+md5sum = 720e90a829c63371696bc3009917a743
+
+[jupyter-set-password]
+filename = jupyter_set_password.cgi.jinja
+md5sum = ac10fbcf790bd8e58750cfdd069812d2
+
+[erp5-kernel]
+filename = ERP5kernel.py
+md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+
+[kernel-json]
+filename = kernel.json.jinja
+md5sum = 33547be93a67530165e079dc3ecfdac3
+
+[custom-js]
+filename = custom.js
+md5sum = 147ccce38f7d7cf10664975d0dd80e33

component/jupyter-py2/custom.js

+// leave at least 2 line with only a star on it below, or doc generation fails
+/**
+ *
+ *
+ * Placeholder for custom user javascript
+ * mainly to be overridden in profile/static/custom/custom.js
+ * This will always be an empty file in IPython
+ *
+ * User could add any javascript in the `profile/static/custom/custom.js` file.
+ * It will be executed by the ipython notebook at load time.
+ *
+ * Same thing with `profile/static/custom/custom.css` to inject custom css into the notebook.
+ *
+ *
+ * The object available at load time depend on the version of IPython in use.
+ * there is no guaranties of API stability.
+ *
+ * The example below explain the principle, and might not be valid.
+ *
+ * Instances are created after the loading of this file and might need to be accessed using events:
+ *     define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *     ], function(IPython, events) {
+ *         events.on("app_initialized.NotebookApp", function () {
+ *             IPython.keyboard_manager....
+ *         });
+ *     });
+ *
+ * __Example 1:__
+ *
+ * Create a custom button in toolbar that execute `%qtconsole` in kernel
+ * and hence open a qtconsole attached to the same kernel as the current notebook
+ *
+ *    define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *    ], function(IPython, events) {
+ *        events.on('app_initialized.NotebookApp', function(){
+ *            IPython.toolbar.add_buttons_group([
+ *                {
+ *                    'label'   : 'run qtconsole',
+ *                    'icon'    : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
+ *                    'callback': function () {
+ *                        IPython.notebook.kernel.execute('%qtconsole')
+ *                    }
+ *                }
+ *                // add more button here if needed.
+ *                ]);
+ *        });
+ *    });
+ *
+ * __Example 2:__
+ *
+ * At the completion of the dashboard loading, load an unofficial javascript extension
+ * that is installed in profile/static/custom/
+ *
+ *    define([
+ *        'base/js/events'
+ *    ], function(events) {
+ *        events.on('app_initialized.DashboardApp', function(){
+ *            require(['custom/unofficial_extension.js'])
+ *        });
+ *    });
+ *
+ * __Example 3:__
+ *
+ *  Use `jQuery.getScript(url [, success(script, textStatus, jqXHR)] );`
+ *  to load custom script into the notebook.
+ *
+ *    // to load the metadata ui extension example.
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/example.js');
+ *    // or
+ *    // to load the metadata ui extension to control slideshow mode / reveal js for nbconvert
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/slideshow.js');
+ *
+ *
+ * @module IPython
+ * @namespace IPython
+ * @class customjs
+ * @static
+ */
+
+$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
+  var kernelname = Jupyter.notebook.kernel_selector.current_selection;
+  var display_text="<div class='output_subarea output_text output_result'>\
+  <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
+  1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
+  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
+  It would be better to set the reference to match with erp5 reference pattern.</br>\
+  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+  <p><u>OTHER USEFUL MAGICS</u> -</br>\
+  <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+  <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
+  NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
+  so changes to them would be disregarded and would throw an error.</br>\
+  <p><u>About classes, functions and global state on modules:</u></p>\
+  Your code is going to be executed by ERP5, which can have many nodes </br>\
+  and there is no guarantee that your code is always going to be executed by the same server.</br>\
+  This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
+  won't be available across nodes. To solve this issue, you need to use a special object </br>\
+  called 'environment' to store your global setup. This object was designed to hold global </br>\
+  state and restore it for each code cell. Example:</br></br>\
+  <b>def my_setup():</br>\
+     # import modules, define functions and classes</br>\
+     # and set global state on modules</br>\
+     # return dict of variables to be available in code cells</br>\
+     {'my_var': 1}</br>\
+  environment.define(my_setup, 'my custom setup')</b></br></br>\
+  After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
+  following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
+  <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
+  because they will be executed in every code cell and can cause a substantial slow down.\
+  </pre></div>";
+  if (kernelname=="erp5"){
+  $('div#notebook-container').prepend(display_text);
+  }
+  });

component/jupyter-py2/instance.cfg.in

+[buildout]
+parts =
+  instance
+  jupyter_notebook
+  read-knowledge0
+  publish-connection-parameter
+  jupyter-notebook-config
+  erp5-kernel
+  kernel-json
+  custom-js
+  monitor-base
+extends =
+  {{ monitor_template_rendered }}/template-monitor.cfg
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
+[instance-parameter]
+port = 8888
+host = ${slapconfiguration:ipv6-random}
+cert_file = ${generate-certificate:cert_file}
+key_file = ${generate-certificate:key_file}
+logfile = ${directory:log}/jupyter_notebook.log
+notebook_dir = ${directory:notebook_dir}
+
+[dynamic-jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 0644
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${instance-parameter:key_file} ]
+  then
+    {{ openssl_output }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
+      -out ${instance-parameter:cert_file}
+  fi
+update-command = ${:command}
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ bin_directory }}/jupyter-lab
+  --no-browser
+  --ip=${instance-parameter:host}
+  --port=${instance-parameter:port}
+  --port-retries=0
+  --certfile=${instance-parameter:cert_file}
+  --keyfile=${instance-parameter:key_file}
+  --notebook-dir=${instance-parameter:notebook_dir}
+  --log-level="DEBUG"
+wrapper-path = ${directory:service}/jupyter-lab
+environment =
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
+  LANG=C.UTF-8
+
+[jupyter-notebook-config]
+<= dynamic-jinja2-template-base
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
+mode = 0744
+context =
+  raw config_cfg ${buildout:directory}/knowledge0.cfg
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+notebook_dir = ${:var}/notebooks
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[jupyter_notebook]
+# This part is called like this because knowledge0.write uses the part name for
+# the section name in the config file.
+recipe = slapos.cookbook:zero-knowledge.write
+password = 
+filename = knowledge0.cfg
+
+[read-knowledge0]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = knowledge0.cfg
+password =
+
+[monitor-instance-parameter]
+monitor-base-url = ${monitor-frontend-promise:url}
+# In case you're using a developer instance you should edit these in:
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
+# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
+instance-configuration =
+  raw jupyter-password ${read-knowledge0:password}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
+url = ${:jupyter-classic-url}
+jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
+
+[erp5-kernel]
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js

component/jupyter-py2/jupyter_notebook_config.py.jinja

+'''
+  This script initializes Jupyter's configuration such as passwords and other
+  things. It is run by IPython hence why it can use functions like get_config().
+'''
+import ConfigParser
+import random
+from notebook.auth import passwd
+import os
+
+def random_password(length = 10):
+  result = ""
+  for i in range(0, length):
+    result = result + chr(random.randint(0, 25) + ord('a'))
+  return result
+
+knowledge_0 = '{{ config_cfg }}'
+
+if not os.path.exists(knowledge_0):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+c = get_config()
+
+parser = ConfigParser.ConfigParser()
+parser.read(knowledge_0)
+
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
+
+if not parser.has_option("jupyter_notebook", "password") or \
+  parser.get("jupyter_notebook", "password") == "":
+  parser.set("jupyter_notebook", "password", random_password())
+
+c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
+
+with open(knowledge_0, 'w') as file:
+  parser.write(file)

software/jupyter/jupyter_set_password.cgi.jinja → component/jupyter-py2/jupyter_set_password.cgi.jinja

@@ -61,4 +61,3 @@ print """<div class="pure-control-group">
 <script type="text/javascript" src="static/monitor-register.js"></script>
 </body></html>
 """
-

component/jupyter-py2/kernel.json.jinja

+{
+  "argv": [
+    "{{python_executable}}",
+    "{{kernel_dir}}",
+    "{{erp5_url}}",
+    "-f",
+    "{connection_file}"
+  ],
+  "display_name": "{{display_name}}",
+  "language": "{{language_name}}",
+  "language_info": {"name": "python"}
+}

component/logrotate/buildout.cfg

@@ -8,8 +8,8 @@ parts = logrotate
 [logrotate]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://github.com/logrotate/logrotate/releases/download/3.15.0/logrotate-3.15.0.tar.xz
-md5sum = 320046f0b9fc38337e8827d4c5a866a0
+url = https://github.com/logrotate/logrotate/releases/download/3.17.0/logrotate-3.17.0.tar.xz
+md5sum = ac2a7151fc8a187201872358a20a2813
 # BBB this is only for backward-compatibility.
 post-install =
   ln -nsf . @@LOCATION@@/usr

component/logrotate/logrotate-3.7.9-O_CLOEXEC.optional.patch

-diff --git a/config.c b/config.c
-index e6d5d1d..dd004a9 100644
---- a/config.c
-+++ b/config.c
-@@ -519,7 +519,11 @@ static int readConfigFile(const char *configFile, struct logInfo *defConfig)
-        length arrays -- of course, if we aren't run setuid it doesn't
-        matter much */
- 
-+#ifdef O_CLOEXEC
-     fd = open(configFile, O_RDONLY | O_CLOEXEC);
-+#else
-+    fd = open(configFile, O_RDONLY);
-+#endif
-     if (fd < 0) {
- 	message(MESS_ERROR, "failed to open config file %s: %s\n",
- 		configFile, strerror(errno));

component/openssl/buildout.cfg

@@ -17,8 +17,8 @@ parts =
 [openssl]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.openssl.org/source/openssl-1.1.1g.tar.gz
-md5sum = 76766e98997660138cdaf13a187bd234
+url = https://www.openssl.org/source/openssl-1.1.1i.tar.gz
+md5sum = 08987c3cf125202e2b0840035efb392c
 location = @@LOCATION@@
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with DESTDIR). Used by slapos.package.git/obs

component/postfix/buildout.cfg

@@ -8,19 +8,19 @@ extends =
   ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../cyrus-sasl/buildout.cfg
+  ../m4/buildout.cfg
 
 [postfix]
 recipe = slapos.recipe.cmmi
 shared = true
-url = ftp://ftp.porcupine.org/mirrors/postfix-release/official/postfix-2.11.6.tar.gz
-md5sum = c3277d05b78eaaf5955406bc7b6d2b9f
+url = ftp://ftp.porcupine.org/mirrors/postfix-release/official/postfix-3.5.8.tar.gz
+md5sum = c7c55ccc1db2a30d35c3867c21fe7109
 location = @@LOCATION@@
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/noroot.patch#738bcc97b8044c45b58708bdf3a84b8e
-  ${:_profile_base_location_}/skip-libdb-check.patch#f7fdbd8787874b535fee548b0139c0d8
+  ${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
 configure-command = make
 configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl-1.0:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl-1.0:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
 make-targets = non-interactive-package install_root=${:location}
 environment =
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s

component/postfix/noroot.patch

@@ -2,16 +2,16 @@ diff --git a/src/global/mail_params.c b/src/global/mail_params.c
 index 2d91977..0f06298 100644
 --- a/src/global/mail_params.c
 +++ b/src/global/mail_params.c
-@@ -721,7 +721,9 @@ void    mail_params_init()
+@@ -911,7 +911,9 @@ void    mail_params_init()
      check_default_privs();
      check_mail_owner();
      check_sgid_group();
 +    /*
      check_overlap();
 +    */
- #ifdef HAS_DB
      dict_db_cache_size = var_db_read_buf;
- #endif
+     dict_lmdb_map_size = var_lmdb_map_size;
+     inet_windowsize = var_inet_windowsize;
 diff --git a/src/master/master.c b/src/master/master.c
 index a9d5d1b..db88c55 100644
 --- a/src/master/master.c
@@ -29,12 +29,13 @@ index a9d5d1b..db88c55 100644
  
      /*
       * Process JCL.
-@@ -392,8 +392,10 @@ int     main(int argc, char **argv)
+@@ -433,9 +433,11 @@ int     main(int argc, char **argv)
       * all MTA processes cleanly. Give up if we can't separate from our
       * parent process. We're not supposed to blow away the parent.
       */
 +    /*
-     if (debug_me == 0 && master_detach != 0 && setsid() == -1 && getsid(0) != getpid())
+     if (init_mode == 0 && debug_me == 0 && master_detach != 0
+ 	&& setsid() == -1 && getsid(0) != getpid())
  	msg_fatal("unable to set session and process group ID: %m");
 +    */
  

component/postfix/skip-libdb-check.patch

-diff --git a/makedefs b/makedefs
-index dd5f256..e90880e 100644
---- a/makedefs
-+++ b/makedefs
-@@ -299,13 +299,13 @@ case "$SYSTEM.$RELEASE" in
- 		    elif [ -f /usr/include/db/db.h ]
- 		    then
- 			CCARGS="$CCARGS -I/usr/include/db"
--		    else
-+		    #else
- 			# No, we're not going to try db1 db2 db3 etc.
- 			# On a properly installed system, Postfix builds
- 			# by including <db.h> and by linking with -ldb
--			echo "No <db.h> include file found." 1>&2
--			echo "Install the appropriate db*-devel package first." 1>&2
--			exit 1
-+			#echo "No <db.h> include file found." 1>&2
-+			#echo "Install the appropriate db*-devel package first." 1>&2
-+			#exit 1
- 		    fi
- 		    SYSLIBS="-ldb"
- 		    ;;
-@@ -372,12 +372,12 @@ EOF
- 		    elif [ -f /usr/include/db/db.h ]
- 		    then
- 			CCARGS="$CCARGS -I/usr/include/db"
--		    else
-+		    #else
- 			# On a properly installed system, Postfix builds
- 			# by including <db.h> and by linking with -ldb
--			echo "No <db.h> include file found." 1>&2
--			echo "Install the appropriate db*-devel package first." 1>&2
--			exit 1
-+			#echo "No <db.h> include file found." 1>&2
-+			#echo "Install the appropriate db*-devel package first." 1>&2
-+			#exit 1
- 		    fi
- 		    SYSLIBS="-ldb"
- 		    ;;
-@@ -403,12 +403,12 @@ EOF
- 		    elif [ -f /usr/include/db/db.h ]
- 		    then
- 			CCARGS="$CCARGS -I/usr/include/db"
--		    else
-+		    #else
- 			# On a properly installed system, Postfix builds
- 			# by including <db.h> and by linking with -ldb
--			echo "No <db.h> include file found." 1>&2
--			echo "Install the appropriate db*-devel package first." 1>&2
--			exit 1
-+			#echo "No <db.h> include file found." 1>&2
-+			#echo "Install the appropriate db*-devel package first." 1>&2
-+			#exit 1
- 		    fi
- 		    SYSLIBS="-ldb"
- 		    ;;

component/socat/buildout.cfg

@@ -7,6 +7,7 @@ parts =
 
 [socat]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz
 version = 1.7.3.2
 md5sum = aec3154f7854580cfab0c2d81e910519

component/trafficserver/buildout.cfg

@@ -6,6 +6,7 @@ extends =
   ../make/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
+  ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
@@ -22,9 +23,12 @@ min_version = 8
 
 [trafficserver]
 recipe = slapos.recipe.cmmi
-url = http://apache.claz.org/trafficserver/trafficserver-8.1.0.tar.bz2
-md5sum = 99bfeb61095e55cb151ef58d884cb3f1
+url = http://apache.claz.org/trafficserver/trafficserver-8.1.1.tar.bz2
+md5sum = 4f4d1e7de19c77157be0c2a825b31026
 shared = true
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/trafficserver-8.1-stale-negative-cache-not-returnable.patch#e1a2f8a23f00cee1301ccf1a84e46763
 configure-options =
   --with-openssl=${openssl:location}
   --with-pcre=${pcre:location}
@@ -37,7 +41,7 @@ configure-options =
   --enable-experimental-plugins
   --disable-posix-cap
 environment =
-  PATH=${libtool:location}/bin:${make:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+  PATH=${libtool:location}/bin:${make:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 
 make-target =

component/trafficserver/trafficserver-8.1-stale-negative-cache-not-returnable.patch

+--- trafficserver-8.1.1/proxy/http/HttpTransact.cc.orig	2020-12-01 00:30:26.000000000 +0100
++++ trafficserver-8.1.1/proxy/http/HttpTransact.cc	2021-01-11 11:35:41.946893735 +0100
+@@ -5752,6 +5752,7 @@
+ HttpTransact::is_stale_cache_response_returnable(State *s)
+ {
+   HTTPHdr *cached_response = s->cache_info.object_read->response_get();
++  HTTPStatus cache_response_code = cached_response->status_get();
+ 
+   // First check if client allows cached response
+   // Note does_client_permit_lookup was set to
+@@ -5760,6 +5761,12 @@
+   if (!s->cache_info.directives.does_client_permit_lookup) {
+     return false;
+   }
++  // We don't serve stale negative cache.
++  if (cache_response_code == HTTP_STATUS_INTERNAL_SERVER_ERROR || cache_response_code == HTTP_STATUS_GATEWAY_TIMEOUT ||
++      cache_response_code == HTTP_STATUS_BAD_GATEWAY || cache_response_code == HTTP_STATUS_SERVICE_UNAVAILABLE) {
++    TxnDebug("http_trans", "[is_stale_cache_response_returnable] stale negative cache");
++    return false;
++  }
+   // Spec says that we can not serve a stale document with a
+   //   "must-revalidate header"
+   // How about "s-maxage" and "no-cache" directives?

component/userhosts/buildout.cfg

@@ -2,7 +2,8 @@
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://lab.nexedi.com/nexedi/userhosts/repository/${:revision}/archive.tar.gz
-revision = 1d3b463e7856db6e674a06258c0840206e6a7b72
+revision = a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a
+md5sum = 5a80b4d962d975f290a60cf790c3334d
 configure-command = true
 make-options = PREFIX=@@LOCATION@@
 make-targets = check install

component/wendelin.core/buildout.cfg

@@ -10,19 +10,10 @@ parts =
   wendelin.core
 
 
-# wendelin.core installed from released egg from pypi
+# wendelin.core is installed from git checkout
 [wendelin.core]
-recipe = zc.recipe.egg:custom
-<= wendelin.core-eggcommon
-
-# wendelin.core installed from latest git version
-[wendelin.core-dev]
 recipe = zc.recipe.egg:develop
 setup = ${wendelin.core-repository:location}
-<= wendelin.core-eggcommon
-
-# common parts in between wendelin.core and wendelin.core-dev
-[wendelin.core-eggcommon]
 egg = wendelin.core
 setup-eggs =
     ${pygolang:egg}[pyx.build]
@@ -41,6 +32,8 @@ PATH = ${git:location}/bin:%(PATH)s
 [wendelin.core-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/wendelin.core.git
-# dir is pretty name as top-level -dev recipe
-location = ${buildout:parts-directory}/wendelin.core-dev
+branch   = master
+revision = v0.13-0-gb26ba55
+# dir is pretty name as top-level recipe
+location = ${buildout:parts-directory}/wendelin.core
 git-executable = ${git:location}/bin/git

component/wendelin.core/test-common.cfg

@@ -15,7 +15,7 @@ parts =
 # keep neoppod first and in parts so that ZODB is built correctly
     neoppod
 
-    wendelin.core-dev
+    wendelin.core
 
 #   for instance
     wendelin.core-python

slapos/recipe/README.generic_varnish.rst

@@ -7,7 +7,7 @@ set up.
 How to Use generic_varnish ?
 ============================
 
-On slap console, you can instanciate varnish like this::
+On slap console, you can instantiate varnish like this::
   
   instance = request(
     software_type='varnish',

software/agent/software.cfg

@@ -48,5 +48,4 @@ gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 4.4
 smmap = 0.9.0

software/ajaxplorer/software.cfg

@@ -84,5 +84,4 @@ numpy = 1.16.4
 # websockify 0.4.1 doesn't install well
 websockify = 0.3.0
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.4.2
 z3c.recipe.mkdir = 0.5

software/backupserver/software.cfg

@@ -105,5 +105,4 @@ mode = 0644
 rdiff-backup = 1.0.5+SlapOSPatched001
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.4.3
 PyRSS2Gen = 1.1

software/build-rina/software.cfg

@@ -82,6 +82,3 @@ packages +=
   dh-autoreconf pkg-config doxygen maven xmlto
 # hellorina (shouldn't parts like lxml-python depend on the python of the SR?)
   python-dev
-
-[versions]
-slapos.recipe.template = 4.4

software/buildout-testing/software.cfg

@@ -59,6 +59,3 @@ template =
     key slapos_buildout slapos.buildout-repository:location
     key temp_directory directory:tmp
     raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter}
-
-[versions]
-slapos.recipe.template = 4.4

software/caddy-frontend/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = e8db3179e3278c6390a786cdcc947173
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = b29a4764dd489030030a72770162c157
+md5sum = 2329022227099971a57f710832509153
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
@@ -46,7 +46,7 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
-md5sum = 266f175dbdfc588af7a86b0b1884fe73
+md5sum = bd9e269130bac989faa639e0903814e2
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 18076ae37306aca1f6ef11e2173c887f
+md5sum = b99403e02d1aff471a7d5ebd0afbdb6c
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -111,7 +111,6 @@ context =
 {% set authorized_slave_string_list = [] %}
 {% set authorized_slave_list = [] %}
 {% set rejected_slave_dict = {} %}
-{% set rejected_slave_title_dict = {} %}
 {% set warning_slave_dict = {} %}
 {% set used_host_list = [] %}
 {% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
@@ -247,7 +246,6 @@ context =
 {%     do authorized_slave_list.append(authorized_slave) %}
 {%   else %}
 {%     do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %}
-{%     do rejected_slave_title_dict.__setitem__(slave.get('slave_title'), sorted(slave_error_list)) %}
 {%   endif %}
 {%   if len(slave_warning_list) > 0 %}
 {%     do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
@@ -326,7 +324,7 @@ accepted-slave-amount = {{ authorized_slave_list | length }}
 rejected-slave-amount = {{ rejected_slave_dict | length }}
 backend-client-caucase-url = {{ caucase_url }}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
-rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_title_dict, sort_keys=True)) }}
+rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_dict, sort_keys=True)) }}
 rejected-slave-promise-url = ${rejected-slave-promise:config-url}
 master-key-upload-url = ${request-kedifa:connection-master-key-upload-url}
 master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url}
@@ -485,6 +483,7 @@ user-ca-certificate = ${directory:aikc}/user-ca-certificate.pem
 user-crl = ${directory:aikc}/user-crl.pem
 user-created = ${directory:aikc}/user-created
 csr_id = ${directory:aikc}/csr_id
+data_dir = ${directory:aikc}/caucase-updater
 
 [aikc-user-csr]
 recipe = plone.recipe.command
@@ -547,7 +546,7 @@ command =
      buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/aikc-user-caucase-updater',
      url='${aikc-config:caucase-url}',
-     data_dir='${directory:srv}/caucase-updater',
+     data_dir='${aikc-config:data_dir}',
      crt_path='${aikc-config:key}',
      ca_path='${aikc-config:user-ca-certificate}',
      crl_path='${aikc-config:user-crl}',
@@ -620,6 +619,7 @@ user-ca-certificate = ${directory:aibcc}/user-ca-certificate.pem
 user-crl = ${directory:aibcc}/user-crl.pem
 user-created = ${directory:aibcc}/user-created
 csr_id = ${directory:aibcc}/csr_id
+data_dir = ${directory:aibcc}/caucase-updater
 
 [aibcc-user-csr]
 recipe = plone.recipe.command
@@ -684,7 +684,7 @@ command =
      buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/aibcc-user-caucase-updater',
      url='${aibcc-config:caucase-url}',
-     data_dir='${directory:srv}/caucase-updater',
+     data_dir='${aibcc-config:data_dir}',
      crt_path='${aibcc-config:key}',
      ca_path='${aibcc-config:user-ca-certificate}',
      crl_path='${aibcc-config:user-crl}',
@@ -747,9 +747,9 @@ filename = rejected-slave.json
 directory = ${directory:promise-output}
 rendered = ${:directory}/${:filename}
 template = {{ software_parameter_dict['template_empty'] }}
-{% if rejected_slave_title_dict %}
+{% if rejected_slave_dict %}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
-content = {{ dumps(json_module.dumps(rejected_slave_title_dict, indent=2, sort_keys=True)) }}
+content = {{ dumps(json_module.dumps(rejected_slave_dict, indent=2, sort_keys=True)) }}
 {% else %}
 content =
 {% endif %}

software/caddy-frontend/software.cfg

@@ -246,7 +246,6 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.4
 smmap = 0.9.0
 
 websockify = 0.8.0

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -43,6 +43,9 @@
     timeout {{ slave_parameter['request-timeout'] }}s
     # force reset of X-Forwarded-For
     header_upstream X-Forwarded-For {remote}
+    # workaround for lost connection to haproxy by reconnecting
+    try_duration 3s
+    try_interval 250ms
 {%- endmacro %} {# proxy_header #}
 
 {%- for tls in [True, False] %}

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -23,9 +23,19 @@ CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
 # Implement RFC 5861 with core
 CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}
-# Support stale-if-error by returning cached content on backend 5xx or unavailability
+# Simulate stale-if-error (not supported by TrafficServer), by using internal
+# mechanism
+# This results with replying last know non-5xx response until max_stale_age is reached
+# ignoring max-age returned by the server 
 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
-CONFIG proxy.config.http.negative_revalidating_lifetime INT 86400
+# max_stale_age set here means that for 1 week since last correct response
+# the response will be sent by the system
+CONFIG proxy.config.http.cache.max_stale_age INT 604800
+# negative_revalidating_lifetime just adds Expires header calculated as
+#          Expires = Date + negative_revalidating_lifetime
+# for case when backend replies 5xx, and Age > max-age and Age < max_stale_age
+# and that's not needed, so drop this behaviour
+CONFIG proxy.config.http.negative_revalidating_lifetime INT 0
 
 ##############################################################################
 # Thread configurations. Docs:
@@ -80,10 +90,9 @@ CONFIG proxy.config.net.default_inactivity_timeout INT 86400
 # Origin server connect attempts. Docs:
 #    https://docs.trafficserver.apache.org/records.config#origin-server-connect-attempts
 ##############################################################################
-# Try only once to connect (do not retry)
-CONFIG proxy.config.http.connect_attempts_max_retries INT 0
-# Try only once with server marked dead (do not retry)
-CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 0
+# workaround for lost connection to haproxy by reconnecting
+CONFIG proxy.config.http.connect_attempts_max_retries INT 3
+CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 1
 CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
 CONFIG proxy.config.http.connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}
 CONFIG proxy.config.http.post_connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}

software/caucase/software.cfg

@@ -24,6 +24,3 @@ context =
   key eggs_directory buildout:eggs-directory
   key caucase_jinja2_library caucase-jinja2-library:target
   key instance_caucased instance-caucased:target
-
-[versions]
-slapos.recipe.template = 3.0

software/cdn-me/software.cfg

@@ -39,4 +39,3 @@ eggs =
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.4

software/cythonplus-dev/buildout.hash.cfg

+[instance.cfg]
+filename = instance.cfg.in
+md5sum = a24384487467a07e827edab17a0b7206
+
+[runTestSuite.in]
+_update_hash_filename_ = runTestSuite.in
+md5sum = 21a8a202b14475707c414056ba393b3d

software/cythonplus-dev/instance.cfg.in

+[buildout]
+
+parts =
+  publish-env-path
+  runTestSuite
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[publish-env-path]
+recipe = slapos.cookbook:publish
+readme = Source the script to set up the environment.
+script = ${cythonplus_env.sh:rendered}
+repository = ${cythonplus-repository:repository}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[cythonplus-repository]
+recipe = slapos.recipe.build:gitclone
+repository = ${cythonplus-repository:location}
+git-executable = ${git:location}/bin/git
+shared = true
+
+[runTestSuite]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+template = ${runTestSuite.in:target}
+mode = 0755
+context =
+  key tmpdir directory:tmp
+  key slapparameter_dict slap-configuration:configuration
+  key cythonplus_repository cythonplus-repository:location
+  raw runTestSuite_interpreter ${runTestSuite_interpreter:bin-directory}/${runTestSuite_interpreter:interpreter}
+  raw cythonplus_env_sh ${cythonplus_env.sh:rendered}

software/cython-test/runTestSuite.in → software/cythonplus-dev/runTestSuite.in

-#!{{ runTestSuite_py }}
+#!{{ runTestSuite_interpreter }}
 """
   Script to run the Cython test suite using Nexedi's test node framework.
 """
@@ -9,7 +9,7 @@ from erp5.util import taskdistribution, testsuite
 
 os.environ['TEMP'] = {{ repr(tmpdir) }}
 
-command = """. {{ cython_env_sh }}
+command = """. {{ cythonplus_env_sh }}
 make all test
 """
 
@@ -96,7 +96,7 @@ def main():
       with open(os.devnull) as stdin:
         p = subprocess.Popen(command, shell=True, stdin=stdin,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             cwd={{repr(cython_repository)}})
+                             cwd={{ repr(cythonplus_repository) }})
     except Exception:
       end = time()
       stderr = traceback.format_exc()

software/cythonplus-dev/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../component/cythonplus/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance.cfg
+
+[instance.cfg]
+recipe = slapos.recipe.template
+output = ${buildout:directory}/${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+
+[runTestSuite.in]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+[runTestSuite_interpreter]
+recipe = zc.recipe.egg
+eggs = erp5.util
+interpreter = ${:_buildout_section_name_}

software/davstorage/development.cfg

@@ -34,4 +34,3 @@ command =
 
 [versions]
 openssl = 1.0.1c
-slapos.recipe.template = 2.4.2

software/davstorage/software.cfg

@@ -34,4 +34,3 @@ command =
 
 [versions]
 openssl = 1.0.1c
-slapos.recipe.template = 2.4.2

software/dream/software.cfg

@@ -49,7 +49,6 @@ mysqlclient = 1.3.12
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.4
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5/README.rst

@@ -59,7 +59,7 @@ Setting up replication
 In addition to your usual parameter set, you needs to provide the following parameters::
 
   {
-    "zope-partition-dict": {},      So no zope is instanciated
+    "zope-partition-dict": {},      So no zope is instantiated
     "zodb": [
       {
         "storage-dict": {

software/erp5/test/test/__init__.py

@@ -48,10 +48,6 @@ def setUpModule():
 class ERP5InstanceTestCase(SlapOSInstanceTestCase):
   """ERP5 base test case
   """
-  # ERP5 instanciation needs to run several times before being ready, as
-  # the root instance request more instances.
-  instance_max_retry = 7 # XXX how many times ?
-
   def getRootPartitionConnectionParameterDict(self):
     """Return the output paramters from the root partition"""
     return json.loads(

software/erp5/test/test/test_erp5.py

@@ -43,23 +43,44 @@ setUpModule # pyflakes
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url):
-    # What happens is that instanciation just create the services, but does not
+
+  def _checkERP5IsReachable(self, base_url, site_id, verify):
+    # We access ERP5 trough a "virtual host", which should make
+    # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
+    # as base.
+    virtual_host_url = urlparse.urljoin(
+        base_url,
+        '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
+        .format(site_id))
+
+    # What happens is that instantiation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
-    # instanciated, but zope is still busy creating the site and haproxy replies
+    # instantiated, but zope is still busy creating the site and haproxy replies
     # with 503 Service Unavailable when zope is not started yet, with 404 when
     # erp5 site is not created, with 500 when mysql is not yet reachable, so we
-    # retry in a loop until we get a succesful response.
-    for i in range(1, 60):
-      r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
-      if r.status_code != requests.codes.ok:
-        delay = i * 2
-        self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
-        time.sleep(delay)
-        continue
-      r.raise_for_status()
-      break
-
+    # configure this requests session to retry.
+    # XXX we should probably add a promise instead
+    session = requests.Session()
+    session.mount(
+        base_url,
+        requests.adapters.HTTPAdapter(
+            max_retries=requests.packages.urllib3.util.retry.Retry(
+                total=60,
+                backoff_factor=.5,
+                status_forcelist=(404, 500, 503))))
+
+    r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
+    self.assertEqual(r.status_code, requests.codes.found)
+    # access on / are redirected to login form, with virtual host preserved
+    self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
+
+    # login page can be rendered and contain the text "ERP5"
+    r = session.get(
+        urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
+        verify=verify,
+        allow_redirects=False,
+    )
+    self.assertEqual(r.status_code, requests.codes.ok)
     self.assertIn("ERP5", r.text)
 
   def test_published_family_default_v6_is_reachable(self):
@@ -67,18 +88,24 @@ class TestPublishedURLIsReachableMixin(object):
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      param_dict['family-default-v6'],
+      param_dict['site-id'],
+      verify=False,
+    )
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
+      param_dict['family-default'],
+      param_dict['site-id'],
+      verify=False,
+    )
 
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
-  """Test ERP5 can be instanciated with no parameters
+  """Test ERP5 can be instantiated with no parameters
   """
   __partition_reference__ = 'defp'
 
@@ -92,9 +119,32 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 Jupyter notebook
+  """
+  __partition_reference__ = 'jupyter'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'jupyter': {'enable': True}})}
 
-class TestApacheBalancerPorts(ERP5InstanceTestCase):
-  """Instanciate with two zope families, this should create for each family:
+  def test_jupyter_notebook_is_reachable(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+
+    self.assertEqual(
+      'https://[%s]:8888/tree' % self._ipv6_address,
+      param_dict['jupyter-url']
+    )
+
+    result = requests.get(
+      param_dict['jupyter-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [requests.codes.found, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+class TestBalancerPorts(ERP5InstanceTestCase):
+  """Instantiate with two zope families, this should create for each family:
    - a balancer entry point with corresponding haproxy
    - a balancer entry point for test runner
   """
@@ -151,36 +201,25 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
         3 + 5,
         len([p for p in all_process_info if p['name'].startswith('zope-')]))
 
-  def test_apache_listen(self):
-    # We have 2 families, apache should listen to a total of 3 ports per family
+  def test_haproxy_listen(self):
+    # We have 2 families, haproxy should listen to a total of 3 ports per family
     # normal access on ipv4 and ipv6 and test runner access on ipv4 only
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
-    apache_process = psutil.Process(process_info['pid'])
+    process_info, = [p for p in all_process_info if p['name'].startswith('haproxy-')]
+    haproxy_master_process = psutil.Process(process_info['pid'])
+    haproxy_worker_process, = haproxy_master_process.children()
     self.assertEqual(
         sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
         sorted([
             c.family
-            for c in apache_process.connections()
+            for c in haproxy_worker_process.connections()
             if c.status == 'LISTEN'
         ]))
 
-  def test_haproxy_listen(self):
-    # There is one haproxy per family
-    with self.slap.instance_supervisor_rpc as supervisor:
-      all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [
-        p for p in all_process_info if p['name'].startswith('haproxy-')
-    ]
-    haproxy_process = psutil.Process(process_info['pid'])
-    self.assertEqual([socket.AF_INET, socket.AF_INET], [
-        c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
-    ])
-
 
 class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
-  """Test ERP5 can be instanciated without test runner.
+  """Test ERP5 can be instantiated without test runner.
   """
   __partition_reference__ = 'distr'
   @classmethod
@@ -199,20 +238,22 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     self.assertNotIn('runUnitTest', bin_programs)
     self.assertNotIn('runTestSuite', bin_programs)
 
-  def test_no_apache_testrunner_port(self):
-    # Apache only listen on two ports, there is no apache ports allocated for test runner
+  def test_no_haproxy_testrunner_port(self):
+    # Haproxy only listen on two ports, there is no haproxy ports allocated for test runner
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
-    apache_process = psutil.Process(process_info['pid'])
+    process_info, = [p for p in all_process_info if p['name'].startswith('haproxy')]
+    haproxy_master_process = psutil.Process(process_info['pid'])
+    haproxy_worker_process, = haproxy_master_process.children()
     self.assertEqual(
         sorted([socket.AF_INET, socket.AF_INET6]),
         sorted(
             c.family
-            for c in apache_process.connections()
+            for c in haproxy_worker_process.connections()
             if c.status == 'LISTEN'
         ))
 
+
 class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test override zope node parameters
   """

software/erp5/upgrade_test/README.md

+Upgrade tests for ERP5 software release

software/erp5/upgrade_test/setup.py

+##############################################################################
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.upgrade_erp5'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Upgrade test for SlapOS' ERP5 software release",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'supervisor',
+        'slapos.libnetworkcache',
+        'typing; python_version<"3"',
+      ],
+      test_suite='test',
+    )

software/erp5/upgrade_test/test.py

+##############################################################################
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import glob
+import json
+import os
+import tempfile
+import time
+
+import requests
+import urlparse
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.testcase import installSoftwareUrlList
+from slapos.testing.testcase import SlapOSNodeCommandError
+from slapos.grid.utils import md5digest
+
+old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167/software/erp5/software.cfg'
+new_software_release_url = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
+
+_, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    old_software_release_url, software_id="upgrade_erp5")
+
+
+def setUpModule():
+  installSoftwareUrlList(
+      SlapOSInstanceTestCase,
+      [old_software_release_url, new_software_release_url],
+      debug=SlapOSInstanceTestCase._debug,
+  )
+
+
+class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
+  # use short partition names for unix sockets
+  __partition_reference__ = 'u'
+
+  @classmethod
+  def setUpOldInstance(cls):
+    """setUp hook executed while to old instance is running, before update
+    """
+    pass
+
+  _current_software_url = old_software_release_url
+
+  @classmethod
+  def getSoftwareURL(cls):
+    return cls._current_software_url
+
+  @classmethod
+  def setUpClass(cls):
+    # request and instanciate with old software url
+    super(ERP5UpgradeTestCase, cls).setUpClass()
+
+    cls.setUpOldInstance()
+
+    # request instance on new software
+    cls._current_software_url = new_software_release_url
+    cls.logger.debug('requesting instance on new software')
+    cls.requestDefaultInstance()
+
+    # wait for slapos node instance
+    snapshot_name = "{}.{}.setUpClass new instance".format(
+        cls.__module__, cls.__name__)
+    try:
+      if cls._debug and cls.instance_max_retry:
+        try:
+          cls.slap.waitForInstance(max_retry=cls.instance_max_retry - 1)
+        except SlapOSNodeCommandError:
+          cls.slap.waitForInstance(debug=True)
+      else:
+        cls.slap.waitForInstance(max_retry=cls.instance_max_retry,
+                                 debug=cls._debug)
+      cls.logger.debug("instance on new software done")
+    except BaseException:
+      cls.logger.exception("Error during instance on new software")
+      cls._storeSystemSnapshot(snapshot_name)
+      cls._cleanup(snapshot_name)
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+    else:
+      cls._storeSystemSnapshot(snapshot_name)
+
+    cls.computer_partition = cls.requestDefaultInstance()
+
+
+class TestERP5Upgrade(ERP5UpgradeTestCase):
+  @classmethod
+  def setUpOldInstance(cls):
+    cls._default_instance_old_parameter_dict = json.loads(
+        cls.computer_partition.getConnectionParameterDict()['_'])
+
+  def test_published_url_is_same(self):
+    default_instance_new_parameter_dict = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])
+    self.assertEqual(
+        default_instance_new_parameter_dict['family-default-v6'],
+        self._default_instance_old_parameter_dict['family-default-v6'],
+    )
+
+  def test_published_url_is_reachable(self):
+    default_instance_new_parameter_dict = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])
+
+    # get certificate from caucase
+    with tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    ) as ca_cert:
+      ca_cert.write(
+          requests.get(
+              urlparse.urljoin(
+                  default_instance_new_parameter_dict['caucase-http-url'],
+                  '/cas/crt/ca.crt.pem',
+              )).text)
+      ca_cert.flush()
+
+      # use a session to retry on failures, when ERP5 is not ready.
+      # (see also TestPublishedURLIsReachableMixin)
+      session = requests.Session()
+      session.mount(
+        default_instance_new_parameter_dict['family-default-v6'],
+        requests.adapters.HTTPAdapter(
+            max_retries=requests.packages.urllib3.util.retry.Retry(
+                total=60,
+                backoff_factor=.5,
+                status_forcelist=(404, 500, 503))))
+
+      session.get(
+            '{}/{}/login_form'.format(
+                default_instance_new_parameter_dict['family-default-v6'],
+                default_instance_new_parameter_dict['site-id'],
+            ),
+            verify=False,
+            # TODO: we don't use caucase yet here.
+            # verify=ca_cert.name,
+        ).raise_for_status()
+
+
+  def test_all_instances_use_new_software_release(self):
+    self.assertEqual(
+        {
+            os.path.basename(os.readlink(sr))
+            for sr in glob.glob(
+                os.path.join(
+                    self.slap.instance_directory,
+                    '*',
+                    'software_release',
+                ))
+        },
+        {md5digest(self.getSoftwareURL())},)

software/erp5testnode/software.cfg

@@ -42,7 +42,6 @@ output = ${buildout:directory}/template-default.cfg
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.4
 dnspython = 1.15.0
 PyXML = 0.8.5
 WebOb = 1.8.5

software/erp5testnode/testsuite/dummy/software.cfg

@@ -13,6 +13,3 @@ url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
 md5sum = 9fccb9600a085691ab0f9a20cd615c01
 mode = 0644
-
-[versions]
-slapos.recipe.template = 4.4

software/fluentd/software.cfg

@@ -21,6 +21,3 @@ context =
 gems +=
   fluent-plugin-wendelin==0.4
   fluent-plugin-bin
-
-[versions]
-slapos.recipe.template = 4.4

software/galene/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-cfg]
+filename = instance.cfg.in
+md5sum = 270b39f448ec553fa9e203c5fbb49856

software/galene/instance.cfg.in

+[buildout]
+parts =
+  publish-connection-parameter
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = https://[$${galene-wrapper:ip}]:$${galene-wrapper:port}
+admin-user = $${admin-password:username}
+admin-password = $${admin-password:passwd}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+configuration.ice-servers.json = [{"urls":["stun:turn.api.nexedi.net:3478"]}]
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+run = $${:var}/run
+
+services   = $${:etc}/service
+data       = $${:srv}/data
+groups     = $${:srv}/groups
+recordings = $${:srv}/recordings
+
+[galene-ssl]
+recipe = plone.recipe.command
+cert-file = $${directory:data}/cert.pem
+key-file = $${directory:data}/key.pem
+command = ${openssl:location}/bin/openssl req -newkey rsa:2048 -batch -new -x509  -days 3650 -nodes -keyout "$${:key-file}" -out "$${:cert-file}"
+update-command =
+stop-on-error = true
+
+[admin-password]
+recipe = slapos.cookbook:generate.password
+storage-path = $${directory:data}/.passwd
+username = admin
+
+[ice-servers.json]
+recipe = collective.recipe.template
+input = inline:
+  $${slap-configuration:configuration.ice-servers.json}
+output = $${directory:data}/ice-servers.json
+
+[groups-json]
+recipe = collective.recipe.template
+input = inline:{
+    "public":true,
+    "op": [{"username":"$${admin-password:username}","password":"$${admin-password:passwd}"}],
+    "other": [],
+    "presenter": [{"username": "", "password": "nexedi"}],
+    "max-users":100
+  }
+output = $${directory:groups}/public.json
+
+[galene-wrapper]
+recipe = slapos.cookbook:wrapper
+port = 8443
+ip = $${slap-configuration:ipv6-random}
+command-line = 
+  ${galene:location}/bin/galene
+    -static ${galene-repository:location}/static
+    -recordings $${directory:recordings}
+    -groups $${directory:groups}
+    -data $${directory:data}
+    -http [$${:ip}]:$${:port}
+wrapper-path = $${directory:services}/galene
+depends =
+  $${ice-servers.json:recipe}
+  $${groups-json:recipe}
+  $${galene-ssl:recipe}
+

software/galene/software.cfg

+[buildout]
+extends =
+  ../../component/golang/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../stack/slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  galene
+  eggs
+  instance-cfg
+
+# eggs for instance.cfg
+[eggs]
+recipe  = zc.recipe.egg
+eggs    =
+    plone.recipe.command
+    collective.recipe.template
+
+
+[galene-repository]
+<= git-repository
+repository = https://lab.nexedi.com/nexedi/galene.git
+revision = galene-0.2
+location = ${buildout:parts-directory}/galene-repository
+
+[galene]
+recipe = slapos.recipe.cmmi
+path = ${galene-repository:location}
+location = @@LOCATION@@
+golang  = ${golang1.13:location}
+configure-command = true
+make-binary = 
+make-targets =
+  go install -ldflags='-s -w'
+environment =
+  PATH=${:location}/bin:${:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:$PATH
+  GOPATH=${:location}
+  CGO_ENABLED=0
+
+[instance-cfg]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg

software/gitlab/software.cfg

@@ -405,5 +405,4 @@ strip-top-level-dir = true
 cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1

software/grafana/software.cfg

@@ -157,5 +157,4 @@ curl-bin = ${curl:location}/bin/curl
 dash-bin = ${dash:location}/bin/dash
 
 [versions]
-slapos.recipe.template = 4.2
 inotifyx = 0.2.2

software/hellorina/software.cfg

@@ -40,6 +40,3 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_buildout_section_name_}
 mode = 755
 md5sum = 78b77a6bda9958f547f7d89b747731e3
-
-[versions]
-slapos.recipe.template = 4.4

software/helloworld/software.cfg

@@ -53,7 +53,3 @@ context =
   raw template_monitor ${monitor2-template:rendered}
 # md5sum is fetched from buildout.hash.cfg and can be recalculated automatically by
 # calling update-hash
-
-# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
-[versions]
-slapos.recipe.template = 4.4

software/html5as/instance.cfg

-[buildout]
-parts =
-  switch-softwaretype
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${instance_html5as:output}

software/html5as/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[profile-common]
+nginx_location = {{ nginx_location }}
+dash_location = {{ dash_location }}
+template_nginx_conf = {{ template_nginx_conf_target }}
+template_mime_types = {{ template_mime_types_target }} 
+template_launcher = {{ template_launcher_target }} 
+
+[instance-html5as]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_instance_html5as_target }}
+rendered = ${buildout:directory}/${:filename}
+filename = instance-html5as.cfg
+context = 
+  section buildout buildout
+  section parameter_list profile-common
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${instance-html5as:rendered}

software/html5as/instance_html5as.cfg → software/html5as/instance_html5as.cfg.in

+#############################
+#
+# Deploy html5as instance
+#
+#############################
 [buildout]
 parts =
     nginx_conf
     downloader
-    launcher
     mime_types
+    launcher
     publish-connection-information
 
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+# Always the same.
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
 
 # partition tree
@@ -27,90 +35,97 @@ offline = true
 # |    |        |- index.html
 # |    |- backup/
 
-[rootdirectory]
+# Create all needed directories, depending on your needs
+[directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-tmp = $${buildout:directory}/tmp
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+srv = ${:home}/srv
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
-services = $${rootdirectory:etc}/run
-log = $${rootdirectory:var}/log
-run = $${rootdirectory:var}/run
-backup = $${rootdirectory:srv}/backup
-data = $${rootdirectory:srv}/html5as
+# Executables put here will be started but not monitored (for startup scripts)
+script = ${directory:etc}/run
+# Executables put here will be started and monitored (for daemons)
+service = ${directory:etc}/service
+log = ${directory:var}/log
+run = ${directory:var}/run
+backup = ${directory:srv}/backup
+data = ${directory:srv}/html5as
 
 [tempdirectory]
 recipe = slapos.cookbook:mkdirectory
-client_body_temp_path = $${rootdirectory:tmp}/client_body_temp_path
-proxy_temp_path = $${rootdirectory:tmp}/proxy_temp_path
-fastcgi_temp_path = $${rootdirectory:tmp}/fastcgi_temp_path
-uwsgi_temp_path = $${rootdirectory:tmp}/uwsgi_temp_path
-scgi_temp_path = $${rootdirectory:tmp}/scgi_temp_path
+tmp = ${directory:home}/tmp
+client_body_temp_path = ${:tmp}/client_body_temp_path
+proxy_temp_path = ${:tmp}/proxy_temp_path
+fastcgi_temp_path = ${:tmp}/fastcgi_temp_path
+uwsgi_temp_path = ${:tmp}/uwsgi_temp_path
+scgi_temp_path = ${:tmp}/scgi_temp_path
 
+# List of options for html5as configuration
+# It will run a simple nginx serving the content of srv/html5as
 [html5as]
 # Options
 nb_workers = 2
 
 # Network
-ip = $${slap-network-information:global-ipv6}
+ip = ${slap-network-information:global-ipv6}
 port = 8081
+access_url = http://[${:ip}]:${:port}
 
 # Paths
 # Log
-path_pid = $${basedirectory:run}/nginx.pid
-path_log = $${basedirectory:log}/nginx.log
-path_access_log = $${basedirectory:log}/nginx.access.log
-path_error_log = $${basedirectory:log}/nginx.error.log
-path_tmp = $${buildout:directory}/tmp
+path_pid = ${basedirectory:run}/nginx.pid
+path_log = ${basedirectory:log}/nginx.log
+path_access_log = ${basedirectory:log}/nginx.access.log
+path_error_log = ${basedirectory:log}/nginx.error.log
+path_tmp = ${tempdirectory:tmp}
 # Docroot
-docroot = $${basedirectory:data}
-default_index = $${basedirectory:data}/index.html
+docroot = ${basedirectory:data}
+default_index = ${basedirectory:data}/index.html
 # Config files
-path_nginx_conf = $${rootdirectory:etc}/nginx.conf
-path_mime_types = $${rootdirectory:etc}/mime_types
+path_nginx_conf = ${directory:etc}/nginx.conf
+path_mime_types = ${directory:etc}/mime_types
+# Binaries
+path_shell = {{ parameter_list['dash_location'] }}/bin/dash
+
 # Executables
-bin_nginx = ${nginx:location}/sbin/nginx
-bin_launcher = $${basedirectory:services}/launcher
-bin_downloader = $${basedirectory:services}/downloader
+bin_launcher = ${basedirectory:script}/launcher
 
 # Utils
-path_shell = ${dash:location}/bin/dash
-path_curl = ${curl:location}/bin/curl
-path_tar = ${tar:location}/bin/tar
+path_nginx = {{ parameter_list['nginx_location'] }}/sbin/nginx
 
+# Render nginx conf
 [nginx_conf]
 recipe = slapos.recipe.template:jinja2
-template = ${template_nginx_conf:location}/${template_nginx_conf:filename}
-rendered = $${html5as:path_nginx_conf}
+template = {{ parameter_list['template_nginx_conf'] }}
+rendered = ${html5as:path_nginx_conf}
 context = 
     section param_html5as html5as
     section param_tempdir tempdirectory
 
+# Render necessary mime types file for nginx
 [mime_types]
 recipe = slapos.recipe.template:jinja2
-template = ${template_mime_types:location}/${template_mime_types:filename}
-rendered = $${html5as:path_mime_types}
-
-[downloader]
-recipe = slapos.recipe.template:jinja2
-template = ${template_downloader:location}/${template_downloader:filename}
-rendered = $${html5as:bin_downloader}
-mode = 700
-context = 
-    section param_html5as html5as
-    key download_url slap-parameter:download_url
+template = {{ parameter_list['template_mime_types'] }}
+rendered = ${html5as:path_mime_types}
 
+# Render the script launching nginx
 [launcher]
 recipe = slapos.recipe.template:jinja2
-template = ${template_launcher:location}/${template_launcher:filename}
-rendered = $${html5as:bin_launcher}
+template = {{ parameter_list['template_launcher'] }}
+rendered = ${html5as:bin_launcher}
 mode = 700
 context = 
     section param_html5as html5as
 
+# Simple command to put content in the docroot
+[downloader]
+recipe = plone.recipe.command
+command = rm -r ${html5as:docroot}/*; echo "Hello World!" >  ${html5as:docroot}/index.html
+
+# Publish nginx address
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-server_url = http://[$${html5as:ip}]:$${html5as:port}
+server_url = ${html5as:access_url}

software/html5as/software.cfg

 [buildout]
 extends =
+# "slapos" stack describes basic things needed for 99.9% of SlapOS Software
+# Releases
   ../../stack/slapos.cfg
+# Extend here component profiles, like openssl, apache, mariadb, curl...
+# Or/and extend a stack (lamp, tomcat) that does most of the work for you
+# In this example we extend needed components for html5as.
   ../../component/nginx/buildout.cfg
-  ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
-  ../../component/tar/buildout.cfg
 
 parts =
-    dash
-    tar
-    curl
-    nginx
+# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
+# in 99,9% of Slapos Software Releases)
     slapos-cookbook
-    template
-    template_nginx_conf
-    template_downloader
-    template_launcher
-    template_mime_types
-    instance_html5as
+# Call creation of instance.cfg file that will be called for deployment of
+# instance
+    template-cfg
 
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/template.cfg
-md5sum = 918e0d6513e4d1c92051431d83261dab
+# Download instance.cfg.in (buildout profile used to deployment of instance),
+# replace all {{ foo_bar }} parameters by real values
+# The recipe, template and mode are fetched from jijna-template
+[template-cfg]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/template.cfg
+template = ${:_profile_base_location_}/${:filename}
+filename = instance.cfg.in
+md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
 mode = 0644
+context =
+  section buildout buildout
+  key nginx_location nginx:location
+  key dash_location dash:location
+  key template_nginx_conf_target template_nginx_conf:target
+  key template_mime_types_target template_mime_types:target
+  key template_launcher_target template_launcher:target
+  key template_instance_html5as_target instance_html5as:target
 
+# Download instance_html5as.cfg.in
 [instance_html5as]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance_html5as.cfg
-output = ${buildout:directory}/template_html5as.cfg
-md5sum = 41cb6178f760238ca276854873ef9364
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = instance_html5as.cfg.in
+md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
 mode = 0644
 
 [template_nginx_conf]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/nginx_conf.in
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = templates/nginx_conf.in
 md5sum = 61dc4c82bf48563228ce4dea6c5c6319
-filename = nginx_conf.in
 mode = 0644
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [template_launcher]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/launcher.in
-md5sum = acf5bb55ceac2e826259d28ed5c1de3a
-filename = launcher.in
-mode = 0644
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-
-[template_downloader]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/downloader.in
-md5sum = 9779e2db6c73d282f802b3407b390ede
-filename = downloader.in
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = templates/launcher.in
+md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
 mode = 0644
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [template_mime_types]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/mime_types.in
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+_update_hash_filename_ = templates/mime_types.in
 md5sum = 4ef94a7b458d885cd79ba0b930a5727e
-filename = mime_types.in
 mode = 0644
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
+# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
-async = 0.6.1
-gitdb = 0.5.4
-pycrypto = 2.6
-rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.4
-smmap = 0.8.2
-plone.recipe.command = 1.1

software/html5as/templates/downloader.in

-#! {{ param_html5as['path_shell'] }}
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-
-# Remove exsistant website
-rm -r {{ param_html5as['docroot'] }}/*
-
-# Download and extract website tarball into docroot directory
-exec {{ param_html5as['path_curl'] }} -ks {{ download_url }} | {{ param_html5as['path_tar'] }} -z -x -C {{ param_html5as['docroot'] }}

software/html5as/templates/launcher.in

@@ -3,4 +3,4 @@
 # BEWARE: It will be overwritten automatically
 
 # Run nginx
-exec {{ param_html5as['bin_nginx'] }} -c {{ param_html5as['path_nginx_conf'] }}
+exec {{ param_html5as['path_nginx'] }} -c {{ param_html5as['path_nginx_conf'] }}

software/html5ide/software.cfg

@@ -47,4 +47,3 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.4

software/htmlvalidatorserver/software.cfg

@@ -50,5 +50,4 @@ mode = 0644
 # but '1.3.4nxd2-SlapOSPatched001'.
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.4.3
 inotifyx = 0.2.2

software/jstestnode/software.cfg

@@ -118,7 +118,6 @@ output = ${buildout:directory}/runTestSuite.in
 
 [versions]
 
-slapos.recipe.template = 4.4
 selenium = 3.14.1
 urllib3 = 1.24
 certifi = 2018.10.15

software/jupyter/ERP5kernel.py

@@ -75,7 +75,7 @@ class ERP5Kernel(Kernel):
     self.title = None
     # Allowed HTTP request code list for making request to erp5 from Kernel
     # This list should be to used check status_code before making requests to erp5
-    self.allowed_HTTP_request_code_list = range(500, 511)
+    self.allowed_HTTP_request_code_list = list(range(500, 511))
     # Append request code 200 in the allowed HTTP status code list
     self.allowed_HTTP_request_code_list.append(200)
 

software/jupyter/buildout.hash.cfg

@@ -13,21 +13,21 @@
 # section inheritance (< = ...) are NOT supported (but you should really
 # not need these here).
 
-[instance-jupyter-notebook]
+[instance]
 filename = instance.cfg.in
-md5sum = 1d5fe6cc4e48672ae7be1c223794a932
+md5sum = f33fc954eba47b5ddbebc72c453304e3
+
+[instance-jupyter]
+filename = instance-jupyter.cfg.in
+md5sum = 9340498841caa5771f40f8c9e561eacd
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja
-md5sum = 720e90a829c63371696bc3009917a743
-
-[jupyter-set-password]
-filename = jupyter_set_password.cgi.jinja
-md5sum = b8d31441780b524a7e52d1710dd78385
+md5sum = 6e01da7d35c1d65d4a450b0f011b296d
 
 [erp5-kernel]
 filename = ERP5kernel.py
-md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+md5sum = da04b99b70b2e327c9e9b4cdd056098e
 
 [kernel-json]
 filename = kernel.json.jinja
@@ -35,4 +35,4 @@ md5sum = 33547be93a67530165e079dc3ecfdac3
 
 [custom-js]
 filename = custom.js
-md5sum = 40d938bb09261c65421a7725b40f87dc
+md5sum = 295aeee765bc4d09bf0686021f32f72c

software/jupyter/custom.js

@@ -81,41 +81,51 @@
  * @static
  */
 
-$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
+function prependERP5Help() {
   var kernelname = Jupyter.notebook.kernel_selector.current_selection;
-  var display_text="<div class='output_subarea output_text output_result'>\
-  <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
-  1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
-  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
-  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
-  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
-  It would be better to set the reference to match with erp5 reference pattern.</br>\
-  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
-  <p><u>OTHER USEFUL MAGICS</u> -</br>\
-  <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
-  <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
-  NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
-  so changes to them would be disregarded and would throw an error.</br>\
-  <p><u>About classes, functions and global state on modules:</u></p>\
-  Your code is going to be executed by ERP5, which can have many nodes </br>\
-  and there is no guarantee that your code is always going to be executed by the same server.</br>\
-  This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
-  won't be available across nodes. To solve this issue, you need to use a special object </br>\
-  called 'environment' to store your global setup. This object was designed to hold global </br>\
-  state and restore it for each code cell. Example:</br></br>\
-  <b>def my_setup():</br>\
-     # import modules, define functions and classes</br>\
-     # and set global state on modules</br>\
-     # return dict of variables to be available in code cells</br>\
-     {'my_var': 1}</br>\
-  environment.define(my_setup, 'my custom setup')</b></br></br>\
-  After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
-  following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
-  <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
-  because they will be executed in every code cell and can cause a substantial slow down.\
-  </pre></div>";
-  if (kernelname=="erp5"){
-  $('div#notebook-container').prepend(display_text);
-  }
+  var display_text = "<div class='output_subarea output_text output_result'>\
+    <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
+    1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
+    2. <b>%erp5_url &lt;your_erp5_url&gt;/Base_executeJupyter</b></br>\
+    3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+    4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+    5. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
+    It would be better to set the reference to match with erp5 reference pattern.</br>\
+    6. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+    <p><u>OTHER USEFUL MAGICS</u> -</br>\
+    <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+    <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
+    NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
+    so changes to them would be disregarded and would throw an error.</br>\
+    <p><u>About classes, functions and global state on modules:</u></p>\
+    Your code is going to be executed by ERP5, which can have many nodes </br>\
+    and there is no guarantee that your code is always going to be executed by the same server.</br>\
+    This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
+    won't be available across nodes. To solve this issue, you need to use a special object </br>\
+    called 'environment' to store your global setup. This object was designed to hold global </br>\
+    state and restore it for each code cell. Example:</br></br>\
+    <b>def my_setup():</br>\
+      # import modules, define functions and classes</br>\
+      # and set global state on modules</br>\
+      # return dict of variables to be available in code cells</br>\
+      {'my_var': 1}</br>\
+    environment.define(my_setup, 'my custom setup')</b></br></br>\
+    After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
+    following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
+    <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
+    because they will be executed in every code cell and can cause a substantial slow down.\
+    </pre></div>";
+
+    if (kernelname=="erp5"){
+      $('div#notebook-container').prepend(display_text);
+    }
+}
+
+define([
+  'base/js/namespace',
+  'base/js/promises'
+], function(Jupyter, promises) {
+  promises.notebook_loaded.then(function() {
+    prependERP5Help();
   });
-  
+});
\ No newline at end of file

software/jupyter/instance-jupyter.cfg.in

+{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
+
+[buildout]
+parts =
+  instance
+  publish-connection-parameter
+  jupyter-notebook-config
+  erp5-kernel
+  kernel-json
+  custom-js
+  frontend-promise
+  {% if additional_frontend %}
+  frontend-additional-promise
+  {% endif %}
+  monitor-base
+extends =
+  {{ monitor_template_rendered }}/template-monitor.cfg
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
+[instance-parameter]
+port = 8888
+host = ${slapconfiguration:ipv6-random}
+cert_file = ${generate-certificate:cert_file}
+key_file = ${generate-certificate:key_file}
+logfile = ${directory:log}/jupyter_notebook.log
+notebook_dir = ${directory:notebook_dir}
+
+[slap-parameter]
+frontend-software-type = RootSoftwareInstance
+frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+frontend-instance-guid =
+frontend-instance-name = Jupyter Frontend
+frontend-additional-software-type = RootSoftwareInstance
+frontend-additional-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+frontend-additional-instance-guid =
+frontend-additional-instance-name = Jupyter Frontend Additional
+
+[dynamic-jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 0644
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${instance-parameter:key_file} ]
+  then
+    {{ openssl_output }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
+      -out ${instance-parameter:cert_file}
+  fi
+update-command = ${:command}
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ bin_directory }}/jupyter-lab
+  --no-browser
+  --ip=${instance-parameter:host}
+  --port=${instance-parameter:port}
+  --port-retries=0
+  --certfile=${instance-parameter:cert_file}
+  --keyfile=${instance-parameter:key_file}
+  --notebook-dir=${instance-parameter:notebook_dir}
+  --log-level="DEBUG"
+wrapper-path = ${directory:service}/jupyter-lab
+environment =
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
+  LANG=C.UTF-8
+
+[jupyter-password]
+recipe = slapos.cookbook:generate.password
+bytes = 10
+
+[jupyter-notebook-config]
+<= dynamic-jinja2-template-base
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
+mode = 0744
+context =
+  key password jupyter-password:passwd
+  raw gcc_location {{ gcc_location }}
+  raw cythonplus_repository {{ cythonplus_repository }}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+notebook_dir = ${:var}/notebooks
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[request-slave-frontend-base]
+recipe = slapos.cookbook:requestoptional
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+slave = true
+config-type = notebook
+config-url = https://[${instance-parameter:host}]:${instance-parameter:port}
+return = secure_access
+
+[request-slave-frontend]
+<= request-slave-frontend-base
+software-url = ${slap-parameter:frontend-software-url}
+software-type = ${slap-parameter:frontend-software-type}
+name = ${slap-parameter:frontend-instance-name}
+sla-instance_guid = ${slap-parameter:frontend-instance-guid}
+
+[frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = frontend_promise.py
+config-url = ${publish-connection-parameter:url}
+
+{% if additional_frontend %}
+[request-slave-frontend-additional]
+<= request-slave-frontend-base
+software-url = ${slap-parameter:frontend-additional-software-url}
+software-type = ${slap-parameter:frontend-additional-software-type}
+name = ${slap-parameter:frontend-additional-instance-name}
+sla-instance_guid = ${slap-parameter:frontend-additional-instance-guid}
+
+[frontend-additional-promise]
+<= monitor-promise-base
+module = check_url_available
+name = frontend_additional_promise.py
+config-url = ${publish-connection-parameter:url-additional}
+{% endif %}
+
+[monitor-instance-parameter]
+monitor-base-url = ${monitor-frontend-promise:url}
+# In case you're using a developer instance you should edit these in:
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
+# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
+instance-configuration =
+  raw jupyter-password ${jupyter-password:passwd}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+jupyter-classic-url = ${request-slave-frontend:connection-secure_access}/tree
+url = ${:jupyter-classic-url}
+jupyterlab-url = ${request-slave-frontend:connection-secure_access}/lab
+{% if additional_frontend %}
+jupyter-classic-url-additional = ${request-slave-frontend-additional:connection-secure_access}/tree
+url-additional = ${:jupyter-classic-url-additional}
+jupyterlab-url-additional = ${request-slave-frontend-additional:connection-secure_access}/lab
+{% endif %}
+password = ${jupyter-password:passwd}
+
+[erp5-kernel]
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js

software/jupyter/instance.cfg.in

 [buildout]
 parts =
-  instance
-  jupyter_notebook
-  read-knowledge0
-  publish-connection-parameter
-  jupyter-notebook-config
-  erp5-kernel
-  kernel-json
-  custom-js
-  monitor-base
-extends =
-  {{ monitor_template_rendered }}/template-monitor.cfg
+  switch_softwaretype
 
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
 
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
+[switch_softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default = $${:jupyter}
+jupyter = instance-jupyter:rendered
+RootSoftwareInstance = $${:default}
 
-[slapconfiguration]
+[slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
-computer = ${slap-connection:computer-id}
-partition = ${slap-connection:partition-id}
-url = ${slap-connection:server-url}
-key = ${slap-connection:key-file}
-cert = ${slap-connection:cert-file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
 
-# ERP5 URL to use in Jupyter by default
-# default value is empty - which means no default ERP5 URL
-configuration.erp5-url =
-
-[instance-parameter]
-port = 8888
-host = ${slapconfiguration:ipv6-random}
-cert_file = ${generate-certificate:cert_file}
-key_file = ${generate-certificate:key_file}
-logfile = ${directory:log}/jupyter_notebook.log
-notebook_dir = ${directory:notebook_dir}
-
-[dynamic-jinja2-template-base]
+[instance-jupyter]
 recipe = slapos.recipe.template:jinja2
+filename = instance-jupyter.cfg.in
+template = ${instance:_profile_base_location_}/$${:filename}
+rendered = $${buildout:directory}/template.cfg
 mode = 0644
-
-[generate-certificate]
-; TODO: there is a slapos recipe to generate certificates. Use it instead
-recipe = plone.recipe.command
-command =
-  if [ ! -e ${instance-parameter:key_file} ]
-  then
-    {{ openssl_output }} req -x509 -nodes -days 3650 \
-      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
-      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
-      -out ${instance-parameter:cert_file}
-  fi
-update-command = ${:command}
-cert_file = ${directory:etc}/jupyter_cert.crt
-key_file = ${directory:etc}/jupyter_cert.key
-
-[instance]
-recipe = slapos.cookbook:wrapper
-command-line =
-  {{ bin_directory }}/jupyter-lab
-  --no-browser
-  --ip=${instance-parameter:host}
-  --port=${instance-parameter:port}
-  --port-retries=0
-  --certfile=${instance-parameter:cert_file}
-  --keyfile=${instance-parameter:key_file}
-  --notebook-dir=${instance-parameter:notebook_dir}
-  --log-level="DEBUG"
-wrapper-path = ${directory:service}/jupyter-lab
-environment =
-  JUPYTER_PATH=${directory:jupyter_dir}
-  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
-  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
-  LANG=C.UTF-8
-
-[jupyter-notebook-config]
-<= dynamic-jinja2-template-base
-template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
-rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
-mode = 0744
-context =
-  raw config_cfg ${buildout:directory}/knowledge0.cfg
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-home = ${buildout:directory}
-etc = ${:home}/etc
-var = ${:home}/var
-script = ${:etc}/run/
-service = ${:etc}/service
-log = ${:var}/log
-notebook_dir = ${:var}/notebooks
-# Add folders to explicitly define jupyter directory
-jupyter_dir = ${:home}/jupyter
-jupyter_config_dir = ${:jupyter_dir}/etc
-jupyter_kernel_dir = ${:jupyter_dir}/kernels
-jupyter_runtime_dir = ${:jupyter_dir}/runtime
-jupyter_custom_dir = ${:jupyter_config_dir}/custom
-jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
-erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
-
-[jupyter_notebook]
-# This part is called like this because knowledge0.write uses the part name for
-# the section name in the config file.
-recipe = slapos.cookbook:zero-knowledge.write
-password = 
-filename = knowledge0.cfg
-
-[read-knowledge0]
-recipe = slapos.cookbook:zero-knowledge.read
-filename = knowledge0.cfg
-password =
-
-[monitor-instance-parameter]
-monitor-base-url = ${monitor-frontend-promise:url}
-# In case you're using a developer instance you should edit these in:
-# monitor-base-url = ${monitor-httpd-conf-parameter:url}
-# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
-# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
-instance-configuration =
-  raw jupyter-password ${read-knowledge0:password}
-
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish.serialised
-jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
-url = ${:jupyter-classic-url}
-jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
-
-[erp5-kernel]
-recipe = slapos.cookbook:symbolic.link
-link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
-target-directory = ${directory:erp5_kernel_dir}
-
-[kernel-json]
-<= dynamic-jinja2-template-base
-template = {{ kernel_json_location }}/{{ kernel_json_filename }}
-rendered = ${directory:erp5_kernel_dir}/kernel.json
-# Use python2.7 executable bin file for kernel config
 context =
-  raw python_executable {{ python_executable }}
-  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
-  key erp5_url slapconfiguration:configuration.erp5-url 
-  raw display_name ERP5
-  raw language_name python
-
-[custom-js]
-recipe = slapos.cookbook:symbolic.link
-target-directory = ${directory:jupyter_custom_dir}
-link-binary = {{ custom_js_location }}/custom.js
+  key slapparameter_dict slap-configuration:configuration
+  raw bin_directory ${buildout:bin-directory}
+  raw develop_eggs_directory ${buildout:develop-eggs-directory}
+  raw eggs_directory ${buildout:eggs-directory}
+  raw openssl_output ${openssl-output:openssl}
+  raw python_executable ${jupyter:python_executable}
+  raw jupyter_config_location ${jupyter-notebook-config:location}
+  raw jupyter_config_filename ${jupyter-notebook-config:filename}
+  raw erp5_kernel_location ${erp5-kernel:location}
+  raw erp5_kernel_filename ${erp5-kernel:filename}
+  raw kernel_json_location ${kernel-json:location}
+  raw kernel_json_filename ${kernel-json:filename}
+  raw custom_js_location ${custom-js:location}
+  raw custom_js_filename ${custom-js:filename}
+  raw monitor_template_rendered ${buildout:directory}
+  raw cythonplus_repository ${cythonplus-repository:location}
+  raw gcc_location ${gcc:prefix}

software/jupyter/jupyter_notebook_config.py.jinja

@@ -2,37 +2,13 @@
   This script initializes Jupyter's configuration such as passwords and other
   things. It is run by IPython hence why it can use functions like get_config().
 '''
-import ConfigParser
-import random
+import configparser
 from notebook.auth import passwd
 import os
 
-def random_password(length = 10):
-  result = ""
-  for i in range(0, length):
-    result = result + chr(random.randint(0, 25) + ord('a'))
-  return result
-
-knowledge_0 = '{{ config_cfg }}'
-
-if not os.path.exists(knowledge_0):
-  print "Your software does <b>not</b> embed 0-knowledge. \
-  This interface is useless in this case</body></html>"
-  exit(0)
-
 c = get_config()
 
-parser = ConfigParser.ConfigParser()
-parser.read(knowledge_0)
-
-if not parser.has_section("jupyter_notebook"):
-  parser.add_section("jupyter_notebook")
-
-if not parser.has_option("jupyter_notebook", "password") or \
-  parser.get("jupyter_notebook", "password") == "":
-  parser.set("jupyter_notebook", "password", random_password())
-
-c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
+c.NotebookApp.password = passwd("{{ password }}")
 
-with open(knowledge_0, 'w') as file:
-  parser.write(file)
+os.environ['PATH'] = "{{ gcc_location }}/bin" + os.pathsep + os.environ['PATH']
+os.environ['PYTHONPATH'] = "{{ cythonplus_repository }}" + os.pathsep + os.environ['PYTHONPATH']

software/jupyter/software.cfg

@@ -2,6 +2,7 @@
 extends = 
   buildout.hash.cfg
   ../../stack/slapos.cfg
+  ../../component/cythonplus/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/jupyter/buildout.cfg
   ../../stack/monitor/buildout.cfg
@@ -9,7 +10,10 @@ parts +=
   slapos-cookbook
   jupyter
   jupyter-notebook-initialized-scripts
-  instance-jupyter-notebook
+  instance
+
+[python]
+part = python3
 
 [gcc]
 # Always build GCC for Fortran (see openblas).
@@ -27,9 +31,6 @@ mode = 0644
 [jupyter-notebook-config]
 <= download-file-base
 
-[jupyter-set-password]
-<= download-file-base
-
 [erp5-kernel]
 <= download-file-base
 
@@ -39,64 +40,55 @@ mode = 0644
 [custom-js]
 <= download-file-base
 
-[instance-jupyter-notebook]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/${:filename}
-rendered = ${buildout:directory}/template.cfg
+[instance]
+recipe = slapos.recipe.template
+url =  ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
 mode = 0644
-context =
-  key bin_directory buildout:bin-directory
-  key develop_eggs_directory buildout:develop-eggs-directory
-  key eggs_directory buildout:eggs-directory
-  key openssl_output openssl-output:openssl
-  key python_executable jupyter:python_executable
-  key jupyter_config_location jupyter-notebook-config:location
-  key jupyter_config_filename jupyter-notebook-config:filename
-  key jupyter_set_password_location jupyter-set-password:location
-  key jupyter_set_password_filename jupyter-set-password:filename
-  key erp5_kernel_location erp5-kernel:location
-  key erp5_kernel_filename erp5-kernel:filename
-  key kernel_json_location kernel-json:location
-  key kernel_json_filename kernel-json:filename
-  key custom_js_location custom-js:location
-  key custom_js_filename custom-js:filename
-  key monitor_template_rendered buildout:directory
 
 [versions]
-Pygments = 2.2.0
+Pygments = 2.7.2
 astor = 0.5
+async-generator = 1.10
 backports-abc = 0.5
 backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
+bleach = 3.2.1
 cycler = 0.10.0
-ipykernel = 4.5.2
+defusedxml = 0.6.0
+entrypoints = 0.3
+ipykernel = 5.3.4
 ipython = 5.3.0
 ipython-genutils = 0.1.0
 ipywidgets = 6.0.0
-jupyter-client = 5.0.0
-jupyter-core = 4.3.0
+jupyter-client = 6.1.7
+jupyter-core = 4.7.0
 jupyterlab = 0.26.3
 jupyterlab-launcher = 0.3.1
+jupyterlab-pygments = 0.1.2
 matplotlib = 2.1.2
-mistune = 0.7.3
-nbformat = 4.3.0
-notebook = 4.4.1
-pandas = 0.19.2
+mistune = 0.8.4
+nest-asyncio = 1.4.3
+nbclient = 0.5.1
+nbformat = 5.0.8
+notebook = 6.1.5
+numpy = 1.14.6
+pandas = 0.25.3
+pandocfilters = 1.4.3
 plone.recipe.command = 1.1
 prompt-toolkit = 1.0.13
 ptyprocess = 0.5.1
-pyzmq = 16.0.2
-scikit-learn = 0.18.1
+pyzmq = 20.0.0
+scikit-learn = 0.20.4
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 4.4
-statsmodels = 0.8.0
-terminado = 0.6
-tornado = 4.4.2
-traitlets = 4.3.2
+statsmodels = 0.11.1
+testpath = 0.4.4
+terminado = 0.9.1
+tornado = 6.1
+traitlets = 5.0.5
+webencodings = 0.5.1
 widgetsnbextension = 2.0.0
-# numpy >= 1.13.1 is required for numpy.core.multiarray
-numpy = 1.13.1
 
 
 # Required by:
@@ -104,41 +96,55 @@ numpy = 1.13.1
 certifi = 2020.6.20
 
 # Required by:
-# notebook==4.3.2
-# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
-nbconvert = 4.1.0
+# notebook==6.1.5
+Send2Trash = 1.5.0
+
+# Required by:
+# notebook==6.1.5
+argon2-cffi = 20.1.0
+
+# Required by:
+# notebook==6.1.5
+nbconvert = 6.0.7
 
 # Required by:
 # ipython==5.3.0
 pathlib2 = 2.2.1
 
 # Required by:
-# statsmodels==0.8.0
-patsy = 0.4.1
+# statsmodels==0.11.1
+patsy = 0.5.1
 
 # Required by:
 # ipython==5.3.0
-pexpect = 4.2.1
+pexpect = 4.8.0
 
 # Required by:
 # ipython==5.3.0
 pickleshare = 0.7.4
 
 # Required by:
-# matplotlib==2.1.2
-# pandas==0.19.2
-python-dateutil = 2.6.0
+# notebook==6.1.5
+prometheus-client = 0.9.0
+
+# Required by:
+# statsmodels==0.11.1
+python-dateutil = 2.8.1
 
 # Required by:
 # pathlib2==2.2.1
 scandir = 1.5
 
 # Required by:
-# statsmodels==0.8.0
-scipy = 0.19.0
+# statsmodels==0.11.1
+pytz = 2020.4
 
 # Required by:
-# tornado==4.4.2
+# statsmodels==0.11.1
+scipy = 1.0.1
+
+# Required by:
+# tornado==6.1
 singledispatch = 3.4.0.3
 
 # Required by:

software/jupyter/test/test.py

@@ -26,7 +26,7 @@
 ##############################################################################
 
 
-import httplib
+import http.client
 import json
 import os
 import requests
@@ -48,10 +48,14 @@ class TestJupyter(InstanceTestCase):
     except Exception as e:
       self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
 
+    self.assertTrue('password' in connection_dict)
+    password = connection_dict['password']
+
     self.assertEqual(
       {
         'jupyter-classic-url': 'https://[%s]:8888/tree' % (self._ipv6_address, ),
         'jupyterlab-url': 'https://[%s]:8888/lab' % (self._ipv6_address, ),
+        'password': '%s' % (password, ),
         'url': 'https://[%s]:8888/tree' % (self._ipv6_address, )
       },
       connection_dict
@@ -60,7 +64,7 @@ class TestJupyter(InstanceTestCase):
     result = requests.get(
       connection_dict['url'], verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Ftree'],
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
@@ -68,7 +72,7 @@ class TestJupyter(InstanceTestCase):
       connection_dict['jupyter-classic-url'],
       verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Ftree'],
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
@@ -76,6 +80,92 @@ class TestJupyter(InstanceTestCase):
       connection_dict['jupyterlab-url'],
       verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Flab'],
+      [http.client.FOUND, True, '/login?next=%2Flab'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+
+class TestJupyterPassword(InstanceTestCase):
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    url = connection_dict['url']
+    with requests.Session() as s:
+      resp = s.get(url, verify=False)
+      result = s.post(
+        resp.url,
+        verify = False,
+        data={"_xsrf": s.cookies["_xsrf"], "password": connection_dict['password']}
+      )
+      self.assertEqual(
+        [http.client.OK, url],
+        [result.status_code, result.url]
+      )
+
+
+class TestJupyterAdditional(InstanceTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'frontend-additional-instance-guid': 'SOMETHING'
+    }
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    result = requests.get(
+      connection_dict['url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyter-classic-url'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyterlab-url'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Flab'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['url-additional'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyter-classic-url-additional'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyterlab-url-additional'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Flab'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )

software/kvm/software.cfg

@@ -205,5 +205,4 @@ websockify = 0.9.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 4.4
 smmap = 0.9.0

software/metabase/software.cfg

@@ -24,6 +24,3 @@ md5sum = 0a5e780dcf7d9ffe73f1ed789f863a57
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/instance.cfg
-
-[versions]
-slapos.recipe.template = 4.4

software/monitor/software.cfg

@@ -76,7 +76,6 @@ eggs +=
 scripts =
 
 [versions]
-slapos.recipe.template = 4.4
 
 surykatka = 0.5.0
 

software/mysql-5.1/software.cfg

@@ -45,4 +45,3 @@ mode = 0644
 
 [versions]
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 2.2

software/nayuos/software.cfg

@@ -67,6 +67,3 @@ dependencies =
   libexpat
   libaio
   boost-lib
-
-[versions]
-slapos.recipe.template = 4.4

software/neoppod/software-common.cfg

@@ -146,7 +146,6 @@ persistent = 4.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
-slapos.recipe.template = 4.4
 transaction = 1.7.0
 zodbpickle = 1.0.4
 cython-zstd = 0.2

software/neoppod/stress-testing/software.cfg

@@ -84,6 +84,3 @@ packages +=
   libnetfilter-queue-dev nftables
 # extra requirements for this SR
   screen xz-utils
-
-[versions]
-slapos.recipe.template = 4.4

software/neotest/gowork.cfg

@@ -3,6 +3,7 @@
 # list of go git repositories to fetch
 [gowork.goinstall]
 depends_gitfetch  =
+    ${go_crawshaw.io_sqlite:recipe}
     ${go_github.com_DataDog_czlib:recipe}
     ${go_github.com_cznic_strutil:recipe}
     ${go_github.com_davecgh_go-spew:recipe}
@@ -37,6 +38,12 @@ depends_gitfetch  =
     ${go_lab.nexedi.com_kirr_neo:recipe}
 
 
+[go_crawshaw.io_sqlite]
+<= go-git-package
+go.importpath = crawshaw.io/sqlite
+repository    = https://github.com/crawshaw/sqlite
+revision      = v0.3.2-7-gc582b9de4f
+
 [go_github.com_DataDog_czlib]
 <= go-git-package
 go.importpath = github.com/DataDog/czlib
@@ -221,10 +228,10 @@ revision      = v2.1.1-97-geeeca48fe7
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/go123
 repository    = https://lab.nexedi.com/kirr/go123.git
-revision      = 316617668e
+revision      = c8d9907ef7
 
 [go_lab.nexedi.com_kirr_neo]
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/neo
 repository    = https://lab.nexedi.com/kirr/neo.git
-revision      = v1.9-2480-g5ee3c077b3
+revision      = v1.12-2791-gf3effa6c53

software/neotest/software.cfg

@@ -34,7 +34,7 @@ parts =
 
     neoppod-develop
     neoppod
-    wendelin.core-dev
+    wendelin.core
     scripts
 
 # for ZEO scripts (runzeo)
@@ -107,7 +107,7 @@ template = inline:
 [neotest-python]
 <= python-interpreter
 eggs +=
-    ${wendelin.core-dev:egg}
+    ${wendelin.core:egg}
     ${neoppod:eggs}
 # for instance
     plone.recipe.command
@@ -116,7 +116,7 @@ eggs +=
 recipe  = zc.recipe.egg:scripts
 eggs    =
 # to install not only wendelin.core modules but also scripts
-    ${wendelin.core-dev:egg}
+    ${wendelin.core:egg}
 
 
 # wendelin.core: latest not yet released

software/nginx-push-stream/software.cfg

@@ -42,5 +42,4 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.4
 inotifyx = 0.2.2

software/plantuml/software.cfg

@@ -32,7 +32,3 @@ output = ${buildout:directory}/${:_buildout_section_name_}
 recipe = slapos.recipe.build:download
 url = https://sourceforge.net/projects/plantuml/files/1.2020.15/plantuml.1.2020.15.war
 md5sum = ed203cb3b90df8f77492fa36ea6490a5
-
-
-[versions]
-slapos.recipe.template = 4.4

software/postgres/software.cfg

@@ -19,6 +19,3 @@ url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
 md5sum = d8b833a2054b82b6031a9420008b58fd
 mode = 0644
-
-[versions]
-slapos.recipe.template = 2.4.2

software/powerdns/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 20c37ea06a8fa405bc02470d5115fd11
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
-md5sum = c2bd424f588ad57d37f4cf1329734fb6
+md5sum = 72ce30bee3b8a9da8ac9be7eb65d83a2
 
 [iso-list]
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
@@ -34,4 +34,4 @@ md5sum = c4dc8c141d81b92d92cdb82ca67a13ee
 
 [template-zones-file]
 _update_hash_filename_ = template/zones-file.yml.jinja2
-md5sum = 03037141ad1d3467ae878c9798724f70
+md5sum = 612de569ac3d1e8cc10b830683ff92ae

software/powerdns/instance-powerdns-input-schema.json

@@ -33,12 +33,12 @@
       "default": "",
       "type": "string"
     },
-    "zone": {
+    "supported-zone-list": {
       "title": "Zone",
       "description": "Zone to be handled by the DNS cluster",
       "type": "string",
       "default": "domain.com",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}(\\s([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6})*$"
     },
     "server-admin": {
       "title": "Zone Administrator Email",
@@ -50,7 +50,7 @@
       "title": "DNS domains template string",
       "description": "Template used to generate DNS domain name",
       "type": "string",
-      "default": "ns%s. + zone"
+      "default": "ns%s.domain.com"
     },
     "monitor-interface-url": {
       "title": "Monitor Web Interface URL",

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -33,9 +33,9 @@ context =
 {% endif -%}
 
 ## DNS set up
-{% set zone = slapparameter_dict.pop('zone', 'domain.com') %}
+{%- set supported_zone_list = slapparameter_dict.pop('supported-zone-list', 'domain.com').split() %}
 {% set server_admin = slapparameter_dict.pop('server-admin', 'admin@domain.com') %}
-{% set dns_name_template_string = slapparameter_dict.pop('dns-name-template-string', 'ns%s.' + zone) %}
+{% set dns_name_template_string = slapparameter_dict.pop('dns-name-template-string', 'ns%s.domain.com') %}
 
 # Here we request individualy each dns.
 # The presence of sla parameters is checked and added if found
@@ -65,7 +65,7 @@ name = {{dns_name}}
 {%   if state_key in slapparameter_dict %}
 state = {{ slapparameter_dict.pop(state_key) }}
 {%   endif%}
-config-zone = {{ zone }}
+config-supported-zone-list = {{ ' '.join(supported_zone_list) }}
 config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
 {%   for parameter in sla_parameters -%}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
@@ -74,11 +74,9 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 [{{promise_section_title}}]
 <= monitor-promise-base
 module = check_port_listening
-name = pdns-port-listening.py
-{%   set ipv6 = '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' -%}
-config-hostname = {{ipv6}}
-{%   set port = '${' ~ request_section_title ~ ':connection-powerdns-port}' -%}
-config-port = {{port}}
+name = {{promise_section_title}}.py
+config-hostname = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
+config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}
 
 {%  do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
 {% endfor -%}

software/powerdns/slave-instance-powerdns-input-schema.json

@@ -8,6 +8,13 @@
       "description": "Record for the configuration",
       "type": "string"
     },
+    "applicable-zone": {
+      "title": "Applicable Zone",
+      "description": "Zone to which this record belongs. You can put only one zone here. If the record belongs to several zones, you should create several slaves.",
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
+      "default": "domain.com",
+      "type": "string"
+    },
     "origin": {
       "title": "Origin",
       "description": "Used to qualify RR in the configuration. i.e.: if your origin is a.example.com and the RR for Europe is 'eu' the european clients will use eu.a.example.com",

software/powerdns/software.cfg

@@ -54,7 +54,6 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.4
 passlib = 1.7.1
 GitPython = 2.1.11
 lockfile = 0.12.2

software/powerdns/template/zones-file.yml.jinja2

 # See https://doc.powerdns.com/authoritative/backends/geoip.html
 
 {%- set slave_instance_list = json_module.loads(slapparameter_dict.get('extra_slave_instance_list', '[]')) %}
-{%- set zone = slapparameter_dict.get('zone', 'example.com') %}
+{%- set supported_zone_list = slapparameter_dict.get('supported-zone-list', 'example.com').split() %}
 
 {%- macro disambiguate_domain_name(a, b) %}
 {#- See http://www.dns-sd.org/trailingdotsindomainnames.html #}
@@ -13,6 +13,8 @@
 {%- endmacro %}
 
 domains:
+{%- for zone in supported_zone_list %}
+
 - domain: {{ zone }}
   # TODO: what value for ttl?
   ttl: 300
@@ -48,6 +50,7 @@ domains:
 
     {%- for slave in slave_instance_list %}
 
+    {%- if slave['applicable-zone'] == zone %}
     {%-   set origin = slave['origin'] %}
     {%-   set unique_slave_id = slave['slave_reference'] %}
     {#-   Set the RR to use for each region, as described in
@@ -77,13 +80,15 @@ domains:
       - cname: {{ disambiguate_domain_name(rr_dict[region], origin) }}
     {%-     endfor %}
     {%-   endfor %}
+    {%- endif %}
     {%- endfor %}
 
   services:
     {%- for slave in slave_instance_list %}
+    {%- if slave['applicable-zone'] == zone %}
     {%-   set origin = slave['origin'] %}
     {%-   set unique_slave_id = slave['slave_reference'] %}
-    {{ disambiguate_domain_name(slave['record'], zone) }}:
+    {{ disambiguate_domain_name(slave['record'], slave['applicable-zone']) }}:
       {#- Note: Placeholders (i.e. "country." and "continent.") are used to avoid
           possible name collisions, e.g.:
           - %cc for American Samoa is 'as'
@@ -94,4 +99,6 @@ domains:
       {%- for ip_range, country_code in china %}
       {{ ip_range }}: {{ country_code }}.country.{{ unique_slave_id }}
       {%- endfor %}
+    {%- endif %}
     {%- endfor %}
+{%- endfor %}

software/proftpd/software.cfg

@@ -46,5 +46,4 @@ context =
 [versions]
 collective.recipe.environment = 1.1.0
 collective.recipe.grp = 1.1.0
-slapos.recipe.template = 4.4
 plone.recipe.command = 1.1