ERP5 and Jupyter integrated together
This patch teaches ERP5 software release to automatically instantiate Jupyter notebook web UI and tune it to connect to ERP5 by default. When Jupyter is enabled, it also installs on-server erp5_data_notebook bt5 (nexedi/erp5!29) which handles code execution requested for Jupyter. For ERP5 - for security and backward compatibility reasons - Jupyter instantiation and erp5_data_notebook bt5 install happen only if jupyter is explicitly enabled in instance parameters. The default is not to have Jupyter out of the box. On the other hand for Wendelin SR, which inherits from ERP5 SR, the default is to have Jupyter out of the box, because Wendelin SR is fresh enough without lots of backward compatibility needs, and Jupyter is usually very handy for people who use Wendelin. ~~~~ For integration, we reuse already established in ERP5 infrastructure, to request various slave instances, and request Jupyter in a way so it automatically tunes and connects to balancer of one of Zope family. Jupyter code itself is compiled by reusing software/ipython_notebook/software.cfg, and Jupyter instance code is reused by hooking software/ipython_notebook/instance.cfg.in into ERP5 SR properly (the idea to override instance-jupyter not to render into default template.cfg is taken from previous work by @tiwariayush). ~~~~ I tested this patch inside webrunner with create-erp5-site software type and various configurations (whether to have or not have jupyter, to which zope family to connect it, etc). I have not tested frontend instantiation fully - because tests were done only in webrunner, but I've tried to make sure generated buildout code is valid for cases with frontend. NOTE the code in this patch depends erp5_data_notebook bt5 (nexedi/erp5!29) which just got merged to erp5.git recently (see nexedi/erp5@f662b5a2) NOTE even when erp5_data_notebook bt5 is installed, on a freshly installed ERP5, it is required to "check site consistency" first, so that initial bt5(s) are actually installed and erp5 is ready to function. /cc @vpelletier, @Tyagov, @klaus, @Camata, @tiwariayush, @Kreisel, @jerome, @nexedi /proposed-for-review-on nexedi/slapos!43
| ... | @@ -40,6 +40,11 @@ | ... | @@ -40,6 +40,11 @@ |
| "description": "Relational database access information", | "description": "Relational database access information", | ||
| "type": "string" | "type": "string" | ||
| } | } | ||
| "jupyter-url": { | |||
|
|||
| "description": "Jupyter notebook web UI access information", | |||
| "type": "string", | |||
| "optional": true | |||
| } | |||
| }, | }, | ||
| "patternProperties": { | "patternProperties": { | ||
| "family-.*": { | "family-.*": { | ||
| ... | ... | ||
| ... | @@ -5,6 +5,9 @@ | ... | @@ -5,6 +5,9 @@ |
| {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%} | {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%} | ||
| {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%} | {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%} | ||
| {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%} | {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%} | ||
| {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%} | |||
| {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default).lower() in ('true', 'yes') -%} | |||
| {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%} | |||
| [request-common] | [request-common] | ||
| <= request-common-base | <= request-common-base | ||
| config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} | config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} | ||
| ... | @@ -119,7 +122,11 @@ name = neo-${gen-neo-cluster-base:passwd} | ... | @@ -119,7 +122,11 @@ name = neo-${gen-neo-cluster-base:passwd} |
| return = | return = | ||
| zope-address-list | zope-address-list | ||
| hosts-dict | hosts-dict | ||
| config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }} | {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%} | ||
| {% if has_jupyter -%} | |||
| {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} | |||
|
|||
| {% endif -%} | |||
| config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }} | |||
| config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} | config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} | ||
| config-cloudooo-url = ${request-cloudooo:connection-url} | config-cloudooo-url = ${request-cloudooo:connection-url} | ||
| config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} | config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} | ||
| ... | @@ -150,10 +157,17 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} | ... | @@ -150,10 +157,17 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} |
| software-type = zope | software-type = zope | ||
| {% set zope_family_dict = {} -%} | {% set zope_family_dict = {} -%} | ||
| {% set jupyter_zope_family_default = [] -%} | |||
| {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%} | {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%} | ||
| {% set partition_name = 'zope-' ~ custom_name -%} | {% set partition_name = 'zope-' ~ custom_name -%} | ||
| {% set section_name = 'request-' ~ partition_name -%} | {% set section_name = 'request-' ~ partition_name -%} | ||
| {% do zope_family_dict.setdefault(zope_parameter_dict.get('family', 'default'), []).append(section_name) -%} | {% set zope_family = zope_parameter_dict.get('family', 'default') -%} | ||
| {# # default jupyter zope family is first zope family. -#} | |||
| {# # use list.append() to update it, because in jinja2 set changes only local scope. -#} | |||
| {% if not jupyter_zope_family_default -%} | |||
| {% do jupyter_zope_family_default.append(zope_family) -%} | |||
| {% endif -%} | |||
| {% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%} | |||
| [{{ section_name }}] | [{{ section_name }}] | ||
| <= request-zope-base | <= request-zope-base | ||
| name = {{ partition_name }} | name = {{ partition_name }} | ||
| ... | @@ -168,6 +182,12 @@ config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} | ... | @@ -168,6 +182,12 @@ config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} |
| config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} | config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} | ||
| {% endfor -%} | {% endfor -%} | ||
| {# if not explicitly configured, connect jupyter to first zope family, which -#} | |||
| {# will be 'default' if zope families are not configured also -#} | |||
| {% if not jupyter_zope_family and jupyter_zope_family_default -%} | |||
| {% set jupyter_zope_family = jupyter_zope_family_default[0] -%} | |||
| {% endif -%} | |||
| {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#} | {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#} | ||
| {% set zope_address_list_id_dict = {} -%} | {% set zope_address_list_id_dict = {} -%} | ||
| {% set zope_family_parameter_dict = {} -%} | {% set zope_family_parameter_dict = {} -%} | ||
| ... | @@ -190,6 +210,20 @@ config-url = ${request-balancer:connection-{{ family_name }}-v6} | ... | @@ -190,6 +210,20 @@ config-url = ${request-balancer:connection-{{ family_name }}-v6} |
| {% endif -%} | {% endif -%} | ||
| {% endfor -%} | {% endfor -%} | ||
| {% if has_jupyter -%} | |||
| {# request jupyter connected to balancer of proper zope family -#} | |||
| {{ request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }} | |||
| {% if has_frontend -%} | |||
| [frontend-jupyter] | |||
| <= request-frontend-base | |||
| name = frontend-jupyter | |||
| config-url = ${request-jupyter:connection-url} | |||
| {# # override jupyter-url in publish_dict with frontend address -#} | |||
| {% do publish_dict.__setitem__('jupyter-url', '${frontend-jupyter:connection-site_url}') -%} | |||
| {% endif -%} | |||
| {%- endif %} | |||
| {% set balancer_dict = slapparameter_dict.get('balancer', {}) -%} | {% set balancer_dict = slapparameter_dict.get('balancer', {}) -%} | ||
| [request-balancer] | [request-balancer] | ||
| <= request-common | <= request-common | ||
| ... | ... | ||
