software/jupyter: Add frontend

software/jupyter/buildout.hash.cfg

@@ -13,9 +13,13 @@
 # section inheritance (< = ...) are NOT supported (but you should really
 # not need these here).
 
-[instance-jupyter-notebook]
+[instance]
 filename = instance.cfg.in
-md5sum = 659e0d583acd70ebb8d815c22735d91b
+md5sum = 13fcabee7821bc4cdda2f99e4190be73
+
+[instance-jupyter]
+filename = instance-jupyter.cfg.in
+md5sum = 521d5a742d5304a6f88c000db28d6e28
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja

software/jupyter/instance-jupyter.cfg.in

+{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
+
+[buildout]
+parts =
+  instance
+  jupyter_notebook
+  read-knowledge0
+  publish-connection-parameter
+  jupyter-notebook-config
+  erp5-kernel
+  kernel-json
+  custom-js
+  frontend-promise
+  {% if additional_frontend %}
+  frontend-additional-promise
+  {% endif %}
+  monitor-base
+extends =
+  {{ monitor_template_rendered }}/template-monitor.cfg
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
+[instance-parameter]
+port = 8888
+host = ${slapconfiguration:ipv6-random}
+cert_file = ${generate-certificate:cert_file}
+key_file = ${generate-certificate:key_file}
+logfile = ${directory:log}/jupyter_notebook.log
+notebook_dir = ${directory:notebook_dir}
+
+[slap-parameter]
+frontend-software-type = RootSoftwareInstance
+frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+frontend-instance-guid =
+frontend-instance-name = Jupyter Frontend
+frontend-additional-software-type = RootSoftwareInstance
+frontend-additional-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+frontend-additional-instance-guid =
+frontend-additional-instance-name = Jupyter Frontend Additional
+
+[dynamic-jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 0644
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${instance-parameter:key_file} ]
+  then
+    {{ openssl_output }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
+      -out ${instance-parameter:cert_file}
+  fi
+update-command = ${:command}
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ bin_directory }}/jupyter-lab
+  --no-browser
+  --ip=${instance-parameter:host}
+  --port=${instance-parameter:port}
+  --port-retries=0
+  --certfile=${instance-parameter:cert_file}
+  --keyfile=${instance-parameter:key_file}
+  --notebook-dir=${instance-parameter:notebook_dir}
+  --log-level="DEBUG"
+wrapper-path = ${directory:service}/jupyter-lab
+environment =
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
+  LANG=C.UTF-8
+
+[jupyter-notebook-config]
+<= dynamic-jinja2-template-base
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
+mode = 0744
+context =
+  raw config_cfg ${buildout:directory}/knowledge0.cfg
+  raw gcc_location {{ gcc_location }}
+  raw cythonplus_repository {{ cythonplus_repository }}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+notebook_dir = ${:var}/notebooks
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[jupyter_notebook]
+# This part is called like this because knowledge0.write uses the part name for
+# the section name in the config file.
+recipe = slapos.cookbook:zero-knowledge.write
+password = 
+filename = knowledge0.cfg
+
+[read-knowledge0]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = knowledge0.cfg
+password =
+
+[request-slave-frontend-base]
+recipe = slapos.cookbook:requestoptional
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+slave = true
+config-type = notebook
+config-url = https://[${instance-parameter:host}]:${instance-parameter:port}
+return = secure_access
+
+[request-slave-frontend]
+<= request-slave-frontend-base
+software-url = ${slap-parameter:frontend-software-url}
+software-type = ${slap-parameter:frontend-software-type}
+name = ${slap-parameter:frontend-instance-name}
+sla-instance_guid = ${slap-parameter:frontend-instance-guid}
+
+[frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = frontend_promise.py
+config-url = ${publish-connection-parameter:url}
+
+{% if additional_frontend %}
+[request-slave-frontend-additional]
+<= request-slave-frontend-base
+software-url = ${slap-parameter:frontend-additional-software-url}
+software-type = ${slap-parameter:frontend-additional-software-type}
+name = ${slap-parameter:frontend-additional-instance-name}
+sla-instance_guid = ${slap-parameter:frontend-additional-instance-guid}
+
+[frontend-additional-promise]
+<= monitor-promise-base
+module = check_url_available
+name = frontend_additional_promise.py
+config-url = ${publish-connection-parameter:url-additional}
+{% endif %}
+
+[monitor-instance-parameter]
+monitor-base-url = ${monitor-frontend-promise:url}
+# In case you're using a developer instance you should edit these in:
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
+# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
+instance-configuration =
+  raw jupyter-password ${read-knowledge0:password}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+jupyter-classic-url = ${request-slave-frontend:connection-secure_access}/tree
+url = ${:jupyter-classic-url}
+jupyterlab-url = ${request-slave-frontend:connection-secure_access}/lab
+{% if additional_frontend %}
+jupyter-classic-url-additional = ${request-slave-frontend-additional:connection-secure_access}/tree
+url-additional = ${:jupyter-classic-url-additional}
+jupyterlab-url-additional = ${request-slave-frontend-additional:connection-secure_access}/lab
+{% endif %}
+password = ${read-knowledge0:password}
+
+[erp5-kernel]
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js

software/jupyter/instance.cfg.in

 [buildout]
 parts =
-  instance
-  jupyter_notebook
-  read-knowledge0
-  publish-connection-parameter
-  jupyter-notebook-config
-  erp5-kernel
-  kernel-json
-  custom-js
-  monitor-base
-extends =
-  {{ monitor_template_rendered }}/template-monitor.cfg
+  switch_softwaretype
 
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
 
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
+[switch_softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default = $${:jupyter}
+jupyter = instance-jupyter:rendered
+RootSoftwareInstance = $${:default}
 
-[slapconfiguration]
+[slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
-computer = ${slap-connection:computer-id}
-partition = ${slap-connection:partition-id}
-url = ${slap-connection:server-url}
-key = ${slap-connection:key-file}
-cert = ${slap-connection:cert-file}
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
 
-# ERP5 URL to use in Jupyter by default
-# default value is empty - which means no default ERP5 URL
-configuration.erp5-url =
-
-[instance-parameter]
-port = 8888
-host = ${slapconfiguration:ipv6-random}
-cert_file = ${generate-certificate:cert_file}
-key_file = ${generate-certificate:key_file}
-logfile = ${directory:log}/jupyter_notebook.log
-notebook_dir = ${directory:notebook_dir}
-
-[dynamic-jinja2-template-base]
+[instance-jupyter]
 recipe = slapos.recipe.template:jinja2
+filename = instance-jupyter.cfg.in
+template = ${instance:_profile_base_location_}/$${:filename}
+rendered = $${buildout:directory}/template.cfg
 mode = 0644
-
-[generate-certificate]
-; TODO: there is a slapos recipe to generate certificates. Use it instead
-recipe = plone.recipe.command
-command =
-  if [ ! -e ${instance-parameter:key_file} ]
-  then
-    {{ openssl_output }} req -x509 -nodes -days 3650 \
-      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
-      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
-      -out ${instance-parameter:cert_file}
-  fi
-update-command = ${:command}
-cert_file = ${directory:etc}/jupyter_cert.crt
-key_file = ${directory:etc}/jupyter_cert.key
-
-[instance]
-recipe = slapos.cookbook:wrapper
-command-line =
-  {{ bin_directory }}/jupyter-lab
-  --no-browser
-  --ip=${instance-parameter:host}
-  --port=${instance-parameter:port}
-  --port-retries=0
-  --certfile=${instance-parameter:cert_file}
-  --keyfile=${instance-parameter:key_file}
-  --notebook-dir=${instance-parameter:notebook_dir}
-  --log-level="DEBUG"
-wrapper-path = ${directory:service}/jupyter-lab
-environment =
-  JUPYTER_PATH=${directory:jupyter_dir}
-  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
-  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
-  LANG=C.UTF-8
-
-[jupyter-notebook-config]
-<= dynamic-jinja2-template-base
-template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
-rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
-mode = 0744
-context =
-  raw config_cfg ${buildout:directory}/knowledge0.cfg
-  raw gcc_location {{ gcc_location }}
-  raw cythonplus_repository {{ cythonplus_repository }}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-home = ${buildout:directory}
-etc = ${:home}/etc
-var = ${:home}/var
-script = ${:etc}/run/
-service = ${:etc}/service
-log = ${:var}/log
-notebook_dir = ${:var}/notebooks
-# Add folders to explicitly define jupyter directory
-jupyter_dir = ${:home}/jupyter
-jupyter_config_dir = ${:jupyter_dir}/etc
-jupyter_kernel_dir = ${:jupyter_dir}/kernels
-jupyter_runtime_dir = ${:jupyter_dir}/runtime
-jupyter_custom_dir = ${:jupyter_config_dir}/custom
-jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
-erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
-
-[jupyter_notebook]
-# This part is called like this because knowledge0.write uses the part name for
-# the section name in the config file.
-recipe = slapos.cookbook:zero-knowledge.write
-password = 
-filename = knowledge0.cfg
-
-[read-knowledge0]
-recipe = slapos.cookbook:zero-knowledge.read
-filename = knowledge0.cfg
-password =
-
-[monitor-instance-parameter]
-monitor-base-url = ${monitor-frontend-promise:url}
-# In case you're using a developer instance you should edit these in:
-# monitor-base-url = ${monitor-httpd-conf-parameter:url}
-# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
-# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
-instance-configuration =
-  raw jupyter-password ${read-knowledge0:password}
-
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish.serialised
-jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
-url = ${:jupyter-classic-url}
-jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
-
-[erp5-kernel]
-recipe = slapos.cookbook:symbolic.link
-link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
-target-directory = ${directory:erp5_kernel_dir}
-
-[kernel-json]
-<= dynamic-jinja2-template-base
-template = {{ kernel_json_location }}/{{ kernel_json_filename }}
-rendered = ${directory:erp5_kernel_dir}/kernel.json
-# Use python2.7 executable bin file for kernel config
 context =
-  raw python_executable {{ python_executable }}
-  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
-  key erp5_url slapconfiguration:configuration.erp5-url 
-  raw display_name ERP5
-  raw language_name python
-
-[custom-js]
-recipe = slapos.cookbook:symbolic.link
-target-directory = ${directory:jupyter_custom_dir}
-link-binary = {{ custom_js_location }}/custom.js
+  key slapparameter_dict slap-configuration:configuration
+  raw bin_directory ${buildout:bin-directory}
+  raw develop_eggs_directory ${buildout:develop-eggs-directory}
+  raw eggs_directory ${buildout:eggs-directory}
+  raw openssl_output ${openssl-output:openssl}
+  raw python_executable ${jupyter:python_executable}
+  raw jupyter_config_location ${jupyter-notebook-config:location}
+  raw jupyter_config_filename ${jupyter-notebook-config:filename}
+  raw jupyter_set_password_location ${jupyter-set-password:location}
+  raw jupyter_set_password_filename ${jupyter-set-password:filename}
+  raw erp5_kernel_location ${erp5-kernel:location}
+  raw erp5_kernel_filename ${erp5-kernel:filename}
+  raw kernel_json_location ${kernel-json:location}
+  raw kernel_json_filename ${kernel-json:filename}
+  raw custom_js_location ${custom-js:location}
+  raw custom_js_filename ${custom-js:filename}
+  raw monitor_template_rendered ${buildout:directory}
+  raw cythonplus_repository ${cythonplus-repository:location}
+  raw gcc_location ${gcc:prefix}

software/jupyter/software.cfg

@@ -10,7 +10,7 @@ parts +=
   slapos-cookbook
   jupyter
   jupyter-notebook-initialized-scripts
-  instance-jupyter-notebook
+  instance
 
 [python]
 part = python3
@@ -43,30 +43,11 @@ mode = 0644
 [custom-js]
 <= download-file-base
 
-[instance-jupyter-notebook]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/${:filename}
-rendered = ${buildout:directory}/template.cfg
+[instance]
+recipe = slapos.recipe.template
+url =  ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
 mode = 0644
-context =
-  key bin_directory buildout:bin-directory
-  key develop_eggs_directory buildout:develop-eggs-directory
-  key eggs_directory buildout:eggs-directory
-  key openssl_output openssl-output:openssl
-  key python_executable jupyter:python_executable
-  key jupyter_config_location jupyter-notebook-config:location
-  key jupyter_config_filename jupyter-notebook-config:filename
-  key jupyter_set_password_location jupyter-set-password:location
-  key jupyter_set_password_filename jupyter-set-password:filename
-  key erp5_kernel_location erp5-kernel:location
-  key erp5_kernel_filename erp5-kernel:filename
-  key kernel_json_location kernel-json:location
-  key kernel_json_filename kernel-json:filename
-  key custom_js_location custom-js:location
-  key custom_js_filename custom-js:filename
-  key monitor_template_rendered buildout:directory
-  key cythonplus_repository cythonplus-repository:location
-  key gcc_location gcc:prefix
 
 [versions]
 Pygments = 2.7.2

software/jupyter/test/test.py

@@ -48,10 +48,14 @@ class TestJupyter(InstanceTestCase):
     except Exception as e:
       self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
 
+    self.assertTrue('password' in connection_dict)
+    password = connection_dict['password']
+
     self.assertEqual(
       {
         'jupyter-classic-url': 'https://[%s]:8888/tree' % (self._ipv6_address, ),
         'jupyterlab-url': 'https://[%s]:8888/lab' % (self._ipv6_address, ),
+        'password': '%s' % (password, ),
         'url': 'https://[%s]:8888/tree' % (self._ipv6_address, )
       },
       connection_dict
@@ -79,3 +83,89 @@ class TestJupyter(InstanceTestCase):
       [http.client.FOUND, True, '/login?next=%2Flab'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
+
+
+class TestJupyterPassword(InstanceTestCase):
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    url = connection_dict['url']
+    with requests.Session() as s:
+      resp = s.get(url, verify=False)
+      result = s.post(
+        resp.url,
+        verify = False,
+        data={"_xsrf": s.cookies["_xsrf"], "password": connection_dict['password']}
+      )
+      self.assertEqual(
+        [http.client.OK, url],
+        [result.status_code, result.url]
+      )
+
+
+class TestJupyterAdditional(InstanceTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'frontend-additional-instance-guid': 'SOMETHING'
+    }
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    result = requests.get(
+      connection_dict['url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyter-classic-url'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyterlab-url'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Flab'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['url-additional'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyter-classic-url-additional'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
+    result = requests.get(
+      connection_dict['jupyterlab-url-additional'],
+      verify=False, allow_redirects=False)
+    self.assertEqual(
+      [http.client.FOUND, True, '/login?next=%2Flab'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )