Commit db4f576d authored by Xavier Thompson's avatar Xavier Thompson

software/jupyter: Add frontend

See merge request nexedi/slapos!886
parents 1871c204 8e3ac372
Pipeline #13158 failed with stage
in 0 seconds
......@@ -13,17 +13,17 @@
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-jupyter-notebook]
[instance]
filename = instance.cfg.in
md5sum = 659e0d583acd70ebb8d815c22735d91b
md5sum = f33fc954eba47b5ddbebc72c453304e3
[instance-jupyter]
filename = instance-jupyter.cfg.in
md5sum = 9340498841caa5771f40f8c9e561eacd
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
md5sum = 4514df152740dbd7990da9705f40e978
[jupyter-set-password]
filename = jupyter_set_password.cgi.jinja
md5sum = b8d31441780b524a7e52d1710dd78385
md5sum = 6e01da7d35c1d65d4a450b0f011b296d
[erp5-kernel]
filename = ERP5kernel.py
......
{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
[buildout]
parts =
instance
publish-connection-parameter
jupyter-notebook-config
erp5-kernel
kernel-json
custom-js
frontend-promise
{% if additional_frontend %}
frontend-additional-promise
{% endif %}
monitor-base
extends =
{{ monitor_template_rendered }}/template-monitor.cfg
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slapconfiguration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# ERP5 URL to use in Jupyter by default
# default value is empty - which means no default ERP5 URL
configuration.erp5-url =
[instance-parameter]
port = 8888
host = ${slapconfiguration:ipv6-random}
cert_file = ${generate-certificate:cert_file}
key_file = ${generate-certificate:key_file}
logfile = ${directory:log}/jupyter_notebook.log
notebook_dir = ${directory:notebook_dir}
[slap-parameter]
frontend-software-type = RootSoftwareInstance
frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
frontend-instance-guid =
frontend-instance-name = Jupyter Frontend
frontend-additional-software-type = RootSoftwareInstance
frontend-additional-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
frontend-additional-instance-guid =
frontend-additional-instance-name = Jupyter Frontend Additional
[dynamic-jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 0644
[generate-certificate]
; TODO: there is a slapos recipe to generate certificates. Use it instead
recipe = plone.recipe.command
command =
if [ ! -e ${instance-parameter:key_file} ]
then
{{ openssl_output }} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
-newkey rsa:1024 -keyout ${instance-parameter:key_file} \
-out ${instance-parameter:cert_file}
fi
update-command = ${:command}
cert_file = ${directory:etc}/jupyter_cert.crt
key_file = ${directory:etc}/jupyter_cert.key
[instance]
recipe = slapos.cookbook:wrapper
command-line =
{{ bin_directory }}/jupyter-lab
--no-browser
--ip=${instance-parameter:host}
--port=${instance-parameter:port}
--port-retries=0
--certfile=${instance-parameter:cert_file}
--keyfile=${instance-parameter:key_file}
--notebook-dir=${instance-parameter:notebook_dir}
--log-level="DEBUG"
wrapper-path = ${directory:service}/jupyter-lab
environment =
JUPYTER_PATH=${directory:jupyter_dir}
JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
LANG=C.UTF-8
[jupyter-password]
recipe = slapos.cookbook:generate.password
bytes = 10
[jupyter-notebook-config]
<= dynamic-jinja2-template-base
template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
mode = 0744
context =
key password jupyter-password:passwd
raw gcc_location {{ gcc_location }}
raw cythonplus_repository {{ cythonplus_repository }}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
script = ${:etc}/run/
service = ${:etc}/service
log = ${:var}/log
notebook_dir = ${:var}/notebooks
# Add folders to explicitly define jupyter directory
jupyter_dir = ${:home}/jupyter
jupyter_config_dir = ${:jupyter_dir}/etc
jupyter_kernel_dir = ${:jupyter_dir}/kernels
jupyter_runtime_dir = ${:jupyter_dir}/runtime
jupyter_custom_dir = ${:jupyter_config_dir}/custom
jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
[request-slave-frontend-base]
recipe = slapos.cookbook:requestoptional
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
slave = true
config-type = notebook
config-url = https://[${instance-parameter:host}]:${instance-parameter:port}
return = secure_access
[request-slave-frontend]
<= request-slave-frontend-base
software-url = ${slap-parameter:frontend-software-url}
software-type = ${slap-parameter:frontend-software-type}
name = ${slap-parameter:frontend-instance-name}
sla-instance_guid = ${slap-parameter:frontend-instance-guid}
[frontend-promise]
<= monitor-promise-base
module = check_url_available
name = frontend_promise.py
config-url = ${publish-connection-parameter:url}
{% if additional_frontend %}
[request-slave-frontend-additional]
<= request-slave-frontend-base
software-url = ${slap-parameter:frontend-additional-software-url}
software-type = ${slap-parameter:frontend-additional-software-type}
name = ${slap-parameter:frontend-additional-instance-name}
sla-instance_guid = ${slap-parameter:frontend-additional-instance-guid}
[frontend-additional-promise]
<= monitor-promise-base
module = check_url_available
name = frontend_additional_promise.py
config-url = ${publish-connection-parameter:url-additional}
{% endif %}
[monitor-instance-parameter]
monitor-base-url = ${monitor-frontend-promise:url}
# In case you're using a developer instance you should edit these in:
# monitor-base-url = ${monitor-httpd-conf-parameter:url}
# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
instance-configuration =
raw jupyter-password ${jupyter-password:passwd}
[publish-connection-parameter]
recipe = slapos.cookbook:publish.serialised
jupyter-classic-url = ${request-slave-frontend:connection-secure_access}/tree
url = ${:jupyter-classic-url}
jupyterlab-url = ${request-slave-frontend:connection-secure_access}/lab
{% if additional_frontend %}
jupyter-classic-url-additional = ${request-slave-frontend-additional:connection-secure_access}/tree
url-additional = ${:jupyter-classic-url-additional}
jupyterlab-url-additional = ${request-slave-frontend-additional:connection-secure_access}/lab
{% endif %}
password = ${jupyter-password:passwd}
[erp5-kernel]
recipe = slapos.cookbook:symbolic.link
link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
template = {{ kernel_json_location }}/{{ kernel_json_filename }}
rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
context =
raw python_executable {{ python_executable }}
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
[custom-js]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:jupyter_custom_dir}
link-binary = {{ custom_js_location }}/custom.js
[buildout]
parts =
instance
jupyter_notebook
read-knowledge0
publish-connection-parameter
jupyter-notebook-config
erp5-kernel
kernel-json
custom-js
monitor-base
extends =
{{ monitor_template_rendered }}/template-monitor.cfg
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[switch_softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = $${:jupyter}
jupyter = instance-jupyter:rendered
RootSoftwareInstance = $${:default}
[slapconfiguration]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# ERP5 URL to use in Jupyter by default
# default value is empty - which means no default ERP5 URL
configuration.erp5-url =
[instance-parameter]
port = 8888
host = ${slapconfiguration:ipv6-random}
cert_file = ${generate-certificate:cert_file}
key_file = ${generate-certificate:key_file}
logfile = ${directory:log}/jupyter_notebook.log
notebook_dir = ${directory:notebook_dir}
[dynamic-jinja2-template-base]
[instance-jupyter]
recipe = slapos.recipe.template:jinja2
filename = instance-jupyter.cfg.in
template = ${instance:_profile_base_location_}/$${:filename}
rendered = $${buildout:directory}/template.cfg
mode = 0644
[generate-certificate]
; TODO: there is a slapos recipe to generate certificates. Use it instead
recipe = plone.recipe.command
command =
if [ ! -e ${instance-parameter:key_file} ]
then
{{ openssl_output }} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
-newkey rsa:1024 -keyout ${instance-parameter:key_file} \
-out ${instance-parameter:cert_file}
fi
update-command = ${:command}
cert_file = ${directory:etc}/jupyter_cert.crt
key_file = ${directory:etc}/jupyter_cert.key
[instance]
recipe = slapos.cookbook:wrapper
command-line =
{{ bin_directory }}/jupyter-lab
--no-browser
--ip=${instance-parameter:host}
--port=${instance-parameter:port}
--port-retries=0
--certfile=${instance-parameter:cert_file}
--keyfile=${instance-parameter:key_file}
--notebook-dir=${instance-parameter:notebook_dir}
--log-level="DEBUG"
wrapper-path = ${directory:service}/jupyter-lab
environment =
JUPYTER_PATH=${directory:jupyter_dir}
JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
LANG=C.UTF-8
[jupyter-notebook-config]
<= dynamic-jinja2-template-base
template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
mode = 0744
context =
raw config_cfg ${buildout:directory}/knowledge0.cfg
raw gcc_location {{ gcc_location }}
raw cythonplus_repository {{ cythonplus_repository }}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
script = ${:etc}/run/
service = ${:etc}/service
log = ${:var}/log
notebook_dir = ${:var}/notebooks
# Add folders to explicitly define jupyter directory
jupyter_dir = ${:home}/jupyter
jupyter_config_dir = ${:jupyter_dir}/etc
jupyter_kernel_dir = ${:jupyter_dir}/kernels
jupyter_runtime_dir = ${:jupyter_dir}/runtime
jupyter_custom_dir = ${:jupyter_config_dir}/custom
jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
[jupyter_notebook]
# This part is called like this because knowledge0.write uses the part name for
# the section name in the config file.
recipe = slapos.cookbook:zero-knowledge.write
password =
filename = knowledge0.cfg
[read-knowledge0]
recipe = slapos.cookbook:zero-knowledge.read
filename = knowledge0.cfg
password =
[monitor-instance-parameter]
monitor-base-url = ${monitor-frontend-promise:url}
# In case you're using a developer instance you should edit these in:
# monitor-base-url = ${monitor-httpd-conf-parameter:url}
# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
instance-configuration =
raw jupyter-password ${read-knowledge0:password}
[publish-connection-parameter]
recipe = slapos.cookbook:publish.serialised
jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
url = ${:jupyter-classic-url}
jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
[erp5-kernel]
recipe = slapos.cookbook:symbolic.link
link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
template = {{ kernel_json_location }}/{{ kernel_json_filename }}
rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
context =
raw python_executable {{ python_executable }}
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
[custom-js]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:jupyter_custom_dir}
link-binary = {{ custom_js_location }}/custom.js
key slapparameter_dict slap-configuration:configuration
raw bin_directory ${buildout:bin-directory}
raw develop_eggs_directory ${buildout:develop-eggs-directory}
raw eggs_directory ${buildout:eggs-directory}
raw openssl_output ${openssl-output:openssl}
raw python_executable ${jupyter:python_executable}
raw jupyter_config_location ${jupyter-notebook-config:location}
raw jupyter_config_filename ${jupyter-notebook-config:filename}
raw erp5_kernel_location ${erp5-kernel:location}
raw erp5_kernel_filename ${erp5-kernel:filename}
raw kernel_json_location ${kernel-json:location}
raw kernel_json_filename ${kernel-json:filename}
raw custom_js_location ${custom-js:location}
raw custom_js_filename ${custom-js:filename}
raw monitor_template_rendered ${buildout:directory}
raw cythonplus_repository ${cythonplus-repository:location}
raw gcc_location ${gcc:prefix}
......@@ -3,39 +3,12 @@
things. It is run by IPython hence why it can use functions like get_config().
'''
import configparser
import random
from notebook.auth import passwd
import os
def random_password(length = 10):
result = ""
for i in range(0, length):
result = result + chr(random.randint(0, 25) + ord('a'))
return result
knowledge_0 = '{{ config_cfg }}'
if not os.path.exists(knowledge_0):
print("Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>")
exit(0)
c = get_config()
parser = configparser.ConfigParser()
parser.read(knowledge_0)
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("jupyter_notebook", "password") or \
parser.get("jupyter_notebook", "password") == "":
parser.set("jupyter_notebook", "password", random_password())
c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
with open(knowledge_0, 'w') as file:
parser.write(file)
c.NotebookApp.password = passwd("{{ password }}")
os.environ['PATH'] = "{{ gcc_location }}/bin" + os.pathsep + os.environ['PATH']
os.environ['PYTHONPATH'] = "{{ cythonplus_repository }}" + os.pathsep + os.environ['PYTHONPATH']
#!{{ python_executable }}
import cgi
import cgitb
import ConfigParser
import os
import re
import subprocess
from IPython.lib import passwd
#cgitb.enable(display=0, logdir="/tmp/cgi.log")
cgitb.enable()
form = cgi.FieldStorage()
config_file = "{{ config_cfg }}"
if not os.path.exists(config_file):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
exit(0)
parser = ConfigParser.ConfigParser()
parser.read(config_file)
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("jupyter_notebook", "password"):
parser.set("jupyter_notebook", "password", "")
if "password" in form:
parser.set("jupyter_notebook", "password", passwd(form["password"].value))
# subprocess.call('{{ httpd_graceful }}')
# TODO: we should restart jupyter
with open(config_file, 'w') as file:
parser.write(file)
# TODO cleanup
print "<html><head>"
print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"static/style.css\">"
print "</head><body>"
print "<h1>Jupyter Notebook Password :</h1>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
print """<div class="pure-control-group">
<label for="password">Password*:</label>
<input placeholder="Set your password" type="password" name="password" id="password"></br>
</div><div class="pure-control-group">
<label for="password">Verify Password*:</label>
<input placeholder="Verify password" type="password" name="password_2" id="password_2"></br>
</div><p id="validate-status" style="color:red"></p>
<div class="pure-controls">
<button id="register-button" type="submit" class="pure-button pure-button-primary" disabled>Access</button></div>
</form>
<script type="text/javascript" src="static/jquery-1.10.2.min.js"></script>
<script type="text/javascript" src="static/monitor-register.js"></script>
</body></html>
"""
......@@ -10,7 +10,7 @@ parts +=
slapos-cookbook
jupyter
jupyter-notebook-initialized-scripts
instance-jupyter-notebook
instance
[python]
part = python3
......@@ -31,9 +31,6 @@ mode = 0644
[jupyter-notebook-config]
<= download-file-base
[jupyter-set-password]
<= download-file-base
[erp5-kernel]
<= download-file-base
......@@ -43,30 +40,11 @@ mode = 0644
[custom-js]
<= download-file-base
[instance-jupyter-notebook]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template.cfg
[instance]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key openssl_output openssl-output:openssl
key python_executable jupyter:python_executable
key jupyter_config_location jupyter-notebook-config:location
key jupyter_config_filename jupyter-notebook-config:filename
key jupyter_set_password_location jupyter-set-password:location
key jupyter_set_password_filename jupyter-set-password:filename
key erp5_kernel_location erp5-kernel:location
key erp5_kernel_filename erp5-kernel:filename
key kernel_json_location kernel-json:location
key kernel_json_filename kernel-json:filename
key custom_js_location custom-js:location
key custom_js_filename custom-js:filename
key monitor_template_rendered buildout:directory
key cythonplus_repository cythonplus-repository:location
key gcc_location gcc:prefix
[versions]
Pygments = 2.7.2
......
......@@ -48,10 +48,14 @@ class TestJupyter(InstanceTestCase):
except Exception as e:
self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
self.assertTrue('password' in connection_dict)
password = connection_dict['password']
self.assertEqual(
{
'jupyter-classic-url': 'https://[%s]:8888/tree' % (self._ipv6_address, ),
'jupyterlab-url': 'https://[%s]:8888/lab' % (self._ipv6_address, ),
'password': '%s' % (password, ),
'url': 'https://[%s]:8888/tree' % (self._ipv6_address, )
},
connection_dict
......@@ -79,3 +83,89 @@ class TestJupyter(InstanceTestCase):
[http.client.FOUND, True, '/login?next=%2Flab'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestJupyterPassword(InstanceTestCase):
def test(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
self.assertTrue('_' in parameter_dict)
try:
connection_dict = json.loads(parameter_dict['_'])
except Exception as e:
self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
url = connection_dict['url']
with requests.Session() as s:
resp = s.get(url, verify=False)
result = s.post(
resp.url,
verify = False,
data={"_xsrf": s.cookies["_xsrf"], "password": connection_dict['password']}
)
self.assertEqual(
[http.client.OK, url],
[result.status_code, result.url]
)
class TestJupyterAdditional(InstanceTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'frontend-additional-instance-guid': 'SOMETHING'
}
def test(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
self.assertTrue('_' in parameter_dict)
try:
connection_dict = json.loads(parameter_dict['_'])
except Exception as e:
self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
result = requests.get(
connection_dict['url'], verify=False, allow_redirects=False)
self.assertEqual(
[http.client.FOUND, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
result = requests.get(
connection_dict['jupyter-classic-url'],
verify=False, allow_redirects=False)
self.assertEqual(
[http.client.FOUND, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
result = requests.get(
connection_dict['jupyterlab-url'],
verify=False, allow_redirects=False)
self.assertEqual(
[http.client.FOUND, True, '/login?next=%2Flab'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
result = requests.get(
connection_dict['url-additional'], verify=False, allow_redirects=False)
self.assertEqual(
[http.client.FOUND, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
result = requests.get(
connection_dict['jupyter-classic-url-additional'],
verify=False, allow_redirects=False)
self.assertEqual(
[http.client.FOUND, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
result = requests.get(
connection_dict['jupyterlab-url-additional'],
verify=False, allow_redirects=False)
self.assertEqual(
[http.client.FOUND, True, '/login?next=%2Flab'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment