Commit eb896b21 authored by Klaus Wölfel's avatar Klaus Wölfel

jupyter: store parameters for notebook in notebook context

parent 6394ffbf
...@@ -28,6 +28,19 @@ from ipykernel.jsonutil import json_clean, encode_images ...@@ -28,6 +28,19 @@ from ipykernel.jsonutil import json_clean, encode_images
import threading import threading
display_data_wrapper_lock = threading.Lock() display_data_wrapper_lock = threading.Lock()
# Well known unserializable types
from Record import Record
well_known_unserializable_type_tuple = (ModuleType, Record)
# ZBigArray may not be availableK
try:
from wendelin.bigarray.array_zodb import ZBigArray
# FIXME ZBigArrays are regular ZODB objects and must be serializable
# FIXME the bug is probably in CanSerialize()
# FIXME -> see https://lab.nexedi.com/nexedi/erp5/commit/5fb16acd#note_33582 for details
well_known_unserializable_type_tuple = tuple(list(well_known_unserializable_type_tuple) + [ZBigArray])
except ImportError:
pass
def Base_executeJupyter(self, python_expression=None, reference=None, \ def Base_executeJupyter(self, python_expression=None, reference=None, \
title=None, request_reference=False, **kw): title=None, request_reference=False, **kw):
# Check permissions for current user and display message to non-authorized user # Check permissions for current user and display message to non-authorized user
...@@ -66,14 +79,22 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \ ...@@ -66,14 +79,22 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \
title=title, title=title,
reference=reference, reference=reference,
batch_mode=True) batch_mode=True)
# Add new Data Notebook Line to the Data Notebook # By default, store_history is True
data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine( store_history = kw.get('store_history', True)
# Klaus
store_history = False
data_notebook_line = None
if store_history:
# Add new Data Notebook Line to the Data Notebook
data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
notebook_code=python_expression, notebook_code=python_expression,
batch_mode=True) batch_mode=True)
# Gets the context associated to the data notebook being used # Gets the context associated to the data notebook being used
old_notebook_context = data_notebook.getNotebookContext() old_notebook_context = None
if data_notebook.hasNotebookContext():
old_notebook_context = data_notebook.getNotebookContext().copy()
if not old_notebook_context: if not old_notebook_context:
old_notebook_context = self.Base_createNotebookContext() old_notebook_context = self.Base_createNotebookContext()
...@@ -83,6 +104,9 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \ ...@@ -83,6 +104,9 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \
new_notebook_context = final_result['notebook_context'] new_notebook_context = final_result['notebook_context']
# Klaus
new_notebook_context["variables"] = old_notebook_context["variables"]
result = { result = {
u'code_result': final_result['result_string'], u'code_result': final_result['result_string'],
u'print_result': final_result['print_result'], u'print_result': final_result['print_result'],
...@@ -110,6 +134,7 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \ ...@@ -110,6 +134,7 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \
transaction.abort() transaction.abort()
exception_dict = getErrorMessageForException(self, e, new_notebook_context) exception_dict = getErrorMessageForException(self, e, new_notebook_context)
result.update(exception_dict) result.update(exception_dict)
result['notebook_context'] = None
return json.dumps(result) return json.dumps(result)
# Catch exception while seriaizing the result to be passed to jupyter frontend # Catch exception while seriaizing the result to be passed to jupyter frontend
...@@ -128,10 +153,11 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \ ...@@ -128,10 +153,11 @@ def Base_executeJupyter(self, python_expression=None, reference=None, \
u'status': u'error', u'status': u'error',
u'mime_type': result['mime_type']} u'mime_type': result['mime_type']}
serialized_result = json.dumps(result) serialized_result = json.dumps(result)
data_notebook_line.edit( if data_notebook_line is not None:
notebook_code_result = result['code_result'], data_notebook_line.edit(
mime_type = result['mime_type']) notebook_code_result = result['code_result'],
mime_type = result['mime_type'])
return serialized_result return serialized_result
...@@ -147,7 +173,7 @@ def mergeTracebackListIntoResultDict(result_dict, error_result_dict_list): ...@@ -147,7 +173,7 @@ def mergeTracebackListIntoResultDict(result_dict, error_result_dict_list):
def matplotlib_pre_run(): def matplotlib_pre_run():
matplotlib.interactive(True) matplotlib.interactive(False)
rc = {'figure.figsize': (6.0,4.0), rc = {'figure.figsize': (6.0,4.0),
'figure.facecolor': (1,1,1,0), 'figure.facecolor': (1,1,1,0),
'figure.edgecolor': (1,1,1,0), 'figure.edgecolor': (1,1,1,0),
...@@ -176,7 +202,10 @@ def matplotlib_post_run(data_list): ...@@ -176,7 +202,10 @@ def matplotlib_post_run(data_list):
class Displayhook(object): class Displayhook(object):
def hook(self, value): def hook(self, value):
if value is not None: if value is not None:
self.result = repr(value) if getattr(value, '_repr_html_', None) is not None:
self.result = {'data':{'text/html':value._repr_html_()}, 'metadata':{}}
else:
self.result = repr(value)
def pre_run(self): def pre_run(self):
self.old_hook = sys.displayhook self.old_hook = sys.displayhook
sys.displayhook = self.hook sys.displayhook = self.hook
...@@ -277,6 +306,20 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -277,6 +306,20 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
print_fixer = PrintFixer() print_fixer = PrintFixer()
environment_collector = EnvironmentParser() environment_collector = EnvironmentParser()
ast_node = import_fixer.visit(ast_node) ast_node = import_fixer.visit(ast_node)
# Whenever we have new imports we need to warn the user about the
# environment
if (import_fixer.warning_module_names != []):
warning = ("print '"
"WARNING: You imported from the modules %s without "
"using the environment object, which is not recomended. "
"Your import was automatically converted to use such method. "
"The setup functions were named as *module*_setup. "
"'") % (', '.join(import_fixer.warning_module_names))
tree = ast.parse(warning)
tree.body[0].lineno = ast_node.body[-1].lineno+5
ast_node.body.append(tree.body[0])
ast_node = print_fixer.visit(ast_node) ast_node = print_fixer.visit(ast_node)
ast.fix_missing_locations(ast_node) ast.fix_missing_locations(ast_node)
...@@ -335,6 +378,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -335,6 +378,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
'_print': CustomPrint()} '_print': CustomPrint()}
user_context.update(inject_variable_dict) user_context.update(inject_variable_dict)
user_context.update(notebook_context['variables']) user_context.update(notebook_context['variables'])
user_context['parameter_dict'] = notebook_context['parameter_dict']
# Getting the environment setup defined in the current code cell # Getting the environment setup defined in the current code cell
current_setup_dict = environment_collector.getEnvironmentSetupDict() current_setup_dict = environment_collector.getEnvironmentSetupDict()
...@@ -381,6 +425,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -381,6 +425,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
del notebook_context['setup'][key] del notebook_context['setup'][key]
# Running all the setup functions that we got # Running all the setup functions that we got
failed_setup_key_list = []
for key, value in notebook_context['setup'].iteritems(): for key, value in notebook_context['setup'].iteritems():
try: try:
code = compile(value['code'], '<string>', 'exec') code = compile(value['code'], '<string>', 'exec')
...@@ -388,12 +433,15 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -388,12 +433,15 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
# An error happened, so we show the user the stacktrace along with a # An error happened, so we show the user the stacktrace along with a
# note that the exception happened in a setup function's code. # note that the exception happened in a setup function's code.
except Exception as e: except Exception as e:
failed_setup_key_list.append(key)
if value['func_name'] in user_context: if value['func_name'] in user_context:
del user_context[value['func_name']] del user_context[value['func_name']]
error_return_dict = getErrorMessageForException(self, e, notebook_context) error_return_dict = getErrorMessageForException(self, e, notebook_context)
additional_information = "An error happened when trying to run the one of your setup functions:" additional_information = "An error happened when trying to run the one of your setup functions:"
error_return_dict['traceback'].insert(0, additional_information) error_return_dict['traceback'].insert(0, additional_information)
setup_error_return_dict_list.append(error_return_dict) setup_error_return_dict_list.append(error_return_dict)
for failed_setup_key in failed_setup_key_list:
del notebook_context['setup'][failed_setup_key]
# Iterating over envinronment.define calls captured by the environment collector # Iterating over envinronment.define calls captured by the environment collector
# that are functions and saving them as setup functions. # that are functions and saving them as setup functions.
...@@ -420,8 +468,8 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -420,8 +468,8 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
user_context['_volatile_variable_list'] += variable user_context['_volatile_variable_list'] += variable
if environment_collector.showEnvironmentSetup(): if environment_collector.showEnvironmentSetup():
inject_variable_dict.write("%s\n" % str(notebook_context['setup'])) inject_variable_dict['_print'].write("%s\n" % str(notebook_context['setup']))
# Execute the nodes with 'exec' mode # Execute the nodes with 'exec' mode
for node in to_run_exec: for node in to_run_exec:
mod = ast.Module([node]) mod = ast.Module([node])
...@@ -429,10 +477,12 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -429,10 +477,12 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
try: try:
exec(code, user_context, user_context) exec(code, user_context, user_context)
except Exception as e: except Exception as e:
error_message = getErrorMessageForException(self, e, notebook_context)
# Abort the current transaction. As a consequence, the notebook lines # Abort the current transaction. As a consequence, the notebook lines
# are not added if an exception occurs. # are not added if an exception occurs.
transaction.abort() transaction.abort()
return mergeTracebackListIntoResultDict(getErrorMessageForException(self, e, notebook_context), log(error_message)
return mergeTracebackListIntoResultDict(error_message,
setup_error_return_dict_list) setup_error_return_dict_list)
# Execute the interactive nodes with 'single' mode # Execute the interactive nodes with 'single' mode
...@@ -456,32 +506,36 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -456,32 +506,36 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context.get('_volatile_variable_list', []) volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context.get('_volatile_variable_list', [])
volatile_variable_list.append('__builtins__') volatile_variable_list.append('__builtins__')
for key, val in user_context.items(): # Klaus
if not key in globals_dict.keys() and not isinstance(val, ModuleType) and not key in volatile_variable_list: #for key, val in user_context.items():
if canSerialize(val): # if not key in globals_dict.keys() and not isinstance(val, well_known_unserializable_type_tuple) and not key in volatile_variable_list:
notebook_context['variables'][key] = val # if canSerialize(val):
else: # notebook_context['variables'][key] = val
del user_context[key] # else:
message = ( # del user_context[key]
"Cannot serialize the variable named %s whose value is %s, " # message = (
"thus it will not be stored in the context. " # "Cannot serialize the variable named %s whose value is %s, "
"You should move it's definition to a function and " # "thus it will not be stored in the context. "
"use the environment object to load it.\n" # "You should move it's definition to a function and "
) % (key, val) # "use the environment object to load it.\n"
inject_variable_dict['_print'].write(message) # ) % (key, val)
# inject_variable_dict['_print'].write(message)
# Deleting from the variable storage the keys that are not in the user # Deleting from the variable storage the keys that are not in the user
# context anymore (i.e., variables that are deleted by the user). # context anymore (i.e., variables that are deleted by the user).
for key in notebook_context['variables'].keys(): #for key in notebook_context['variables'].keys():
if not key in user_context: # if not key in user_context:
del notebook_context['variables'][key] # del notebook_context['variables'][key]
if inject_variable_dict.get('_print') is not None: if inject_variable_dict.get('_print') is not None:
output = inject_variable_dict['_print'].getCapturedOutputString() output = inject_variable_dict['_print'].getCapturedOutputString()
displayhook_result = {"data":{}, "metadata":{}} displayhook_result = {"data":{}, "metadata":{}}
if displayhook.result is not None: if displayhook.result is not None:
displayhook_result["data"]["text/plain"] = displayhook.result if isinstance(displayhook.result, str):
displayhook_result["data"]["text/plain"] = displayhook.result
elif isinstance(displayhook.result, dict):
displayhook_result = displayhook.result
result = { result = {
'result_string': output, 'result_string': output,
'print_result': {"data":{"text/plain":output}, "metadata":{}}, 'print_result': {"data":{"text/plain":output}, "metadata":{}},
...@@ -506,7 +560,7 @@ class EnvironmentDefinitionError(TypeError): ...@@ -506,7 +560,7 @@ class EnvironmentDefinitionError(TypeError):
def canSerialize(obj): def canSerialize(obj):
container_type_tuple = (list, tuple, dict, set, frozenset) container_type_tuple = (list, tuple, dict, set, frozenset)
# if object is a container, we need to check its elements for presence of # if object is a container, we need to check its elements for presence of
# objects that cannot be put inside the zodb # objects that cannot be put inside the zodb
if isinstance(obj, container_type_tuple): if isinstance(obj, container_type_tuple):
...@@ -524,7 +578,11 @@ def canSerialize(obj): ...@@ -524,7 +578,11 @@ def canSerialize(obj):
# Need to unwrap the variable, otherwise we get a TypeError, because # Need to unwrap the variable, otherwise we get a TypeError, because
# objects cannot be pickled while inside an acquisition wrapper. # objects cannot be pickled while inside an acquisition wrapper.
unwrapped_obj = Acquisition.aq_base(obj) unwrapped_obj = Acquisition.aq_base(obj)
writer = ObjectWriter(unwrapped_obj) try:
writer = ObjectWriter(unwrapped_obj)
except:
# Ignore any exceptions, otherwise Jupyter becomes permanent unusble state.
return False
for obj in writer: for obj in writer:
try: try:
writer.serialize(obj) writer.serialize(obj)
...@@ -727,6 +785,7 @@ class ImportFixer(ast.NodeTransformer): ...@@ -727,6 +785,7 @@ class ImportFixer(ast.NodeTransformer):
def __init__(self): def __init__(self):
self.import_func_dict = {} self.import_func_dict = {}
self.warning_module_names = []
def visit_FunctionDef(self, node): def visit_FunctionDef(self, node):
""" """
...@@ -835,24 +894,26 @@ class ImportFixer(ast.NodeTransformer): ...@@ -835,24 +894,26 @@ class ImportFixer(ast.NodeTransformer):
if not self.import_func_dict.get(name): if not self.import_func_dict.get(name):
final_module_names.append(name) final_module_names.append(name)
log("module_names[0]: " + module_names[0])
log("result_name: " + result_name)
if final_module_names: if final_module_names:
# try to import module before it is added to environment # try to import module before it is added to environment
# this way if user tries to import non existent module Exception # this way if user tries to import non existent module Exception
# is immediately raised and doesn't block next Jupyter cell execution # is immediately raised and doesn't block next Jupyter cell execution
exec(test_import_string) exec(test_import_string)
empty_function = self.newEmptyFunction("%s_setup" %result_name) dotless_result_name = ""
for character in result_name:
if character == '.':
dotless_result_name = dotless_result_name + '_dot_'
else:
dotless_result_name = dotless_result_name + character
empty_function = self.newEmptyFunction("%s_setup" %dotless_result_name)
return_dict = self.newReturnDict(final_module_names) return_dict = self.newReturnDict(final_module_names)
log(return_dict)
empty_function.body = [node, return_dict] empty_function.body = [node, return_dict]
environment_set = self.newEnvironmentSetCall("%s_setup" %result_name) environment_set = self.newEnvironmentSetCall("%s_setup" %dotless_result_name)
warning = self.newImportWarningCall(root_module_name, result_name) self.newImportWarningCall(root_module_name, dotless_result_name)
return [empty_function, environment_set, warning] return [empty_function, environment_set]
else: else:
return node return node
...@@ -872,7 +933,11 @@ class ImportFixer(ast.NodeTransformer): ...@@ -872,7 +933,11 @@ class ImportFixer(ast.NodeTransformer):
""" """
return_dict = "return {" return_dict = "return {"
for name in module_names: for name in module_names:
return_dict = return_dict + "'%s': %s, " % (name, name) if name.find('.') != -1:
base_name = name[:name.find('.')]
else:
base_name = name
return_dict = return_dict + "'%s': %s, " % (base_name, base_name)
return_dict = return_dict + '}' return_dict = return_dict + '}'
return ast.parse(return_dict).body[0] return ast.parse(return_dict).body[0]
...@@ -887,18 +952,10 @@ class ImportFixer(ast.NodeTransformer): ...@@ -887,18 +952,10 @@ class ImportFixer(ast.NodeTransformer):
def newImportWarningCall(self, module_name, function_name): def newImportWarningCall(self, module_name, function_name):
""" """
Return an AST.Expr representanting a print statement with a warning to an Adds a new module to the warning to the user about the importing of new
user about the import of a module named `module_name` and instructs him modules.
on how to fix it.
""" """
warning = ("print '" self.warning_module_names.append(module_name)
"WARNING: Your imported from the module %s without "
"using the environment object, which is not recomended. "
"Your import was automatically converted to use such method."
"The setup function was named as: %s_setup.\\n"
"'") % (module_name, function_name)
tree = ast.parse(warning)
return tree.body[0]
def renderAsHtml(self, renderable_object): def renderAsHtml(self, renderable_object):
...@@ -947,11 +1004,12 @@ def getErrorMessageForException(self, exception, notebook_context): ...@@ -947,11 +1004,12 @@ def getErrorMessageForException(self, exception, notebook_context):
'traceback': traceback_text 'traceback': traceback_text
} }
def createNotebookContext(self): def createNotebookContext(self, parameter_dict={}):
""" """
Function to create an empty notebook context. Function to create an empty notebook context.
""" """
return {'variables': {}, 'setup': {}} nbc = {'variables': {}, 'setup': {}, 'parameter_dict': parameter_dict}
return nbc
class ObjectProcessor(object): class ObjectProcessor(object):
''' '''
...@@ -1148,3 +1206,12 @@ def erp5PivotTableUI(self, df): ...@@ -1148,3 +1206,12 @@ def erp5PivotTableUI(self, df):
iframe_host = self.REQUEST['HTTP_X_FORWARDED_HOST'].split(',')[0] iframe_host = self.REQUEST['HTTP_X_FORWARDED_HOST'].split(',')[0]
url = "https://%s/erp5/Base_displayPivotTableFrame?key=%s" % (iframe_host, key) url = "https://%s/erp5/Base_displayPivotTableFrame?key=%s" % (iframe_host, key)
return IFrame(src=url, width='100%', height='500') return IFrame(src=url, width='100%', height='500')
def Base_checkExistingReference(self, reference):
existing_notebook = self.portal_catalog.getResultValue(
owner=self.portal_membership.getAuthenticatedMember().getUserName(),
portal_type='Data Notebook',
reference=reference)
if not existing_notebook is None:
return True
return False
\ No newline at end of file
...@@ -46,13 +46,15 @@ ...@@ -46,13 +46,15 @@
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple>
<string>W:312, 10: Use of exec (exec-used)</string> <string>W:432, 10: Use of exec (exec-used)</string>
<string>W:355, 10: Use of exec (exec-used)</string> <string>W:478, 10: Use of exec (exec-used)</string>
<string>W:368, 10: Use of exec (exec-used)</string> <string>W:493, 10: Use of exec (exec-used)</string>
<string>W:453, 6: No exception type(s) specified (bare-except)</string> <string>W:286, 2: Unused variable \'globals_dict\' (unused-variable)</string>
<string>W:706, 6: Use of exec (exec-used)</string> <string>W:583, 4: No exception type(s) specified (bare-except)</string>
<string>W:932, 2: Redefining name \'IFrame\' from outer scope (line 4) (redefined-outer-name)</string> <string>W:591, 6: No exception type(s) specified (bare-except)</string>
<string>W: 18, 0: Unused log imported from Products.ERP5Type.Log (unused-import)</string> <string>W:901, 6: Use of exec (exec-used)</string>
<string>W:1007, 0: Dangerous default value {} as argument (dangerous-default-value)</string>
<string>W:1138, 2: Redefining name \'IFrame\' from outer scope (line 4) (redefined-outer-name)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment