Skip to content

erp5
erp5
Commit a0d21451 authored by Douglas's avatar Douglas Committed by Ivan Tyagov
Browse files
Options

erp5_data_notebook: environment object implementation and refactoring to ERP5 kernel 

@Tyagov review, please. I'm creating a test suite now and will post about the test results as soon as they are available. 

- An environment object was implemented to help us deal with the multiprocess
architecture of ERP5 and objects that cannot be easily stored in the ZODB.
It stores definition of functions, classes and variables as string. The implementation
uses a dumb Environment class to allow users to make `define` and `undefine` calls,
which are captured and processed by an AST transformer before code execution.

- Along with the environment object, an automatic "import fixer" was created. It does
not allow users to import modules as they normally would, because this may cause
collateral effects on other users' code. A good example is the plot settings in the
matplotlib module. It will fix normal imports, make them use the environment object
mentione earlier automatically and warn the user about it.

Some bugs were fixed with this new implementation: 

- https://nexedi.erp5.net/bug_module/20160318-7098DD, which reports an inconsistency
on portal catalog queries between Jupyter and Python (Script) objects. Probably an
issue with user context storage in ActiveProcess

- https://nexedi.erp5.net/bug_module/20160330-13AA193, which reports an error related
to acquisition when trying to plot images, which happened in other situations, although
this is not officially reported in Nexedi's ERP5. This probably also was happening because
of old user context storage.


/reviewed-on !131
parents fb52f4a5 23e06437
Show whitespace changes
Inline Side-by-side
Showing with 792 additions and 325 deletions
bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.py
View file @ a0d21451
# -*- coding: utf-8 -*-
from cStringIO import StringIO
from Products.ERP5Type.Globals import PersistentMapping
import cPickle
from erp5.portal_type import Image
from types import ModuleType
from ZODB.serialize import ObjectWriter
import sys
import traceback
import ast
import types
import base64
import json
import transaction
import Acquisition
import astor
from matplotlib.figure import Figure
from IPython.core.display import DisplayObject
from IPython.lib.display import IFrame
def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
"""
Function to execute jupyter code and update the local_varibale dictionary.
Code execution depends on 'interactivity', a.k.a , if the ast.node object has
ast.Expr instance(valid for expressions) or not.
def Base_executeJupyter(self, python_expression=None, reference=None, title=None, request_reference=False, **kw):
# Check permissions for current user and display message to non-authorized user
if not self.Base_checkPermission('portal_components', 'Manage Portal'):
return "You are not authorized to access the script"
# Convert the request_reference argument string to their respeced boolean values
request_reference = {'True': True, 'False': False}.get(request_reference, False)
# Return python dictionary with title and reference of all notebooks
# for request_reference=True
if request_reference:
data_notebook_list = self.portal_catalog(portal_type='Data Notebook')
notebook_detail_list = [{'reference': obj.getReference(), 'title': obj.getTitle()} for obj in data_notebook_list]
return notebook_detail_list
if not reference:
message = "Please set or use reference for the notebook you want to use"
return message
# Take python_expression as '' for empty code from jupyter frontend
if not python_expression:
python_expression = ''
# Get Data Notebook with the specific reference
data_notebook = self.portal_catalog.getResultValue(portal_type='Data Notebook',
reference=reference)
# Create new Data Notebook if reference doesn't match with any from existing ones
if not data_notebook:
notebook_module = self.getDefaultModule(portal_type='Data Notebook')
data_notebook = notebook_module.DataNotebookModule_addDataNotebook(
title=title,
reference=reference,
batch_mode=True
)
# Add new Data Notebook Line to the Data Notebook
data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
notebook_code=python_expression,
batch_mode=True
)
# Gets the context associated to the data notebook being used
old_notebook_context = data_notebook.getNotebookContext()
if not old_notebook_context:
old_notebook_context = self.Base_createNotebookContext()
# Pass all to code Base_runJupyter external function which would execute the code
# and returns a dict of result
final_result = Base_runJupyterCode(self, python_expression, old_notebook_context)
new_notebook_context = final_result['notebook_context']
result = {
u'code_result': final_result['result_string'],
u'ename': final_result['ename'],
u'evalue': final_result['evalue'],
u'traceback': final_result['traceback'],
u'status': final_result['status'],
u'mime_type': final_result['mime_type'],
}
old_local_variable_dict should contain both variables dict and modules imports.
Here, imports dict is key, value pair of modules and their name in sys.path,
executed separately everytime before execution of jupyter_code to populate
sys modules beforehand.
# Updates the context in the notebook with the resulting context of code
# execution.
data_notebook.setNotebookContext(new_notebook_context)
For example :
old_local_variable_dict = {
'imports': {'numpy': 'np', 'sys': 'sys'},
'variables': {'np.split': <function split at 0x7f4e6eb48b90>}
# We try to commit, but the notebook context property may have variables that
# cannot be serialized into the ZODB and couldn't be captured by our code yet.
# In this case we abort the transaction and warn the user about it. Unforunately,
# the exeception raised when this happens doesn't help to know exactly which
# object caused the problem, so we cannot tell the user what to fix.
try:
transaction.commit()
except transaction.interfaces.TransactionError as e:
transaction.abort()
exception_dict = getErrorMessageForException(self, e, new_notebook_context)
result.update(exception_dict)
return json.dumps(result)
# Catch exception while seriaizing the result to be passed to jupyter frontend
# and in case of error put code_result as None and status as 'error' which would
# be shown by Jupyter frontend
try:
serialized_result = json.dumps(result)
except UnicodeDecodeError:
result = {
u'code_result': None,
u'ename': u'UnicodeDecodeError',
u'evalue': None,
u'traceback': None,
u'status': u'error',
u'mime_type': result['mime_type']
}
serialized_result = json.dumps(result)
data_notebook_line.edit(
notebook_code_result=result['code_result'],
mime_type=result['mime_type']
)
return serialized_result
def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
"""
Function to execute jupyter code and update the context dictionary.
Code execution depends on 'interactivity', a.k.a , if the ast.node object has
ast.Expr instance (valid for expressions) or not.
old_notebook_context should contain both variables dict and setup functions.
Here, setup dict is {key: value} pair of setup function names and another dict,
which contains the function's alias and code, as string. These functions
should be executed before `jupyter_code` to properly create the required
environment.
For example:
old_notebook_context = {
'setup': {
'numpy setup': {
'func_name': 'numpy_setup_function',
'code': ...
}
},
'variables': {
'my_variable': 1
}
}
The behaviour would be similar to that of jupyter notebook:-
......@@ -63,16 +178,8 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# Saving the initial globals dict so as to compare it after code execution
globals_dict = globals()
user_context['context'] = self
result_string = ''
# Update globals dict and use it while running exec command
user_context.update(old_local_variable_dict['variables'])
# XXX: The focus is on 'ok' status only, we're letting errors to be raised on
# erp5 for now, so as not to hinder the transactions while catching them.
# TODO: This can be refactored by using client side error handling instead of
# catching errors on server/erp5.
local_variable_dict = old_local_variable_dict
notebook_context = old_notebook_context
# Execute only if jupyter_code is not empty
if jupyter_code:
......@@ -82,22 +189,19 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
except Exception as e:
# It's not necessary to abort the current transaction here 'cause the
# user's code wasn't executed at all yet.
return getErrorMessageForException(self, e, local_variable_dict)
return getErrorMessageForException(self, e, notebook_context)
# Fixing "normal" imports and detecting environment object usage
import_fixer = ImportFixer()
environment_collector = EnvironmentParser()
ast_node = import_fixer.visit(ast_node)
ast_node = environment_collector.visit(ast_node)
# Get the node list from the parsed tree
nodelist = ast_node.body
# Handle case for empty nodelist(in case of comments as jupyter_code)
if nodelist:
# Import all the modules from local_variable_dict['imports']
# While any execution, in locals() dict, a module is saved as:
# code : 'from os import path'
# {'path': <module 'posixpath'>}
# So, here we would try to get the name 'posixpath' and import it as 'path'
for k, v in old_local_variable_dict['imports'].iteritems():
import_statement_code = 'import %s as %s'%(v, k)
exec(import_statement_code, user_context, user_context)
# If the last node is instance of ast.Expr, set its interactivity as 'last'
# This would be the case if the last node is expression
if isinstance(nodelist[-1], ast.Expr):
......@@ -112,13 +216,15 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
elif interactivity == 'last':
to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
# TODO: fix this global handling by replacing the print statement with
# a custom print function. Tip: create an ast.NodeTransformer, like the
# one used to fix imports.
old_stdout = sys.stdout
result = StringIO()
sys.stdout = result
# Variables used at the display hook to get the proper form to display
# the last returning variable of any code cell.
#
display_data = {'result': '', 'mime_type': None}
# This is where one part of the display magic happens. We create an
......@@ -128,24 +234,103 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
#
# The customized display hook will automatically use the processor
# of the matching class to decide how the object should be displayed.
#
processor_list = ProcessorList()
processor_list.addProcessor(IPythonDisplayObjectProcessor)
processor_list.addProcessor(MatplotlibFigureProcessor)
processor_list.addProcessor(ERP5ImageProcessor)
processor_list.addProcessor(IPythonDisplayObjectProcessor)
# Putting necessary variables in the `exec` calls context.
#
# - result: is required to store the order of manual calls to the rendering
# function;
#
# - display_data: is required to support mime type changes;
#
# - processor_list: is required for the proper rendering of the objects
#
user_context['_display_data'] = display_data
user_context['_processor_list'] = processor_list
# Putting necessary variables in the `exec` calls context and storing
inject_variable_dict = {
'context': self,
'environment': Environment(),
'_display_data': display_data,
'_processor_list': processor_list,
'_volatile_variable_list': []
}
user_context.update(inject_variable_dict)
user_context.update(notebook_context['variables'])
# Getting the environment setup defined in the current code cell
current_setup_dict = environment_collector.getEnvironmentSetupDict()
current_var_dict = environment_collector.getEnvironmentVarDict()
# Removing old setup from the setup functions
removed_setup_message_list = []
for func_alias in environment_collector.getEnvironmentRemoveList():
found = False
for key, data in notebook_context['setup'].items():
if key == func_alias:
found = True
func_name = data['func_name']
del notebook_context['setup'][func_alias]
try:
del user_context[func_alias]
except KeyError:
pass
removed_setup_message = (
"%s (%s) was removed from the setup list. "
"Variables it may have added to the context and are not pickleable "
"were automatically removed.\n"
) % (func_name, func_alias)
removed_setup_message_list.append(removed_setup_message)
break
if not found:
transaction.abort()
raise Exception("Trying to remove non existing function/variable from environment: '%s'\nEnvironment: %s" % (func_alias, str(notebook_context['setup'])))
# Removing all the setup functions if user call environment.clearAll()
if environment_collector.clearAll():
keys = notebook_context ['setup'].keys()
for key in keys:
del notebook_context['setup'][key]
# Running all the setup functions that we got
for key, value in notebook_context['setup'].iteritems():
try:
code = compile(value['code'], '<string>', 'exec')
exec(code, user_context, user_context)
# An error happened, so we show the user the stacktrace along with a
# note that the exception happened in a setup function's code.
except Exception as e:
if value['func_name'] in user_context:
del user_context[value['func_name']]
error_return_dict = getErrorMessageForException(self, e, notebook_context)
additional_information = "An error happened when trying to run the one of your setup functions:"
error_return_dict['traceback'].insert(0, additional_information)
# As in any other user's code execution, transaction needs to be
# aborted.
transaction.abort()
return error_return_dict
# Iterating over envinronment.define calls captured by the environment collector
# that are functions and saving them as setup functions.
for func_name, data in current_setup_dict.iteritems():
setup_string = (
"%s\n"
"_result = %s()\n"
"if _result and isinstance(_result, dict):\n"
" globals().update(_result)\n"
"_volatile_variable_list += _result.keys()\n"
"del %s, _result\n"
) % (data['code'], func_name, func_name)
notebook_context['setup'][data['alias']] = {
"func_name": func_name,
"code": setup_string
}
# Iterating over envinronment.define calls captured by the environment collector
# that are simple variables and saving them in the setup.
for variable, value, in current_var_dict.iteritems():
setup_string = "%s = %s\n" % (variable, repr(value))
notebook_context['setup'][variable] = {
'func_name': variable,
'code': setup_string
}
user_context['_volatile_variable_list'] += variable
if environment_collector.showEnvironmentSetup():
result_string += "%s\n" % str(notebook_context['setup'])
# Execute the nodes with 'exec' mode
for node in to_run_exec:
......@@ -158,11 +343,9 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# are not added if an exception occurs.
#
# TODO: store which notebook line generated which exception.
#
transaction.abort()
# Clear the portal cache from previous transaction
self.getPortalObject().portal_caches.clearAllCache()
return getErrorMessageForException(self, e, local_variable_dict)
return getErrorMessageForException(self, e, notebook_context)
# Execute the interactive nodes with 'single' mode
for node in to_run_interactive:
......@@ -175,51 +358,315 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# are not added if an exception occurs.
#
# TODO: store which notebook line generated which exception.
#
transaction.abort()
# Clear the portal cache from previous transaction
self.getPortalObject().portal_caches.clearAllCache()
return getErrorMessageForException(self, e, local_variable_dict)
return getErrorMessageForException(self, e, notebook_context)
sys.stdout = old_stdout
mime_type = display_data['mime_type'] or mime_type
result_string = result.getvalue() + display_data['result']
result_string += "\n".join(removed_setup_message_list) + result.getvalue() + display_data['result']
# Saves a list of all the variables we injected into the user context and
# shall be deleted before saving the context.
volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context['_volatile_variable_list']
volatile_variable_list.append('__builtins__')
# Difference between the globals variable before and after exec/eval so that
# we don't have to save unnecessary variables in database which might or might
# not be picklabale
for key, val in user_context.items():
if key not in globals_dict.keys():
local_variable_dict['variables'][key] = val
# Differentiate 'module' objects from local_variable_dict and save them as
# string in the dict as {'imports': {'numpy': 'np', 'matplotlib': 'mp']}
if 'variables' and 'imports' in local_variable_dict:
for key, val in local_variable_dict['variables'].items():
# Check if the val in the dict is ModuleType and remove it in case it is
if isinstance(val, types.ModuleType):
# Update local_variable_dict['imports'] dictionary with key, value pairs
# with key corresponding to module name as its imported and value as the
# module name being stored in sys.path
# For example : 'np': <numpy module at ...> -- {'np': numpy}
local_variable_dict['imports'][key] = val.__name__
# XXX: The next line is mutating the dict, beware in case any reference
# is made later on to local_variable_dict['variables'] dictionary
local_variable_dict['variables'].pop(key)
if not key in globals_dict.keys() and not isinstance(val, ModuleType) and not key in volatile_variable_list:
if canSerialize(val):
notebook_context['variables'][key] = val
else:
del user_context[key]
result_string += (
"Cannot serialize the variable named %s whose value is %s, "
"thus it will not be stored in the context. "
"You should move it's definition to a function and "
"use the environment object to load it.\n"
) % (key, val)
# Deleting from the variable storage the keys that are not in the user
# context anymore (i.e., variables that are deleted by the user).
for key in notebook_context['variables'].keys():
if not key in user_context:
del notebook_context['variables'][key]
result = {
'result_string': result_string,
'local_variable_dict': local_variable_dict,
'notebook_context': notebook_context,
'status': status,
'mime_type': mime_type,
'evalue': evalue,
'ename': ename,
'traceback': tb_list,
}
return result
def canSerialize(obj):
result = False
container_type_tuple = (list, tuple, dict, set, frozenset)
# if object is a container, we need to check its elements for presence of
# objects that cannot be put inside the zodb
if isinstance(obj, container_type_tuple):
if isinstance(obj, dict):
result_list = []
for key, value in obj.iteritems():
result_list.append(canSerialize(key))
result_list.append(canSerialize(value))
else:
result_list = [canSerialize(element) for element in obj]
return all(result_list)
# if obj is an object and implements __getstate__, ZODB.serialize can check
# if we can store it
elif isinstance(obj, object) and hasattr(obj, '__getstate__'):
# Need to unwrap the variable, otherwise we get a TypeError, because
# objects cannot be pickled while inside an acquisition wrapper.
unwrapped_obj = Acquisition.aq_base(obj)
writer = ObjectWriter(unwrapped_obj)
for obj in writer:
try:
writer.serialize(obj)
# Because writer.serialize(obj) relies on the implementation of __getstate__
# of obj, all errors can happen, so the "except all" is necessary here.
except:
return False
return True
else:
# If cannot serialize object with ZODB.serialize, try with cPickle
# Only a dump of the object is not enough. Dumping and trying to
# load it will properly raise errors in all possible situations,
# for example: if the user defines a dict with an object of a class
# that he created the dump will stil work, but the load will fail.
try:
cPickle.loads(cPickle.dumps(obj))
# By unknowing reasons, trying to catch cPickle.PicklingError in the "normal"
# way isn't working. This issue might be related to some weirdness in
# pickle/cPickle that is reported in this issue: http://bugs.python.org/issue1457119.
#
# So, as a temporary fix, we're investigating the exception's class name as
# string to be able to identify them.
#
# Even though the issue seems complicated, this quickfix should be
# properly rewritten in a better way as soon as possible.
except Exception as e:
if type(e).__name__ in ('PicklingError', 'TypeError', 'NameError', 'AttributeError'):
return False
else:
raise e
else:
return True
class EnvironmentParser(ast.NodeTransformer):
"""
EnvironmentParser class is an AST transformer that walks in the abstract
code syntax tree to find calls to `define` and `undefine` on a variable
named `environment`.
The `define` call should receive a function, which will have it's code
stored as string in `self.environment_setup_dict`. If only kw args are
provided, the variables definition will be stored in self.environment_var_dict.
The `undefine` call will removed keys in self.environment_setup_dict.
"""
def __init__(self):
self.environment_setup_dict = {}
self.environment_var_dict = {}
self.environment_remove_list = []
self.function_dict = {}
self.environment_clear_all = False
self.show_environment_setup = False
def visit_FunctionDef(self, node):
"""
Stores all the function nodes in a dictionary to be accesed later when
we detect they are used as parameters for an `environment.define` call.
"""
self.function_dict[node.name] = node
return node
def visit_Expr(self, node):
"""
Visits expressions and check if they are in the form of either
`environment.define` or `environment.undefine` properly stores the
arguments definition as string.
"""
value = node.value
if isinstance(value, ast.Call):
function = value.func
if isinstance(function, ast.Attribute):
attribute = function.value
if isinstance(attribute, ast.Name):
name = attribute.id
if name == 'environment' and function.attr == 'define' and not value.keywords:
if not len(value.args) == 2:
message = (
'Not enough arguments for environment definition. Function '
'name and alias are required.'
)
raise Exception(message)
func_name = value.args[0].id
func_alias = value.args[1].s
function_node = self.function_dict[func_name]
function_string = astor.to_source(function_node)
self.environment_setup_dict[func_name] = {
"code": function_string,
"alias": func_alias
}
elif name == 'environment' and function.attr == 'define' and value.keywords:
for keyword in value.keywords:
arg_name = keyword.arg
arg_value_node = keyword.value
# The value can be a number, string or name. We need to handle
# them separatedly. This dict trick was used to avoid the very
# ugly if.
node_value_dict = {
ast.Num: lambda node: str(node.n),
ast.Str: lambda node: node.s,
ast.Name: lambda node: node.id
}
arg_value = node_value_dict[type(arg_value_node)](arg_value_node)
self.environment_var_dict[arg_name] = arg_value
elif name == 'environment' and function.attr == 'undefine':
func_alias = value.args[0].s
self.environment_remove_list.append(func_alias)
elif name == 'environment' and function.attr == 'clearAll':
self.environment_clear_all = True
elif name == 'environment'and function.attr == 'showSetup':
self.show_environment_setup = True
return node
def clearAll(self):
return self.environment_clear_all
def showEnvironmentSetup(self):
return self.show_environment_setup
def getEnvironmentSetupDict(self):
return self.environment_setup_dict
def getEnvironmentVarDict(self):
return self.environment_var_dict
def getEnvironmentRemoveList(self):
return self.environment_remove_list
class Environment(object):
"""
Dumb object used to receive call on an object named `environment` inside
user context. These calls will be tracked by the EnvironmentParser calls.
"""
def define(self, *args, **kwargs):
pass
def undefine(self, name):
pass
def clearAll(self):
pass
def showSetup(self):
pass
class ImportFixer(ast.NodeTransformer):
"""
The ImportFixer class is responsivle for fixing "normal" imports that users
might try to execute.
It will automatically replace them with the proper usage of the environment
object using AST manipulation.
"""
def __init__(self):
self.import_func_dict = {}
def visit_FunctionDef(self, node):
"""
Processes funcion definition nodes. We want to store a list of all the
import that are inside functions, because they do not affect the outter
user context, thus do not imply in any un-pickleable variable being added
there.
"""
for child in node.body:
if isinstance(child, ast.Import):
for alias in child.names:
self.import_func_dict[alias.name] = node.name
return self.generic_visit(node)
def visit_ImportFrom(self, node):
"""
Fixes `import x from y` statements in the same way `import y` is fixed.
"""
return self.visit_Import(node)
def visit_Import(self, node):
"""
This function replaces `normal` imports by creating AST nodes to define
and environment function which setups the module and return it to be merged
with the user context.
"""
module_name = node.names[0].name
if getattr(node.names[0], 'asname'):
module_name = node.names[0].asname
if not self.import_func_dict.get(module_name):
empty_function = self.newEmptyFunction("%s_setup" % module_name)
return_dict = self.newReturnDict(module_name)
empty_function.body = [node, return_dict]
environment_set = self.newEnvironmentSetCall("%s_setup" % module_name)
warning = self.newImportWarningCall(module_name)
return [empty_function, environment_set, warning]
else:
return node
def newEmptyFunction(self, func_name):
"""
Return a AST.Function object representing a function with name `func_name`
and an empty body.
"""
func_body = "def %s(): pass" % func_name
return ast.parse(func_body).body[0]
def newReturnDict(self, module_name):
"""
Return an AST.Expr representing a returned dict with one single key named
`'module_name'` (as string) which returns the variable `module_name` (as
exoression).
"""
return_dict = "return {'%s': %s}" % (module_name, module_name)
return ast.parse(return_dict).body[0]
def newEnvironmentSetCall(self, func_name):
"""
Return an AST.Expr representaion an `environment.define` call receiving
`func_name` (as an expression) and `'func_name'` (as string).
"""
code_string = "environment.define(%s, '%s')" % (func_name, func_name)
tree = ast.parse(code_string)
return tree.body[0]
def newImportWarningCall(self, module_name):
"""
Return an AST.Expr representanting a print statement with a warning to an
user about the import of a module named `module_name` and instructs him
on how to fix it.
"""
warning = ("print '"
"WARNING: Your imported the module %s without using "
"the environment object, which is not recomended. "
"Your import was automatically converted to use such method."
"The setup function registered was named %s_setup.\\n"
"'") % (module_name, module_name)
tree = ast.parse(warning)
return tree.body[0]
def renderAsHtml(self, renderable_object):
'''
renderAsHtml will render its parameter as HTML by using the matching
......@@ -232,7 +679,7 @@ def renderAsHtml(self, renderable_object):
# At this point the stack should be, from top to the bottom:
#
# 5. ExternalMethod Patch call
# 4. Base_compileJupyterCode frame (where we want to change variable)
# 4. Base_runJupyterCode frame (where we want to change variable)
# 3. exec call to run the user's code
# 2. ExternalMethod Patch call through `context.Base_renderAsHtml` in the notebook
# 1. renderAsHtml frame (where the function is)
......@@ -246,10 +693,10 @@ def renderAsHtml(self, renderable_object):
compile_jupyter_locals['result'].write(result)
compile_jupyter_locals['display_data']['mime_type'] = 'text/html'
def getErrorMessageForException(self, exception, local_variable_dict):
def getErrorMessageForException(self, exception, notebook_context):
'''
getErrorMessageForException receives an Expcetion object and a context for
code execution (local_variable_dict) and will return a dict as Jupyter
code execution (notebook_context) and will return a dict as Jupyter
requires for error rendering.
'''
etype, value, tb = sys.exc_info()
......@@ -257,34 +704,18 @@ def getErrorMessageForException(self, exception, local_variable_dict):
return {
'status': 'error',
'result_string': None,
'local_variable_dict': local_variable_dict,
'notebook_context': notebook_context,
'mime_type': 'text/plain',
'evalue': str(value),
'ename': exception.__class__.__name__,
'traceback': traceback_text
}
def AddNewLocalVariableDict(self):
def createNotebookContext(self):
"""
Function to add a new Local Variable for a Data Notebook
Function to create an empty notebook context.
"""
new_dict = PersistentMapping()
variable_dict = PersistentMapping()
module_dict = PersistentMapping()
new_dict['variables'] = variable_dict
new_dict['imports'] = module_dict
return new_dict
def UpdateLocalVariableDict(self, existing_dict):
"""
Function to update local_varibale_dict for a Data Notebook
"""
new_dict = self.Base_addLocalVariableDict()
for key, val in existing_dict['variables'].iteritems():
new_dict['variables'][key] = val
for key, val in existing_dict['imports'].iteritems():
new_dict['imports'][key] = val
return new_dict
return {'variables': {}, 'setup': {}}
class ObjectProcessor(object):
'''
......@@ -403,6 +834,7 @@ def storeIFrame(self, html, key):
self.portal_caches.erp5_pivottable_frame_cache.set(key, html)
return True
# WARNING!
#
# This is a highly experimental PivotTableJs integration which does not follow
......@@ -480,3 +912,4 @@ def erp5PivotTableUI(self, df):
iframe_host = self.REQUEST['HTTP_X_FORWARDED_HOST'].split(',')[0]
url = "https://%s/erp5/Base_displayPivotTableFrame?key=%s" % (iframe_host, key)
return IFrame(src=url, width='100%', height='500')
bt5/erp5_data_notebook/PropertySheetTemplateItem/portal_property_sheets/JupyterNotebook/local_varibale_property.xml → bt5/erp5_data_notebook/PropertySheetTemplateItem/portal_property_sheets/JupyterNotebook/notebook_context_property.xml
View file @ a0d21451
......@@ -10,7 +10,7 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/string</string>
<string>elementary_type/object</string>
</tuple>
</value>
</item>
......@@ -22,7 +22,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>local_varibale_property</string> </value>
<value> <string>notebook_context_property</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
......
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_addLocalVariableDict.xml → bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_createNotebookContext.xml
View file @ a0d21451
......@@ -8,7 +8,7 @@
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>AddNewLocalVariableDict</string> </value>
<value> <string>createNotebookContext</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
......@@ -16,7 +16,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_addLocalVariableDict</string> </value>
<value> <string>Base_createNotebookContext</string> </value>
</item>
<item>
<key> <string>title</string> </key>
......
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_executeJupyter.py deleted 100644 → 0
View file @ fb52f4a5
"""
Python script to create Data Notebook or update existing Data Notebooks
identifying notebook by reference from user.
Expected behaviour from this script:-
1. Return unauthorized message for non-developer user.
2. Create new 'Data Notebook' for new reference.
3. Add new 'Data Notebook Line'to the existing Data Notebook on basis of reference.
4. Return python dictionary containing list of all notebooks for 'request_reference=True'
"""
portal = context.getPortalObject()
# Check permissions for current user and display message to non-authorized user
if not portal.Base_checkPermission('portal_components', 'Manage Portal'):
return "You are not authorized to access the script"
import json
# Convert the request_reference argument string to their respeced boolean values
request_reference = {'True': True, 'False': False}.get(request_reference, False)
# Return python dictionary with title and reference of all notebooks
# for request_reference=True
if request_reference:
data_notebook_list = portal.portal_catalog(portal_type='Data Notebook')
notebook_detail_list = [{'reference': obj.getReference(), 'title': obj.getTitle()} for obj in data_notebook_list]
return notebook_detail_list
if not reference:
message = "Please set or use reference for the notebook you want to use"
return message
# Take python_expression as '' for empty code from jupyter frontend
if not python_expression:
python_expression = ''
# Get Data Notebook with the specific reference
data_notebook = portal.portal_catalog.getResultValue(portal_type='Data Notebook',
reference=reference)
# Create new Data Notebook if reference doesn't match with any from existing ones
if not data_notebook:
notebook_module = portal.getDefaultModule(portal_type='Data Notebook')
data_notebook = notebook_module.DataNotebookModule_addDataNotebook(
title=title,
reference=reference,
batch_mode=True
)
# Add new Data Notebook Line to the Data Notebook
data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
notebook_code=python_expression,
batch_mode=True
)
# Get active_process associated with data_notebook object
process_id = data_notebook.getProcess()
active_process = portal.portal_activities[process_id]
# Add a result object to Active Process object
result_list = active_process.getResultList()
# Get local variables saves in Active Result, local varibales are saved as
# persistent mapping object
old_local_variable_dict = result_list[0].summary
if not old_local_variable_dict:
old_local_variable_dict = context.Base_addLocalVariableDict()
# Pass all to code Base_runJupyter external function which would execute the code
# and returns a dict of result
final_result = context.Base_runJupyter(python_expression, old_local_variable_dict)
code_result = final_result['result_string']
new_local_variable_dict = final_result['local_variable_dict']
ename = final_result['ename']
evalue = final_result['evalue']
traceback = final_result['traceback']
status = final_result['status']
mime_type = final_result['mime_type']
# Call to function to update persistent mapping object with new local variables
# and save the variables in the Active Result pertaining to the current Data Notebook
new_dict = context.Base_updateLocalVariableDict(new_local_variable_dict)
result_list[0].edit(summary=new_dict)
result = {
u'code_result': code_result,
u'ename': ename,
u'evalue': evalue,
u'traceback': traceback,
u'status': status,
u'mime_type': mime_type
}
# Catch exception while seriaizing the result to be passed to jupyter frontend
# and in case of error put code_result as None and status as 'error' which would
# be shown by Jupyter frontend
try:
serialized_result = json.dumps(result)
except UnicodeDecodeError:
result = {
u'code_result': None,
u'ename': u'UnicodeDecodeError',
u'evalue': None,
u'traceback': None,
u'status': u'error',
u'mime_type': mime_type
}
serialized_result = json.dumps(result)
data_notebook_line.edit(notebook_code_result=code_result, mime_type=mime_type)
return serialized_result
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_executeJupyter.xml
View file @ a0d21451
......@@ -2,71 +2,26 @@
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Authenticated</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
<key> <string>_function</string> </key>
<value> <string>Base_executeJupyter</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>python_expression=None, reference=None, title=None, request_reference=False, **kw</string> </value>
<key> <string>_module</string> </key>
<value> <string>JupyterCompile</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_executeJupyter</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
......
bt5/erp5_data_notebook/PropertySheetTemplateItem/portal_property_sheets/JupyterNotebook/process_property.xml → bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_getErrorMessageForException.xml
View file @ a0d21451
......@@ -2,31 +2,25 @@
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/string</string>
</tuple>
</value>
<key> <string>_function</string> </key>
<value> <string>getErrorMessageForException</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
<key> <string>_module</string> </key>
<value> <string>JupyterCompile</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>process_property</string> </value>
<value> <string>Base_getErrorMessageForException</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
......
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_runJupyter.xml
View file @ a0d21451
......@@ -8,7 +8,7 @@
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>Base_compileJupyterCode</string> </value>
<value> <string>Base_runJupyterCode</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
......
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/DataNotebookModule_addDataNotebook.py
View file @ a0d21451
"""
Python script to add a new notebook to Data Notebook module.
This script also concerns for assigning an Active Process for each data notebook
created.
This script also concerns for assigning an empty notebook context for each data
notebook created.
"""
from Products.CMFActivity.ActiveResult import ActiveResult
# Comment out person in case addition of person required to Data Notebook object
#person = context.ERP5Site_getAuthenticatedMemberPersonValue()
# Create new ActiveProcess object and getting its id
active_process = context.portal_activities.newActiveProcess()
active_process_id = active_process.getId()
# Creating new dictionary via external method to save results in ZODB
new_dict = context.Base_addLocalVariableDict()
# Add new ActiveResult object and add it to the activeprocess concerned with ...
# Data Notebook in concern
result = ActiveResult(summary=new_dict)
active_process.activateResult(result)
# Creating new context via external method to save results in ZODB
notebook_context = context.Base_createNotebookContext()
# Create new notebook
notebook = context.newContent(
title=title,
reference=reference,
process=active_process_id,
notebook_context=notebook_context,
portal_type='Data Notebook'
)
......
bt5/erp5_data_notebook/TestTemplateItem/portal_components/test.erp5.testExecuteJupyter.py
View file @ a0d21451
......@@ -110,7 +110,7 @@ portal.%s()
#
result = portal.Base_runJupyter(
jupyter_code=jupyter_code,
old_local_variable_dict=portal.Base_addLocalVariableDict()
old_notebook_context=portal.Base_createNotebookContext()
)
self.assertEquals(result['ename'], 'NameError')
......@@ -126,6 +126,23 @@ portal.%s()
# Test that calling Base_runJupyter shouldn't change the context Title
self.assertNotEqual(portal.getTitle(), new_test_title)
def testJupyterCompileInvalidPythonSyntax(self):
"""
Test how the JupyterCompile extension behaves when it receives Python
code to be executed that has invalid syntax.
"""
self.login('dev_user')
jupyter_code = "a = 1\na++"
reference = 'Test.Notebook.ErrorHandling.SyntaxError'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
result_json = json.loads(result)
self.assertEquals(result_json['ename'], 'SyntaxError')
def testUserCannotAccessBaseExecuteJupyter(self):
"""
Test if non developer user can't access Base_executeJupyter
......@@ -133,9 +150,9 @@ portal.%s()
portal = self.portal
self.login('member_user')
result = portal.Base_executeJupyter.Base_checkPermission('portal_components', 'Manage Portal')
result = portal.Base_executeJupyter(title='Any title', reference='Any reference')
self.assertFalse(result)
self.assertEquals(result, 'You are not authorized to access the script')
def testUserCanCreateNotebookWithoutCode(self):
"""
......@@ -263,10 +280,10 @@ portal.%s()
self.assertEquals(result['ename'], 'NameError')
self.assertEquals(result['code_result'], None)
def testBaseExecuteJupyterSaveActiveResult(self):
def testBaseExecuteJupyterSaveNotebookContext(self):
"""
Test if the result is being saved inside active_process and the user can
access the loacl variable and execute python expression on them
Test if user context is being saved in the notebook_context property and the
user can access access and execute python code on it.
"""
portal = self.portal
self.login('dev_user')
......@@ -286,12 +303,9 @@ portal.%s()
reference=reference
)
notebook = notebook_list[0]
process_id = notebook.getProcess()
active_process = portal.portal_activities[process_id]
result_list = active_process.getResultList()
local_variable_dict = result_list[0].summary['variables']
notebook_context = notebook.getNotebookContext()['variables']
result = {'a':2, 'b':3}
self.assertDictContainsSubset(result, local_variable_dict)
self.assertDictContainsSubset(result, notebook_context)
def testBaseExecuteJupyterRerunWithPreviousLocalVariables(self):
"""
......@@ -321,29 +335,9 @@ portal.%s()
expected_result = '11'
self.assertEquals(json.loads(result)['code_result'].rstrip(), expected_result)
def testBaseExecuteJupyterWithContextObjectsAsLocalVariables(self):
"""
Test Base_executeJupyter with context objects as local variables
"""
portal = self.portal
self.login('dev_user')
python_expression = 'a=context.getPortalObject(); print a.getTitle()'
reference = 'Test.Notebook.ExecutePythonExpressionWithVariables %s' % time.time()
title = 'Test NB Title %s' % time.time()
result = portal.Base_executeJupyter(
title=title,
reference=reference,
python_expression=python_expression
)
self.tic()
expected_result = portal.getTitle()
self.assertEquals(json.loads(result)['code_result'].rstrip(), expected_result)
def testSavingModuleObjectLocalVariables(self):
"""
Test to check the saving of module objects in local_variable_dict
Test to check the saving of module objects in notebook_context
and if they work as expected.
"""
portal = self.portal
......@@ -393,13 +387,13 @@ image = context.portal_catalog.getResultValue(portal_type='Image',reference='%s'
context.Base_renderAsHtml(image)
"""%reference
local_variable_dict = {'imports' : {}, 'variables' : {}}
notebook_context = {'setup' : {}, 'variables' : {}}
result = self.portal.Base_runJupyter(
jupyter_code=jupyter_code,
old_local_variable_dict=local_variable_dict
old_notebook_context=notebook_context
)
self.assertEquals(result['result_string'].rstrip(), data_template % base64.b64encode(data))
self.assertTrue((data_template % base64.b64encode(data)) in result['result_string'])
# Mime_type shouldn't be image/png just because of filename, instead it is
# dependent on file and file data
self.assertNotEqual(result['mime_type'], 'image/png')
......@@ -439,6 +433,187 @@ context.Base_renderAsHtml(image)
)
self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys')
def testEnvironmentObjectWithFunctionAndClass(self):
self.login('dev_user')
environment_define_code = '''
def create_sum_machines():
def sum_function(x, y):
return x + y
class Calculator(object):
def sum(self, x, y):
return x + y
return {'sum_function': sum_function, 'Calculator': Calculator}
environment.clearAll()
environment.define(create_sum_machines, 'creates sum function and class')
'''
reference = 'Test.Notebook.EnvironmentObject.Function'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=environment_define_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
jupyter_code = '''
print sum_function(1, 1)
print Calculator().sum(2, 2)
'''
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
self.tic()
result = json.loads(result)
output = result['code_result']
self.assertEquals(result['status'], 'ok')
self.assertEquals(output.strip(), '2\n4')
def testEnvironmentObjectSimpleVariable(self):
self.login('dev_user')
environment_define_code = '''
environment.clearAll()
environment.define(x='couscous')
'''
reference = 'Test.Notebook.EnvironmentObject.Variable'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=environment_define_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
jupyter_code = 'print x'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
self.tic()
result = json.loads(result)
self.assertEquals(result['status'], 'ok')
self.assertEquals(result['code_result'].strip(), 'couscous')
def testEnvironmentUndefineFunctionClass(self):
self.login('dev_user')
environment_define_code = '''
def create_sum_machines():
def sum_function(x, y):
return x + y
class Calculator(object):
def sum(self, x, y):
return x + y
return {'sum_function': sum_function, 'Calculator': Calculator}
environment.clearAll()
environment.define(create_sum_machines, 'creates sum function and class')
'''
reference = 'Test.Notebook.EnvironmentObject.Function.Undefine'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=environment_define_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
undefine_code = '''
environment.undefine('creates sum function and class')
'''
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=undefine_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
jupyter_code = '''
print 'sum_function' in locals()
print 'Calculator' in locals()
'''
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
result = json.loads(result)
output = result['code_result']
self.assertEquals(result['status'], 'ok')
self.assertEquals(output.strip(), 'False\nFalse')
def testEnvironmentUndefineVariable(self):
self.login('dev_user')
environment_define_code = '''
environment.clearAll()
environment.define(x='couscous')
'''
reference = 'Test.Notebook.EnvironmentObject.Variable.Undefine'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=environment_define_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
undefine_code = 'environment.undefine("x")'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=undefine_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
jupyter_code = "'x' in locals()"
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
self.tic()
result = json.loads(result)
self.assertEquals(result['status'], 'ok')
self.assertEquals(result['code_result'].strip(), 'False')
def testImportFixer(self):
self.login('dev_user')
import_code = '''
import random
'''
reference = 'Test.Notebook.EnvironmentObject.ImportFixer'
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=import_code
)
self.tic()
self.assertEquals(json.loads(result)['status'], 'ok')
jupyter_code = '''
print random.randint(1,1)
'''
result = self.portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
self.tic()
result = json.loads(result)
self.assertEquals(result['status'], 'ok')
self.assertEquals(result['code_result'].strip(), '1')
def testPivotTableJsIntegration(self):
'''
This test ensures the PivotTableJs user interface is correctly integrated
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7