erp5_data_notebook: environment object implementation and refactoring to ERP5 kernel

@Tyagov review, please. I'm creating a test suite now and will post about the test results as soon as they are available. 

- An environment object was implemented to help us deal with the multiprocess
architecture of ERP5 and objects that cannot be easily stored in the ZODB.
It stores definition of functions, classes and variables as string. The implementation
uses a dumb Environment class to allow users to make `define` and `undefine` calls,
which are captured and processed by an AST transformer before code execution.

- Along with the environment object, an automatic "import fixer" was created. It does
not allow users to import modules as they normally would, because this may cause
collateral effects on other users' code. A good example is the plot settings in the
matplotlib module. It will fix normal imports, make them use the environment object
mentione earlier automatically and warn the user about it.

Some bugs were fixed with this new implementation: 

- https://nexedi.erp5.net/bug_module/20160318-7098DD, which reports an inconsistency
on portal catalog queries between Jupyter and Python (Script) objects. Probably an
issue with user context storage in ActiveProcess

- https://nexedi.erp5.net/bug_module/20160330-13AA193, which reports an error related
to acquisition when trying to plot images, which happened in other situations, although
this is not officially reported in Nexedi's ERP5. This probably also was happening because
of old user context storage.


/reviewed-on !131

bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.py

 # -*- coding: utf-8 -*-
 
 from cStringIO import StringIO
-from Products.ERP5Type.Globals import  PersistentMapping
+import cPickle
 from erp5.portal_type import Image
 from types import ModuleType
+from ZODB.serialize import ObjectWriter
+
 
 import sys
 import traceback
 import ast
-import types
-
 import base64
+import json
+
 import transaction
+import Acquisition
+import astor
 
 from matplotlib.figure import Figure
 from IPython.core.display import DisplayObject
 from IPython.lib.display import IFrame
 
 
-def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
-  """
-    Function to execute jupyter code and update the local_varibale dictionary.
-    Code execution depends on 'interactivity', a.k.a , if the ast.node object has
-    ast.Expr instance(valid for expressions) or not.
+def Base_executeJupyter(self, python_expression=None, reference=None, title=None, request_reference=False, **kw):
+  # Check permissions for current user and display message to non-authorized user 
+  if not self.Base_checkPermission('portal_components', 'Manage Portal'):
+    return "You are not authorized to access the script"
+  
+  # Convert the request_reference argument string to their respeced boolean values
+  request_reference = {'True': True, 'False': False}.get(request_reference, False)
+  
+  # Return python dictionary with title and reference of all notebooks
+  # for request_reference=True
+  if request_reference:
+    data_notebook_list = self.portal_catalog(portal_type='Data Notebook')
+    notebook_detail_list = [{'reference': obj.getReference(), 'title': obj.getTitle()} for obj in data_notebook_list]
+    return notebook_detail_list
+  
+  if not reference:
+    message = "Please set or use reference for the notebook you want to use"
+    return message
+  
+  # Take python_expression as '' for empty code from jupyter frontend
+  if not python_expression:
+    python_expression = ''
+  
+  # Get Data Notebook with the specific reference
+  data_notebook = self.portal_catalog.getResultValue(portal_type='Data Notebook',
+                        reference=reference)
+  
+  # Create new Data Notebook if reference doesn't match with any from existing ones
+  if not data_notebook:
+    notebook_module = self.getDefaultModule(portal_type='Data Notebook')
+    data_notebook = notebook_module.DataNotebookModule_addDataNotebook(
+      title=title,
+      reference=reference,
+      batch_mode=True
+    )
+  
+  # Add new Data Notebook Line to the Data Notebook
+  data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
+    notebook_code=python_expression,
+    batch_mode=True
+  )
+  
+  # Gets the context associated to the data notebook being used
+  old_notebook_context = data_notebook.getNotebookContext()
+  if not old_notebook_context:
+    old_notebook_context = self.Base_createNotebookContext()
+  
+  # Pass all to code Base_runJupyter external function which would execute the code
+  # and returns a dict of result
+  final_result = Base_runJupyterCode(self, python_expression, old_notebook_context)
+    
+  new_notebook_context = final_result['notebook_context']
+  
+  result = {
+    u'code_result': final_result['result_string'],
+    u'ename': final_result['ename'],
+    u'evalue': final_result['evalue'],
+    u'traceback': final_result['traceback'],
+    u'status': final_result['status'],
+    u'mime_type': final_result['mime_type'],
+  }
   
-    old_local_variable_dict should contain both variables dict and modules imports.
-    Here, imports dict is key, value pair of modules and their name in sys.path,
-    executed separately everytime before execution of jupyter_code to populate
-    sys modules beforehand.
+  # Updates the context in the notebook with the resulting context of code 
+  # execution.
+  data_notebook.setNotebookContext(new_notebook_context)
   
-    For example :
-    old_local_variable_dict = {
-                                'imports': {'numpy': 'np', 'sys': 'sys'},
-                                'variables': {'np.split': <function split at 0x7f4e6eb48b90>}
+  # We try to commit, but the notebook context property may have variables that
+  # cannot be serialized into the ZODB and couldn't be captured by our code yet.
+  # In this case we abort the transaction and warn the user about it. Unforunately,
+  # the exeception raised when this happens doesn't help to know exactly which
+  # object caused the problem, so we cannot tell the user what to fix.
+  try:
+    transaction.commit()
+  except transaction.interfaces.TransactionError as e:
+    transaction.abort()
+    exception_dict = getErrorMessageForException(self, e, new_notebook_context)
+    result.update(exception_dict)
+    return json.dumps(result)
+  
+  # Catch exception while seriaizing the result to be passed to jupyter frontend
+  # and in case of error put code_result as None and status as 'error' which would
+  # be shown by Jupyter frontend
+  try:
+    serialized_result = json.dumps(result)
+  except UnicodeDecodeError:
+    result = {
+      u'code_result': None,
+      u'ename': u'UnicodeDecodeError',
+      u'evalue': None,
+      u'traceback': None,
+      u'status': u'error',
+      u'mime_type': result['mime_type']
+    }
+    serialized_result = json.dumps(result)
+  
+  data_notebook_line.edit(
+    notebook_code_result=result['code_result'], 
+    mime_type=result['mime_type']
+  )
+  
+  return serialized_result  
+
+
+def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
+  """
+    Function to execute jupyter code and update the context dictionary.
+    Code execution depends on 'interactivity', a.k.a , if the ast.node object has
+    ast.Expr instance (valid for expressions) or not.
+    
+    old_notebook_context should contain both variables dict and setup functions.
+    Here, setup dict is {key: value} pair of setup function names and another dict,
+    which contains the function's alias and code, as string. These functions
+    should be executed before `jupyter_code` to properly create the required
+    environment.
+
+    For example:
+    old_notebook_context =  {
+      'setup': {
+        'numpy setup': {
+          'func_name': 'numpy_setup_function',
+          'code': ...
+        }
+      },
+      'variables': {
+        'my_variable': 1
+      }
     }
 
     The behaviour would be similar to that of jupyter notebook:-
@@ -63,16 +178,8 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
 
   # Saving the initial globals dict so as to compare it after code execution
   globals_dict = globals()
-  user_context['context'] = self
   result_string = ''
-  # Update globals dict and use it while running exec command
-  user_context.update(old_local_variable_dict['variables'])
-
-  # XXX: The focus is on 'ok' status only, we're letting errors to be raised on
-  # erp5 for now, so as not to hinder the transactions while catching them.
-  # TODO: This can be refactored by using client side error handling instead of
-  # catching errors on server/erp5.
-  local_variable_dict = old_local_variable_dict
+  notebook_context = old_notebook_context
 
   # Execute only if jupyter_code is not empty
   if jupyter_code:
@@ -82,22 +189,19 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
     except Exception as e:
       # It's not necessary to abort the current transaction here 'cause the 
       # user's code wasn't executed at all yet.
-      return getErrorMessageForException(self, e, local_variable_dict)
+      return getErrorMessageForException(self, e, notebook_context)
+    
+    # Fixing "normal" imports and detecting environment object usage
+    import_fixer = ImportFixer()
+    environment_collector = EnvironmentParser()
+    ast_node = import_fixer.visit(ast_node)
+    ast_node = environment_collector.visit(ast_node)
     
     # Get the node list from the parsed tree
     nodelist = ast_node.body
 
     # Handle case for empty nodelist(in case of comments as jupyter_code)
     if nodelist:
-      # Import all the modules from local_variable_dict['imports']
-      # While any execution, in locals() dict, a module is saved as:
-      # code : 'from os import path'
-      # {'path': <module 'posixpath'>}
-      # So, here we would try to get the name 'posixpath' and import it as 'path'
-      for k, v in old_local_variable_dict['imports'].iteritems():
-        import_statement_code = 'import %s as %s'%(v, k)
-        exec(import_statement_code, user_context, user_context)
-      
       # If the last node is instance of ast.Expr, set its interactivity as 'last'
       # This would be the case if the last node is expression
       if isinstance(nodelist[-1], ast.Expr):
@@ -112,13 +216,15 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
       elif interactivity == 'last':
         to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
 
+      # TODO: fix this global handling by replacing the print statement with
+      # a custom print function. Tip: create an ast.NodeTransformer, like the
+      # one used to fix imports.
       old_stdout = sys.stdout
       result = StringIO()
       sys.stdout = result
       
       # Variables used at the display hook to get the proper form to display
       # the last returning variable of any code cell.
-      #
       display_data = {'result': '', 'mime_type': None}
       
       # This is where one part of the  display magic happens. We create an 
@@ -128,24 +234,103 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
       #
       # The customized display hook will automatically use the processor
       # of the matching class to decide how the object should be displayed.
-      #        
       processor_list = ProcessorList()
       processor_list.addProcessor(IPythonDisplayObjectProcessor)
       processor_list.addProcessor(MatplotlibFigureProcessor)
       processor_list.addProcessor(ERP5ImageProcessor)
       processor_list.addProcessor(IPythonDisplayObjectProcessor)
       
-      # Putting necessary variables in the `exec` calls context.
-      # 
-      # - result: is required to store the order of manual calls to the rendering
-      #   function;
-      #
-      # - display_data: is required to support mime type changes;
-      #
-      # - processor_list: is required for the proper rendering of the objects
-      #
-      user_context['_display_data'] = display_data
-      user_context['_processor_list'] = processor_list
+      # Putting necessary variables in the `exec` calls context and storing
+      inject_variable_dict = {
+        'context': self,
+        'environment': Environment(),
+        '_display_data': display_data,
+        '_processor_list': processor_list,
+        '_volatile_variable_list': []
+      }
+      user_context.update(inject_variable_dict)
+      user_context.update(notebook_context['variables'])
+      
+      # Getting the environment setup defined in the current code cell
+      current_setup_dict = environment_collector.getEnvironmentSetupDict()
+      current_var_dict = environment_collector.getEnvironmentVarDict()
+
+      # Removing old setup from the setup functions
+      removed_setup_message_list = []
+      for func_alias in environment_collector.getEnvironmentRemoveList():
+        found = False
+        for key, data in notebook_context['setup'].items():
+          if key == func_alias:
+            found = True
+            func_name = data['func_name']
+            del notebook_context['setup'][func_alias]
+            try:
+              del user_context[func_alias]
+            except KeyError:
+              pass
+            removed_setup_message = (
+              "%s (%s) was removed from the setup list. "
+              "Variables it may have added to the context and are not pickleable "
+              "were automatically removed.\n"
+            ) % (func_name, func_alias)
+            removed_setup_message_list.append(removed_setup_message)
+            break
+        if not found:
+          transaction.abort()
+          raise Exception("Trying to remove non existing function/variable from environment: '%s'\nEnvironment: %s" % (func_alias, str(notebook_context['setup'])))
+      
+      # Removing all the setup functions if user call environment.clearAll()
+      if environment_collector.clearAll():
+        keys = notebook_context ['setup'].keys()
+        for key in keys:
+          del notebook_context['setup'][key]
+      
+      # Running all the setup functions that we got
+      for key, value in notebook_context['setup'].iteritems():
+        try:
+          code = compile(value['code'], '<string>', 'exec')
+          exec(code, user_context, user_context)
+        # An error happened, so we show the user the stacktrace along with a
+        # note that the exception happened in a setup function's code.
+        except Exception as e:
+          if value['func_name'] in user_context:
+            del user_context[value['func_name']]
+          error_return_dict =  getErrorMessageForException(self, e, notebook_context)
+          additional_information = "An error happened when trying to run the one of your setup functions:"
+          error_return_dict['traceback'].insert(0, additional_information)
+          # As in any other user's code execution, transaction needs to be 
+          # aborted.
+          transaction.abort()
+          return error_return_dict
+      
+      # Iterating over envinronment.define calls captured by the environment collector
+      # that are functions and saving them as setup functions.
+      for func_name, data in current_setup_dict.iteritems():
+        setup_string = (
+          "%s\n"
+          "_result = %s()\n"
+          "if _result and isinstance(_result, dict):\n"
+          "    globals().update(_result)\n"
+          "_volatile_variable_list += _result.keys()\n"
+          "del %s, _result\n"
+        ) % (data['code'], func_name, func_name)
+        notebook_context['setup'][data['alias']] = {
+          "func_name": func_name,
+          "code": setup_string
+        }
+
+      # Iterating over envinronment.define calls captured by the environment collector
+      # that are simple variables and saving them in the setup.
+      for variable, value, in current_var_dict.iteritems():
+        setup_string = "%s = %s\n" % (variable, repr(value))
+        notebook_context['setup'][variable] = {
+          'func_name': variable,
+          'code': setup_string
+        }
+        user_context['_volatile_variable_list'] += variable
+        
+      if environment_collector.showEnvironmentSetup():
+        result_string += "%s\n" % str(notebook_context['setup'])
 
       # Execute the nodes with 'exec' mode
       for node in to_run_exec:
@@ -158,11 +343,9 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
           # are not added if an exception occurs.
           #
           # TODO: store which notebook line generated which exception.
-          #
           transaction.abort()
           # Clear the portal cache from previous transaction
-          self.getPortalObject().portal_caches.clearAllCache()
-          return getErrorMessageForException(self, e, local_variable_dict)
+          return getErrorMessageForException(self, e, notebook_context)
 
       # Execute the interactive nodes with 'single' mode
       for node in to_run_interactive:
@@ -175,51 +358,315 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
           # are not added if an exception occurs.
           #
           # TODO: store which notebook line generated which exception.
-          #
           transaction.abort()
           # Clear the portal cache from previous transaction
-          self.getPortalObject().portal_caches.clearAllCache()
-          return getErrorMessageForException(self, e, local_variable_dict)
+          return getErrorMessageForException(self, e, notebook_context)
 
       sys.stdout = old_stdout
       mime_type = display_data['mime_type'] or mime_type
-      result_string = result.getvalue() + display_data['result']
+      result_string += "\n".join(removed_setup_message_list) + result.getvalue() + display_data['result']
+    
+    # Saves a list of all the variables we injected into the user context and
+    # shall be deleted before saving the context.
+    volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context['_volatile_variable_list']
+    volatile_variable_list.append('__builtins__')
 
-    # Difference between the globals variable before and after exec/eval so that
-    # we don't have to save unnecessary variables in database which might or might
-    # not be picklabale
     for key, val in user_context.items():
-      if key not in globals_dict.keys():
-        local_variable_dict['variables'][key] = val
-
-    # Differentiate 'module' objects from local_variable_dict and save them as
-    # string in the dict as {'imports': {'numpy': 'np', 'matplotlib': 'mp']}
-    if 'variables' and 'imports' in local_variable_dict:
-      for key, val in local_variable_dict['variables'].items():
-        # Check if the val in the dict is ModuleType and remove it in case it is
-        if isinstance(val, types.ModuleType):
-          # Update local_variable_dict['imports'] dictionary with key, value pairs
-          # with key corresponding to module name as its imported and value as the
-          # module name being stored in sys.path
-          # For example : 'np': <numpy module at ...> -- {'np': numpy}
-          local_variable_dict['imports'][key] = val.__name__
-
-          # XXX: The next line is mutating the dict, beware in case any reference
-          # is made later on to local_variable_dict['variables'] dictionary
-          local_variable_dict['variables'].pop(key)
+      if not key in globals_dict.keys() and not isinstance(val, ModuleType) and not key in volatile_variable_list:
+        if canSerialize(val):
+          notebook_context['variables'][key] = val
+        else:
+          del user_context[key]
+          result_string += (
+            "Cannot serialize the variable named %s whose value is %s, "
+            "thus it will not be stored in the context. "
+            "You should move it's definition to a function and " 
+            "use the environment object to load it.\n"
+          ) % (key, val)
+    
+    # Deleting from the variable storage the keys that are not in the user 
+    # context anymore (i.e., variables that are deleted by the user).
+    for key in notebook_context['variables'].keys():
+      if not key in user_context:
+        del notebook_context['variables'][key]
   
   result = {
     'result_string': result_string,
-    'local_variable_dict': local_variable_dict,
+    'notebook_context': notebook_context,
     'status': status,
     'mime_type': mime_type,
     'evalue': evalue,
     'ename': ename,
     'traceback': tb_list,
   }
-
   return result
 
+
+def canSerialize(obj):
+  result = False
+        
+  container_type_tuple = (list, tuple, dict, set, frozenset)
+  
+  # if object is a container, we need to check its elements for presence of
+  # objects that cannot be put inside the zodb
+  if isinstance(obj, container_type_tuple):
+    if isinstance(obj, dict):
+      result_list = []
+      for key, value in obj.iteritems():
+        result_list.append(canSerialize(key))
+        result_list.append(canSerialize(value))
+    else:
+      result_list = [canSerialize(element) for element in obj]
+    return all(result_list)
+  # if obj is an object and implements __getstate__, ZODB.serialize can check
+  # if we can store it
+  elif isinstance(obj, object) and hasattr(obj, '__getstate__'):
+    # Need to unwrap the variable, otherwise we get a TypeError, because
+    # objects cannot be pickled while inside an acquisition wrapper.
+    unwrapped_obj = Acquisition.aq_base(obj)
+    writer = ObjectWriter(unwrapped_obj)
+    for obj in writer:
+      try:
+        writer.serialize(obj)
+      # Because writer.serialize(obj) relies on the implementation of __getstate__
+      # of obj, all errors can happen, so the "except all" is necessary here. 
+      except:
+        return False
+    return True
+  else:
+    # If cannot serialize object with ZODB.serialize, try with cPickle
+    # Only a dump of the object is not enough. Dumping and trying to
+    # load it will properly raise errors in all possible situations, 
+    # for example: if the user defines a dict with an object of a class 
+    # that he created the dump will stil work, but the load will fail. 
+    try:
+      cPickle.loads(cPickle.dumps(obj))
+    # By unknowing reasons, trying to catch cPickle.PicklingError in the "normal"
+    # way isn't working. This issue might be related to some weirdness in 
+    # pickle/cPickle that is reported in this issue: http://bugs.python.org/issue1457119.
+    #
+    # So, as a temporary fix, we're investigating the exception's class name as
+    # string to be able to identify them.
+    # 
+    # Even though the issue seems complicated, this quickfix should be 
+    # properly rewritten in a better way as soon as possible.
+    except Exception as e:
+      if type(e).__name__ in ('PicklingError', 'TypeError', 'NameError', 'AttributeError'):
+        return False
+      else:
+        raise e
+    else:
+      return True
+  
+
+class EnvironmentParser(ast.NodeTransformer):
+  """
+    EnvironmentParser class is an AST transformer that walks in the abstract
+    code syntax tree to find calls to `define` and `undefine`  on a variable
+    named `environment`.
+    
+    The `define` call should receive a function, which will have it's code
+    stored as string in `self.environment_setup_dict`. If only kw args are 
+    provided, the variables definition will be stored in self.environment_var_dict.
+    
+    The `undefine` call will removed keys in self.environment_setup_dict.
+  """
+
+  def __init__(self):
+    self.environment_setup_dict = {}
+    self.environment_var_dict = {}
+    self.environment_remove_list = []
+    self.function_dict = {}
+    self.environment_clear_all = False
+    self.show_environment_setup = False
+
+  def visit_FunctionDef(self, node):
+    """
+      Stores all the function nodes in a dictionary to be accesed later when
+      we detect they are used as parameters for an `environment.define` call.
+    """
+    self.function_dict[node.name] = node
+    return node
+
+  def visit_Expr(self, node):
+    """
+      Visits expressions and check if they are in the form of either 
+      `environment.define` or `environment.undefine` properly stores the 
+      arguments definition as string.
+    """
+    value = node.value
+    if isinstance(value, ast.Call):
+      function = value.func
+      if isinstance(function, ast.Attribute):
+        attribute = function.value
+        if isinstance(attribute, ast.Name):
+          name = attribute.id
+          if name == 'environment' and function.attr == 'define' and not value.keywords:
+            if not len(value.args) == 2:
+              message = (
+                'Not enough arguments for environment definition. Function '
+                'name and alias are required.'
+              )
+              raise Exception(message)
+            func_name = value.args[0].id
+            func_alias = value.args[1].s
+            function_node = self.function_dict[func_name]
+            function_string = astor.to_source(function_node)
+            self.environment_setup_dict[func_name] = {
+              "code": function_string,
+              "alias": func_alias
+            }
+          elif name == 'environment' and function.attr == 'define' and value.keywords:
+            for keyword in value.keywords:
+              arg_name = keyword.arg
+              arg_value_node = keyword.value
+              
+              # The value can be a number, string or name. We need to handle 
+              # them separatedly. This dict trick was used to avoid the very
+              # ugly if.
+              node_value_dict = {
+                ast.Num: lambda node: str(node.n),
+                ast.Str: lambda node: node.s,
+                ast.Name: lambda node: node.id
+              }
+              arg_value = node_value_dict[type(arg_value_node)](arg_value_node)
+              self.environment_var_dict[arg_name] = arg_value
+          elif name == 'environment' and function.attr == 'undefine':
+            func_alias = value.args[0].s
+            self.environment_remove_list.append(func_alias)
+          elif name == 'environment' and function.attr == 'clearAll':
+            self.environment_clear_all = True
+          elif name == 'environment'and function.attr == 'showSetup':
+            self.show_environment_setup = True
+    return node
+    
+  def clearAll(self):
+    return self.environment_clear_all
+    
+  def showEnvironmentSetup(self):
+    return self.show_environment_setup
+
+  def getEnvironmentSetupDict(self):
+    return self.environment_setup_dict
+    
+  def getEnvironmentVarDict(self):
+    return self.environment_var_dict
+    
+  def getEnvironmentRemoveList(self):
+    return self.environment_remove_list
+
+
+class Environment(object):
+  """
+   Dumb object used to receive call on an object named `environment` inside
+   user context. These calls will be tracked by the EnvironmentParser calls.
+  """
+  
+  def define(self, *args, **kwargs):
+    pass
+  
+  def undefine(self, name):
+    pass
+        
+  def clearAll(self):
+    pass
+    
+  def showSetup(self):
+    pass
+  
+
+class ImportFixer(ast.NodeTransformer):
+  """
+   The ImportFixer class is responsivle for fixing "normal" imports that users
+   might try to execute.
+   
+   It will automatically replace them with the proper usage of the environment
+   object using AST manipulation.
+  """
+  
+  def __init__(self):
+    self.import_func_dict = {}
+  
+  def visit_FunctionDef(self, node):
+    """
+      Processes funcion definition nodes. We want to store a list of all the 
+      import that are inside functions, because they do not affect the outter
+      user context, thus do not imply in any un-pickleable variable being added
+      there.
+    """
+    for child in node.body:
+      if isinstance(child, ast.Import):
+        for alias in child.names:
+          self.import_func_dict[alias.name] = node.name  
+    return self.generic_visit(node)
+    
+  def visit_ImportFrom(self, node):
+    """
+     Fixes `import x from y` statements in the same way `import y` is fixed.
+    """
+    return self.visit_Import(node)
+
+  def visit_Import(self, node):
+    """
+    This function replaces `normal` imports by creating AST nodes to define
+    and environment function which setups the module and return it to be merged
+    with the user context.
+    """
+    module_name = node.names[0].name
+    if getattr(node.names[0], 'asname'):
+      module_name = node.names[0].asname
+    if not self.import_func_dict.get(module_name):
+      empty_function = self.newEmptyFunction("%s_setup" % module_name)
+      return_dict = self.newReturnDict(module_name)
+      empty_function.body = [node, return_dict]
+      environment_set = self.newEnvironmentSetCall("%s_setup" % module_name)
+      warning = self.newImportWarningCall(module_name)
+      return [empty_function, environment_set, warning]
+    else:
+      return node
+
+  def newEmptyFunction(self, func_name):
+    """
+      Return a AST.Function object representing a function with name `func_name`
+      and an empty body.
+    """
+    func_body = "def %s(): pass" % func_name
+    return ast.parse(func_body).body[0]
+
+  def newReturnDict(self, module_name):
+    """
+      Return an AST.Expr representing a returned dict with one single key named
+      `'module_name'` (as string) which returns the variable `module_name` (as 
+      exoression).
+    """
+    return_dict = "return {'%s': %s}" % (module_name, module_name)
+    return ast.parse(return_dict).body[0]
+
+  def newEnvironmentSetCall(self, func_name):
+    """
+      Return an AST.Expr representaion an `environment.define` call receiving
+      `func_name` (as an expression) and `'func_name'` (as string).
+    """
+    code_string = "environment.define(%s, '%s')" % (func_name, func_name)
+    tree = ast.parse(code_string)
+    return tree.body[0]
+
+  def newImportWarningCall(self, module_name):
+    """
+      Return an AST.Expr representanting a print statement with a warning to an
+      user about the import of a module named `module_name` and instructs him
+      on how to fix it.
+    """
+    warning = ("print '"
+               "WARNING: Your imported the module %s without using "
+               "the environment object, which is not recomended. "
+               "Your import was automatically converted to use such method."
+               "The setup function registered was named %s_setup.\\n" 
+               "'") % (module_name, module_name)
+    tree = ast.parse(warning)
+    return tree.body[0]
+
+  
 def renderAsHtml(self, renderable_object):
   '''
     renderAsHtml will render its parameter as HTML by using the matching 
@@ -232,7 +679,7 @@ def renderAsHtml(self, renderable_object):
   # At this point the stack should be, from top to the bottom:
   #
   #   5. ExternalMethod Patch call
-  #   4. Base_compileJupyterCode frame (where we want to change variable)
+  #   4. Base_runJupyterCode frame (where we want to change variable)
   #   3. exec call to run the user's code
   #   2. ExternalMethod Patch call through `context.Base_renderAsHtml` in the notebook
   #   1. renderAsHtml frame (where the function is)
@@ -246,10 +693,10 @@ def renderAsHtml(self, renderable_object):
   compile_jupyter_locals['result'].write(result)
   compile_jupyter_locals['display_data']['mime_type'] = 'text/html'
 
-def getErrorMessageForException(self, exception, local_variable_dict):
+def getErrorMessageForException(self, exception, notebook_context):
   '''
     getErrorMessageForException receives an Expcetion object and a context for
-    code execution (local_variable_dict) and will return a dict as Jupyter
+    code execution (notebook_context) and will return a dict as Jupyter
     requires for error rendering.
   '''
   etype, value, tb = sys.exc_info()
@@ -257,34 +704,18 @@ def getErrorMessageForException(self, exception, local_variable_dict):
   return {
     'status': 'error',
     'result_string': None,
-    'local_variable_dict': local_variable_dict,
+    'notebook_context': notebook_context,
     'mime_type': 'text/plain',
     'evalue': str(value),
     'ename': exception.__class__.__name__,
     'traceback': traceback_text
   }
 
-def AddNewLocalVariableDict(self):
+def createNotebookContext(self):
   """
-  Function to add a new Local Variable for a Data Notebook
+  Function to create an empty notebook context.
   """
-  new_dict = PersistentMapping()
-  variable_dict = PersistentMapping()
-  module_dict = PersistentMapping()
-  new_dict['variables'] = variable_dict
-  new_dict['imports'] = module_dict
-  return new_dict
-
-def UpdateLocalVariableDict(self, existing_dict):
-  """
-  Function to update local_varibale_dict for a Data Notebook
-  """
-  new_dict = self.Base_addLocalVariableDict()
-  for key, val in existing_dict['variables'].iteritems():
-    new_dict['variables'][key] = val
-  for key, val in existing_dict['imports'].iteritems():
-    new_dict['imports'][key] = val
-  return new_dict
+  return {'variables': {}, 'setup': {}}
 
 class ObjectProcessor(object):
   '''
@@ -403,6 +834,7 @@ def storeIFrame(self, html, key):
   self.portal_caches.erp5_pivottable_frame_cache.set(key, html)
   return True
 
+
 # WARNING! 
 # 
 # This is a highly experimental PivotTableJs integration which does not follow
@@ -480,3 +912,4 @@ def erp5PivotTableUI(self, df):
   iframe_host = self.REQUEST['HTTP_X_FORWARDED_HOST'].split(',')[0]
   url = "https://%s/erp5/Base_displayPivotTableFrame?key=%s" % (iframe_host, key)
   return IFrame(src=url, width='100%', height='500')
+

bt5/erp5_data_notebook/PropertySheetTemplateItem/portal_property_sheets/JupyterNotebook/local_varibale_property.xml → bt5/erp5_data_notebook/PropertySheetTemplateItem/portal_property_sheets/JupyterNotebook/notebook_context_property.xml

@@ -10,7 +10,7 @@
             <key> <string>categories</string> </key>
             <value>
               <tuple>
-                <string>elementary_type/string</string>
+                <string>elementary_type/object</string>
               </tuple>
             </value>
         </item>
@@ -22,7 +22,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>local_varibale_property</string> </value>
+            <value> <string>notebook_context_property</string> </value>
         </item>
         <item>
             <key> <string>portal_type</string> </key>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_addLocalVariableDict.xml → bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_createNotebookContext.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>_function</string> </key>
-            <value> <string>AddNewLocalVariableDict</string> </value>
+            <value> <string>createNotebookContext</string> </value>
         </item>
         <item>
             <key> <string>_module</string> </key>
@@ -16,7 +16,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>Base_addLocalVariableDict</string> </value>
+            <value> <string>Base_createNotebookContext</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_executeJupyter.py

-"""
-Python script to create Data Notebook or update existing Data Notebooks
-identifying notebook by reference from user.
-
-Expected behaviour from this script:-
-1. Return unauthorized message for non-developer user.
-2. Create new 'Data Notebook' for new reference.
-3. Add new 'Data Notebook Line'to the existing Data Notebook on basis of reference.
-4. Return python dictionary containing list of all notebooks for 'request_reference=True'
-"""
-
-portal = context.getPortalObject()
-# Check permissions for current user and display message to non-authorized user 
-if not portal.Base_checkPermission('portal_components', 'Manage Portal'):
-  return "You are not authorized to access the script"
-
-import json
-
-# Convert the request_reference argument string to their respeced boolean values
-request_reference = {'True': True, 'False': False}.get(request_reference, False)
-
-# Return python dictionary with title and reference of all notebooks
-# for request_reference=True
-if request_reference:
-  data_notebook_list = portal.portal_catalog(portal_type='Data Notebook')
-  notebook_detail_list = [{'reference': obj.getReference(), 'title': obj.getTitle()} for obj in data_notebook_list]
-  return notebook_detail_list
-
-if not reference:
-  message = "Please set or use reference for the notebook you want to use"
-  return message
-
-# Take python_expression as '' for empty code from jupyter frontend
-if not python_expression:
-  python_expression = ''
-
-# Get Data Notebook with the specific reference
-data_notebook = portal.portal_catalog.getResultValue(portal_type='Data Notebook',
-                      reference=reference)
-
-# Create new Data Notebook if reference doesn't match with any from existing ones
-if not data_notebook:
-  notebook_module = portal.getDefaultModule(portal_type='Data Notebook')
-  data_notebook = notebook_module.DataNotebookModule_addDataNotebook(
-    title=title,
-    reference=reference,
-    batch_mode=True
-  )
-
-# Add new Data Notebook Line to the Data Notebook
-data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
-  notebook_code=python_expression,
-  batch_mode=True
-)
-
-# Get active_process associated with data_notebook object
-process_id = data_notebook.getProcess()
-active_process = portal.portal_activities[process_id]
-# Add a result object to Active Process object
-result_list = active_process.getResultList()
-
-# Get local variables saves in Active Result, local varibales are saved as
-# persistent mapping object
-old_local_variable_dict = result_list[0].summary
-if not old_local_variable_dict:
-  old_local_variable_dict = context.Base_addLocalVariableDict()
-
-# Pass all to code Base_runJupyter external function which would execute the code
-# and returns a dict of result
-final_result = context.Base_runJupyter(python_expression, old_local_variable_dict)
-code_result = final_result['result_string']
-new_local_variable_dict = final_result['local_variable_dict']
-ename = final_result['ename']
-evalue = final_result['evalue']
-traceback = final_result['traceback']
-status = final_result['status']
-mime_type = final_result['mime_type']
-
-# Call to function to update persistent mapping object with new local variables
-# and save the variables in the Active Result pertaining to the current Data Notebook
-new_dict = context.Base_updateLocalVariableDict(new_local_variable_dict)
-result_list[0].edit(summary=new_dict)
-
-result = {
-  u'code_result': code_result,
-  u'ename': ename,
-  u'evalue': evalue,
-  u'traceback': traceback,
-  u'status': status,
-  u'mime_type': mime_type
-}
-
-# Catch exception while seriaizing the result to be passed to jupyter frontend
-# and in case of error put code_result as None and status as 'error' which would
-# be shown by Jupyter frontend
-try:
-  serialized_result = json.dumps(result)
-except UnicodeDecodeError:
-  result = {
-    u'code_result': None,
-    u'ename': u'UnicodeDecodeError',
-    u'evalue': None,
-    u'traceback': None,
-    u'status': u'error',
-    u'mime_type': mime_type
-  }
-  serialized_result = json.dumps(result)
-
-data_notebook_line.edit(notebook_code_result=code_result, mime_type=mime_type)
-
-return serialized_result

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_executeJupyter.xml

@@ -2,71 +2,26 @@
 <ZopeData>
   <record id="1" aka="AAAAAAAAAAE=">
     <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
     </pickle>
     <pickle>
       <dictionary>
         <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_Access_contents_information_Permission</string> </key>
-            <value>
-              <tuple>
-                <string>Authenticated</string>
-                <string>Author</string>
-                <string>Manager</string>
-                <string>Owner</string>
-              </tuple>
-            </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
+            <key> <string>_function</string> </key>
+            <value> <string>Base_executeJupyter</string> </value>
         </item>
         <item>
-            <key> <string>_params</string> </key>
-            <value> <string>python_expression=None, reference=None, title=None, request_reference=False, **kw</string> </value>
+            <key> <string>_module</string> </key>
+            <value> <string>JupyterCompile</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>
             <value> <string>Base_executeJupyter</string> </value>
         </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
       </dictionary>
     </pickle>
   </record>

bt5/erp5_data_notebook/PropertySheetTemplateItem/portal_property_sheets/JupyterNotebook/process_property.xml → bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_getErrorMessageForException.xml

@@ -2,31 +2,25 @@
 <ZopeData>
   <record id="1" aka="AAAAAAAAAAE=">
     <pickle>
-      <global name="Standard Property" module="erp5.portal_type"/>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
     </pickle>
     <pickle>
       <dictionary>
         <item>
-            <key> <string>categories</string> </key>
-            <value>
-              <tuple>
-                <string>elementary_type/string</string>
-              </tuple>
-            </value>
+            <key> <string>_function</string> </key>
+            <value> <string>getErrorMessageForException</string> </value>
         </item>
         <item>
-            <key> <string>description</string> </key>
-            <value>
-              <none/>
-            </value>
+            <key> <string>_module</string> </key>
+            <value> <string>JupyterCompile</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>process_property</string> </value>
+            <value> <string>Base_getErrorMessageForException</string> </value>
         </item>
         <item>
-            <key> <string>portal_type</string> </key>
-            <value> <string>Standard Property</string> </value>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_runJupyter.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>_function</string> </key>
-            <value> <string>Base_compileJupyterCode</string> </value>
+            <value> <string>Base_runJupyterCode</string> </value>
         </item>
         <item>
             <key> <string>_module</string> </key>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/DataNotebookModule_addDataNotebook.py

 """
 Python script to add a new notebook to Data Notebook module.
-This script also concerns for assigning an Active Process for each data notebook
-created.
+This script also concerns for assigning an empty notebook context for each data
+notebook created.
 """
-from Products.CMFActivity.ActiveResult import ActiveResult
-
-# Comment out person in case addition of person required to Data Notebook object
-#person = context.ERP5Site_getAuthenticatedMemberPersonValue()
-
-# Create new ActiveProcess object and getting its id
-active_process = context.portal_activities.newActiveProcess()
-active_process_id = active_process.getId()
-
-# Creating new dictionary via external method to save results in ZODB
-new_dict = context.Base_addLocalVariableDict()
-# Add new ActiveResult object and add it to the activeprocess concerned with ...
-# Data Notebook in concern
-result = ActiveResult(summary=new_dict)
-active_process.activateResult(result)
+# Creating new context via external method to save results in ZODB
+notebook_context = context.Base_createNotebookContext()
 
 # Create new notebook
 notebook = context.newContent(
     title=title,
     reference=reference,
-    process=active_process_id,
+    notebook_context=notebook_context,
     portal_type='Data Notebook'
   )
 

bt5/erp5_data_notebook/TestTemplateItem/portal_components/test.erp5.testExecuteJupyter.py

@@ -110,7 +110,7 @@ portal.%s()
     # 
     result = portal.Base_runJupyter(
       jupyter_code=jupyter_code, 
-      old_local_variable_dict=portal.Base_addLocalVariableDict()
+      old_notebook_context=portal.Base_createNotebookContext()
     )
     
     self.assertEquals(result['ename'], 'NameError')
@@ -126,6 +126,23 @@ portal.%s()
     # Test that calling Base_runJupyter shouldn't change the context Title
     self.assertNotEqual(portal.getTitle(), new_test_title)
     
+  def testJupyterCompileInvalidPythonSyntax(self):
+    """
+    Test how the JupyterCompile extension behaves when it receives Python
+    code to be executed that has invalid syntax. 
+    """
+    self.login('dev_user')
+    jupyter_code = "a = 1\na++"
+    
+    reference = 'Test.Notebook.ErrorHandling.SyntaxError'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    result_json = json.loads(result)
+    
+    self.assertEquals(result_json['ename'], 'SyntaxError')
+
   def testUserCannotAccessBaseExecuteJupyter(self):
     """
     Test if non developer user can't access Base_executeJupyter
@@ -133,9 +150,9 @@ portal.%s()
     portal = self.portal
 
     self.login('member_user')
-    result = portal.Base_executeJupyter.Base_checkPermission('portal_components', 'Manage Portal')
+    result = portal.Base_executeJupyter(title='Any title', reference='Any reference')
 
-    self.assertFalse(result)
+    self.assertEquals(result, 'You are not authorized to access the script')
 
   def testUserCanCreateNotebookWithoutCode(self):
     """
@@ -263,10 +280,10 @@ portal.%s()
     self.assertEquals(result['ename'], 'NameError')
     self.assertEquals(result['code_result'], None)
 
-  def testBaseExecuteJupyterSaveActiveResult(self):
+  def testBaseExecuteJupyterSaveNotebookContext(self):
     """
-    Test if the result is being saved inside active_process and the user can
-    access the loacl variable and execute python expression on them
+    Test if user context is being saved in the notebook_context property and the 
+    user can access access and execute python code on it.
     """
     portal = self.portal
     self.login('dev_user')
@@ -286,12 +303,9 @@ portal.%s()
                                           reference=reference
                                           )
     notebook = notebook_list[0]
-    process_id = notebook.getProcess()
-    active_process = portal.portal_activities[process_id]
-    result_list = active_process.getResultList()
-    local_variable_dict = result_list[0].summary['variables']
+    notebook_context = notebook.getNotebookContext()['variables']
     result = {'a':2, 'b':3}
-    self.assertDictContainsSubset(result, local_variable_dict)
+    self.assertDictContainsSubset(result, notebook_context)
 
   def testBaseExecuteJupyterRerunWithPreviousLocalVariables(self):
     """
@@ -321,29 +335,9 @@ portal.%s()
     expected_result = '11'
     self.assertEquals(json.loads(result)['code_result'].rstrip(), expected_result)
 
-  def testBaseExecuteJupyterWithContextObjectsAsLocalVariables(self):
-    """
-    Test Base_executeJupyter with context objects as local variables
-    """
-    portal = self.portal
-    self.login('dev_user')
-    python_expression = 'a=context.getPortalObject(); print a.getTitle()'
-    reference = 'Test.Notebook.ExecutePythonExpressionWithVariables %s' % time.time()
-    title = 'Test NB Title %s' % time.time()
-
-    result = portal.Base_executeJupyter(
-                                        title=title,
-                                        reference=reference,
-                                        python_expression=python_expression
-                                        )
-    self.tic()
-
-    expected_result = portal.getTitle()
-    self.assertEquals(json.loads(result)['code_result'].rstrip(), expected_result)
-
   def testSavingModuleObjectLocalVariables(self):
     """
-    Test to check the saving of module objects in local_variable_dict
+    Test to check the saving of module objects in notebook_context
     and if they work as expected.
     """
     portal = self.portal
@@ -393,13 +387,13 @@ image = context.portal_catalog.getResultValue(portal_type='Image',reference='%s'
 context.Base_renderAsHtml(image)
 """%reference
 
-    local_variable_dict = {'imports' : {}, 'variables' : {}}
+    notebook_context = {'setup' : {}, 'variables' : {}}
     result = self.portal.Base_runJupyter(
       jupyter_code=jupyter_code,
-      old_local_variable_dict=local_variable_dict
+      old_notebook_context=notebook_context
       )
 
-    self.assertEquals(result['result_string'].rstrip(), data_template % base64.b64encode(data))
+    self.assertTrue((data_template % base64.b64encode(data)) in result['result_string'])
     # Mime_type shouldn't be  image/png just because of filename, instead it is
     # dependent on file and file data
     self.assertNotEqual(result['mime_type'], 'image/png')
@@ -439,6 +433,187 @@ context.Base_renderAsHtml(image)
       )
     self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys')
     
+  def testEnvironmentObjectWithFunctionAndClass(self):
+    self.login('dev_user')
+    environment_define_code = '''
+def create_sum_machines():
+  def sum_function(x, y):
+    return x + y
+    
+  class Calculator(object):
+  
+    def sum(self, x, y):
+      return x + y
+    
+  return {'sum_function': sum_function, 'Calculator': Calculator}
+
+environment.clearAll()
+environment.define(create_sum_machines, 'creates sum function and class')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Function'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = '''
+print sum_function(1, 1)
+print Calculator().sum(2, 2)
+'''
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    output = result['code_result']
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(output.strip(), '2\n4')
+    
+  def testEnvironmentObjectSimpleVariable(self):
+    self.login('dev_user')
+    environment_define_code = '''
+environment.clearAll()
+environment.define(x='couscous')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Variable'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = 'print x'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(result['code_result'].strip(), 'couscous')
+    
+  def testEnvironmentUndefineFunctionClass(self):
+    self.login('dev_user')
+    environment_define_code = '''
+def create_sum_machines():
+  def sum_function(x, y):
+    return x + y
+    
+  class Calculator(object):
+  
+    def sum(self, x, y):
+      return x + y
+    
+  return {'sum_function': sum_function, 'Calculator': Calculator}
+
+environment.clearAll()
+environment.define(create_sum_machines, 'creates sum function and class')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Function.Undefine'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    undefine_code = '''
+environment.undefine('creates sum function and class')
+'''
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=undefine_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = '''
+print 'sum_function' in locals()
+print 'Calculator' in locals()
+'''
+
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    result = json.loads(result)
+    output = result['code_result']
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(output.strip(), 'False\nFalse')
+    
+  def testEnvironmentUndefineVariable(self):
+    self.login('dev_user')
+    environment_define_code = '''
+environment.clearAll()
+environment.define(x='couscous')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Variable.Undefine'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    undefine_code = 'environment.undefine("x")'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=undefine_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = "'x' in locals()"
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(result['code_result'].strip(), 'False')
+    
+  def testImportFixer(self):
+    self.login('dev_user')
+    import_code = '''
+import random
+'''
+
+    reference = 'Test.Notebook.EnvironmentObject.ImportFixer'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=import_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = '''
+print random.randint(1,1)
+'''
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(result['code_result'].strip(), '1')
+    
   def testPivotTableJsIntegration(self):
     '''
       This test ensures the PivotTableJs user interface is correctly integrated