erp5_data_notebook: environment object implementation and refactoring to ERP5 kernel

- An environment object was implemented to help us deal with the multiprocess
architecture of ERP5 and objects that cannot be easily stored in the ZODB.
It stores definition of functions, classes and variables as string. The implementation
uses a dumb Environment class to allow users to make `define` and `undefine` calls,
which are captured and processed by an AST transformer before code execution.

- Along with the environment object, an automatic "import fixer" was created. It does
not allow users to import modules as they normally would, because this may cause
collateral effects on other users' code. A good example is the plot settings in the
matplotlib module. It will fix normal imports, make them use the environment object
mentione earlier automatically and warn the user about it.

A few bugs were fixed with this implementation, for example:

- https://nexedi.erp5.net/bug_module/20160318-7098DD, which reports an inconsistency
on portal catalog queries between Jupyter and Python (Script) objects. Probably an
issue with user context storage in ActiveProcess

- https://nexedi.erp5.net/bug_module/20160330-13AA193, which reports an error related
to acquisition when trying to plot images, which happened in other situations, although
this is not officially reported in Nexedi's ERP5. This probably also was happening because
of old user context storage.

bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.py

@@ -8,10 +8,11 @@ from types import ModuleType
 import sys
 import traceback
 import ast
-import types
-
 import base64
+import cPickle
+
 import transaction
+import astor
 
 from matplotlib.figure import Figure
 from IPython.core.display import DisplayObject
@@ -63,43 +64,44 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
 
   # Saving the initial globals dict so as to compare it after code execution
   globals_dict = globals()
-  user_context['context'] = self
   result_string = ''
-  # Update globals dict and use it while running exec command
-  user_context.update(old_local_variable_dict['variables'])
 
   # XXX: The focus is on 'ok' status only, we're letting errors to be raised on
   # erp5 for now, so as not to hinder the transactions while catching them.
   # TODO: This can be refactored by using client side error handling instead of
   # catching errors on server/erp5.
-  local_variable_dict = old_local_variable_dict
+  #
+  local_variable_dict = copy.deepcopy(old_local_variable_dict)
 
   # Execute only if jupyter_code is not empty
+  #
   if jupyter_code:
     # Create ast parse tree
+    #
     try:
       ast_node = ast.parse(jupyter_code)
     except Exception as e:
       # It's not necessary to abort the current transaction here 'cause the 
       # user's code wasn't executed at all yet.
+      #
       return getErrorMessageForException(self, e, local_variable_dict)
     
+    # Fixing "normal" imports and detecting environment object usage
+    import_fixer = ImportFixer()
+    environment_collector = EnvironmentParser()
+    ast_node = import_fixer.visit(ast_node)
+    ast_node = environment_collector.visit(ast_node)
+    
     # Get the node list from the parsed tree
+    #
     nodelist = ast_node.body
 
     # Handle case for empty nodelist(in case of comments as jupyter_code)
+    #
     if nodelist:
-      # Import all the modules from local_variable_dict['imports']
-      # While any execution, in locals() dict, a module is saved as:
-      # code : 'from os import path'
-      # {'path': <module 'posixpath'>}
-      # So, here we would try to get the name 'posixpath' and import it as 'path'
-      for k, v in old_local_variable_dict['imports'].iteritems():
-        import_statement_code = 'import %s as %s'%(v, k)
-        exec(import_statement_code, user_context, user_context)
-      
       # If the last node is instance of ast.Expr, set its interactivity as 'last'
       # This would be the case if the last node is expression
+      #
       if isinstance(nodelist[-1], ast.Expr):
         interactivity = "last"
       else:
@@ -112,13 +114,15 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
       elif interactivity == 'last':
         to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
 
+      # TODO: fix this global handling by replacing the print statement with
+      # a custom print function. Tip: create an ast.NodeTransformer, like the
+      # one used to fix imports.
       old_stdout = sys.stdout
       result = StringIO()
       sys.stdout = result
       
       # Variables used at the display hook to get the proper form to display
       # the last returning variable of any code cell.
-      #
       display_data = {'result': '', 'mime_type': None}
       
       # This is where one part of the  display magic happens. We create an 
@@ -128,26 +132,113 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
       #
       # The customized display hook will automatically use the processor
       # of the matching class to decide how the object should be displayed.
-      #        
       processor_list = ProcessorList()
       processor_list.addProcessor(IPythonDisplayObjectProcessor)
       processor_list.addProcessor(MatplotlibFigureProcessor)
       processor_list.addProcessor(ERP5ImageProcessor)
       processor_list.addProcessor(IPythonDisplayObjectProcessor)
       
-      # Putting necessary variables in the `exec` calls context.
+      # Putting necessary variables in the `exec` calls context and storing
+      inject_variable_dict = {
+        'context': self,
+        'environment': Environment(),
+        '_display_data': display_data,
+        '_processor_list': processor_list,
+        '_volatile_variable_list': []
+      }
+      user_context.update(inject_variable_dict)
+      user_context.update(local_variable_dict['variables'])
+      
+      # Getting the environment setup defined in the current code cell
       #
-      # - result: is required to store the order of manual calls to the rendering
-      #   function;
+      current_setup_dict = environment_collector.getEnvironmentSetupDict()
+      current_var_dict = environment_collector.getEnvironmentVarDict()
+
+      # Removing old setup from the setup functions
       #
-      # - display_data: is required to support mime type changes;
+      removed_setup_message_list = []
+      for func_alias in environment_collector.getEnvironmentRemoveList():
+        found = False
+        for key, data in local_variable_dict['setup'].items():
+          if key == func_alias:
+            found = True
+            func_name = data['func_name']
+            del local_variable_dict['setup'][func_alias]
+            try:
+              del user_context[func_alias]
+            except KeyError:
+              pass
+            removed_setup_message = (
+              "%s (%s) was removed from the setup list. "
+              "Variables it may have added to the context and are not pickleable "
+              "were automatically removed.\n"
+            ) % (func_name, func_alias)
+            removed_setup_message_list.append(removed_setup_message)
+            break
+        if not found:
+          raise Exception("Trying to remove non existing function/variable from environment: '%s'\nEnvironment: %s" % (func_alias, str(local_variable_dict['setup'])))
+      
+      # Removing all the setup functions if user call environment.clearAll()
+      if environment_collector.clearAll():
+        keys = local_variable_dict['setup'].keys()
+        for key in keys:
+          del local_variable_dict['setup'][key]
+      
+      # Running all the setup functions that we got
       #
-      # - processor_list: is required for the proper rendering of the objects
+      for key, value in local_variable_dict['setup'].iteritems():
+        try:
+          code = compile(value['code'], '<string>', 'exec')
+          exec(code, user_context, user_context)
+        # An error happened, so we show the user the stacktrace along with a
+        # note that the exception happened in a setup funtion's code.
+        except Exception as e:
+          if value['func_name'] in user_context:
+            del user_context[value['func_name']]
+          error_return_dict =  getErrorMessageForException(self, e, notebook_context)
+          additional_information = "An error happened when trying to run the one of your setup functions:"
+          error_return_dict['traceback'].insert(0, additional_information)
+          # As in any other user's code execution, transaction needs to be 
+          # aborted.
+          transaction.abort()
+          return error_return_dict
+      
+      # Iterating over envinronment.define calls captured by the environment collector
+      # that are functions and saving them as setup functions.
+      for func_name, data in current_setup_dict.iteritems():
+        setup_string = (
+          "%s\n"
+          "_result = %s()\n"
+          "if _result and isinstance(_result, dict):\n"
+          "    globals().update(_result)\n"
+          "_volatile_variable_list += _result.keys()\n"
+          "del %s, _result\n"
+        ) % (data['code'], func_name, func_name)
+        local_variable_dict['setup'][data['alias']] = {
+          "func_name": func_name,
+          "code": setup_string
+        }
+
+      # Iterating over envinronment.define calls captured by the environment collector
+      # that are simple variables and saving them in the setup.
       #
-      user_context['_display_data'] = display_data
-      user_context['_processor_list'] = processor_list
+      for variable, value, in current_var_dict.iteritems():
+        setup_string = "%s = %s\n" % (variable, repr(value))
+        local_variable_dict['setup'][variable] = {
+          'func_name': variable,
+          'code': setup_string
+        }
+        user_context['_volatile_variable_list'] += variable
+        
+      if environment_collector.showEnvironmentSetup():
+        result_string += "%s\n" % str(local_variable_dict['setup'])
+        # environment_list = []
+        # for func_alias, data in local_variable_dict['setup'].iteritems():
+        #   environment_list.append([data['func_name'], func_alias])
+        # result_string += "%s\n" % environment_list
 
       # Execute the nodes with 'exec' mode
+      #
       for node in to_run_exec:
         mod = ast.Module([node])
         code = compile(mod, '<string>', "exec")
@@ -160,11 +251,10 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
           # TODO: store which notebook line generated which exception.
           #
           transaction.abort()
-          # Clear the portal cache from previous transaction
-          self.getPortalObject().portal_caches.clearAllCache()
           return getErrorMessageForException(self, e, local_variable_dict)
 
       # Execute the interactive nodes with 'single' mode
+      #
       for node in to_run_interactive:
         mod = ast.Interactive([node])
         try:
@@ -177,36 +267,37 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
           # TODO: store which notebook line generated which exception.
           #
           transaction.abort()
-          # Clear the portal cache from previous transaction
-          self.getPortalObject().portal_caches.clearAllCache()
           return getErrorMessageForException(self, e, local_variable_dict)
 
       sys.stdout = old_stdout
       mime_type = display_data['mime_type'] or mime_type
-      result_string = result.getvalue() + display_data['result']
+      result_string += "\n".join(removed_setup_message_list) + result.getvalue() + display_data['result']
     
-    # Difference between the globals variable before and after exec/eval so that
-    # we don't have to save unnecessary variables in database which might or might
-    # not be picklabale
+    # Checking in the user context what variables are pickleable and we can store
+    # safely. Everything that is not pickleable shall not be stored and the user
+    # needs to be warned about it.
+    #
+    volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context['_volatile_variable_list']
+    del user_context['_volatile_variable_list']
     for key, val in user_context.items():
-      if key not in globals_dict.keys():
+      if not key in globals_dict.keys() and not isinstance(val, ModuleType) and not key in volatile_variable_list:
+        try:
+          import pickle
+          pickle.dumps(val)
           local_variable_dict['variables'][key] = val
-
-    # Differentiate 'module' objects from local_variable_dict and save them as
-    # string in the dict as {'imports': {'numpy': 'np', 'matplotlib': 'mp']}
-    if 'variables' and 'imports' in local_variable_dict:
-      for key, val in local_variable_dict['variables'].items():
-        # Check if the val in the dict is ModuleType and remove it in case it is
-        if isinstance(val, types.ModuleType):
-          # Update local_variable_dict['imports'] dictionary with key, value pairs
-          # with key corresponding to module name as its imported and value as the
-          # module name being stored in sys.path
-          # For example : 'np': <numpy module at ...> -- {'np': numpy}
-          local_variable_dict['imports'][key] = val.__name__
-
-          # XXX: The next line is mutating the dict, beware in case any reference
-          # is made later on to local_variable_dict['variables'] dictionary
-          local_variable_dict['variables'].pop(key)
+        except:
+          del user_context[key]
+          result_string += ("Cannot pickle the variable named %s whose value is %s, "
+                            "thus it will not be stored in the context. "
+                            "You should move it's definition to a function and " 
+                            "use the environment object to load it.\n") % (key, val)
+    
+    # Deleting from the variable storage the keys that are not in the user 
+    # context anymore (i.e., variables that are deleted by the user)
+    #
+    for key in local_variable_dict['variables'].keys():
+      if not key in user_context:
+        del local_variable_dict['variables'][key]
 
   result = {
     'result_string': result_string,
@@ -217,9 +308,217 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
     'ename': ename,
     'traceback': tb_list,
   }
-
   return result
   
+
+class EnvironmentParser(ast.NodeTransformer):
+  """
+    EnvironmentParser class is an AST transformer that walks in the abstract
+    code syntax tree to find calls to `define` and `undefine`  on a variable
+    named `environment`.
+    
+    The `define` call should receive a function, which will have it's code
+    stored as string in `self.environment_setup_dict`. If only kw args are 
+    provided, the variables definition will be stored in self.environment_var_dict.
+    
+    The `undefine` call will removed keys in self.environment_setup_dict.
+  """
+
+  def __init__(self):
+    self.environment_setup_dict = {}
+    self.environment_var_dict = {}
+    self.environment_remove_list = []
+    self.function_dict = {}
+    self.environment_clear_all = False
+    self.show_environment_setup = False
+
+  def visit_FunctionDef(self, node):
+    """
+      Stores all the function nodes in a dictionary to be accesed later when
+      we detect they are used as parameters for an `environment.define` call.
+    """
+    self.function_dict[node.name] = node
+    return node
+
+  def visit_Expr(self, node):
+    """
+      Visits expressions and check if they are in the form of either 
+      `environment.define` or `environment.undefine` properly stores the 
+      arguments definition as string.
+    """
+    value = node.value
+    if isinstance(value, ast.Call):
+      function = value.func
+      if isinstance(function, ast.Attribute):
+        attribute = function.value
+        if isinstance(attribute, ast.Name):
+          name = attribute.id
+          if name == 'environment' and function.attr == 'define' and not value.keywords:
+            if not len(value.args) == 2:
+              message = (
+                'Not enough arguments for environment definition. Function '
+                'name and alias are required.'
+              )
+              raise Exception(message)
+            func_name = value.args[0].id
+            func_alias = value.args[1].s
+            function_node = self.function_dict[func_name]
+            function_string = astor.to_source(function_node)
+            self.environment_setup_dict[func_name] = {
+              "code": function_string,
+              "alias": func_alias
+            }
+          elif name == 'environment' and function.attr == 'define' and value.keywords:
+            for keyword in value.keywords:
+              arg_name = keyword.arg
+              arg_value_node = keyword.value
+              
+              # The value can be a number, string or name. We need to handle 
+              # them separatedly. This dict trick was used to avoid the very
+              # ugly if.
+              node_value_dict = {
+                ast.Num: lambda node: str(node.n),
+                ast.Str: lambda node: node.s,
+                ast.Name: lambda node: node.id
+              }
+              arg_value = node_value_dict[type(arg_value_node)](arg_value_node)
+              self.environment_var_dict[arg_name] = arg_value
+          elif name == 'environment' and function.attr == 'undefine':
+            func_alias = value.args[0].s
+            self.environment_remove_list.append(func_alias)
+          elif name == 'environment' and function.attr == 'clearAll':
+            self.environment_clear_all = True
+          elif name == 'environment'and function.attr == 'showSetup':
+            self.show_environment_setup = True
+    return node
+    
+  def clearAll(self):
+    return self.environment_clear_all
+    
+  def showEnvironmentSetup(self):
+    return self.show_environment_setup
+
+  def getEnvironmentSetupDict(self):
+    return self.environment_setup_dict
+    
+  def getEnvironmentVarDict(self):
+    return self.environment_var_dict
+    
+  def getEnvironmentRemoveList(self):
+    return self.environment_remove_list
+
+
+class Environment(object):
+  """
+   Dumb object used to receive call on an object named `environment` inside
+   user context. These calls will be tracked by the EnvironmentParser calls.
+  """
+  
+  def define(self, *args, **kwargs):
+    pass
+  
+  def undefine(self, name):
+    pass
+        
+  def clearAll(self):
+    pass
+    
+  def showSetup(self):
+    pass
+  
+
+class ImportFixer(ast.NodeTransformer):
+  """
+   The ImportFixer class is responsivle for fixing "normal" imports that users
+   might try to execute.
+   
+   It will automatically replace them with the proper usage of the environment
+   object using AST manipulation.
+  """
+  
+  def __init__(self):
+    self.import_func_dict = {}
+  
+  def visit_FunctionDef(self, node):
+    """
+      Processes funcion definition nodes. We want to store a list of all the 
+      import that are inside functions, because they do not affect the outter
+      user context, thus do not imply in any un-pickleable variable being added
+      there.
+    """
+    for child in node.body:
+      if isinstance(child, ast.Import):
+        for alias in child.names:
+          self.import_func_dict[alias.name] = node.name  
+    return self.generic_visit(node)
+    
+  def visit_ImportFrom(self, node):
+    """
+     Fixes `import x from y` statements in the same way `import y` is fixed.
+    """
+    return self.visit_Import(node)
+
+  def visit_Import(self, node):
+    """
+    This function replaces `normal` imports by creating AST nodes to define
+    and environment function which setups the module and return it to be merged
+    with the user context.
+    """
+    module_name = node.names[0].name
+    if getattr(node.names[0], 'asname'):
+      module_name = node.names[0].asname
+    if not self.import_func_dict.get(module_name):
+      empty_function = self.newEmptyFunction("%s_setup" % module_name)
+      return_dict = self.newReturnDict(module_name)
+      empty_function.body = [node, return_dict]
+      environment_set = self.newEnvironmentSetCall("%s_setup" % module_name)
+      warning = self.newImportWarningCall(module_name)
+      return [empty_function, environment_set, warning]
+    else:
+      return node
+
+  def newEmptyFunction(self, func_name):
+    """
+      Return a AST.Function object representing a function with name `func_name`
+      and an empty body.
+    """
+    func_body = "def %s(): pass" % func_name
+    return ast.parse(func_body).body[0]
+
+  def newReturnDict(self, module_name):
+    """
+      Return an AST.Expr representing a returned dict with one single key named
+      `'module_name'` (as string) which returns the variable `module_name` (as 
+      exoression).
+    """
+    return_dict = "return {'%s': %s}" % (module_name, module_name)
+    return ast.parse(return_dict).body[0]
+
+  def newEnvironmentSetCall(self, func_name):
+    """
+      Return an AST.Expr representaion an `environment.define` call receiving
+      `func_name` (as an expression) and `'func_name'` (as string).
+    """
+    code_string = "environment.define(%s, '%s')" % (func_name, func_name)
+    tree = ast.parse(code_string)
+    return tree.body[0]
+
+  def newImportWarningCall(self, module_name):
+    """
+      Return an AST.Expr representanting a print statement with a warning to an
+      user about the import of a module named `module_name` and instructs him
+      on how to fix it.
+    """
+    warning = ("print '"
+               "WARNING: Your imported the module %s without using "
+               "the environment object, which is not recomended. "
+               "Your import was automatically converted to use such method."
+               "The setup function registered was named %s_setup.\\n" 
+               "'") % (module_name, module_name)
+    tree = ast.parse(warning)
+    return tree.body[0]
+
+  
 def renderAsHtml(self, renderable_object):
   '''
     renderAsHtml will render its parameter as HTML by using the matching 
@@ -271,8 +570,10 @@ def AddNewLocalVariableDict(self):
   new_dict = PersistentMapping()
   variable_dict = PersistentMapping()
   module_dict = PersistentMapping()
+  setup_dict = PersistentMapping()
   new_dict['variables'] = variable_dict
   new_dict['imports'] = module_dict
+  new_dict['setup'] = setup_dict 
   return new_dict
 
 def UpdateLocalVariableDict(self, existing_dict):
@@ -284,6 +585,8 @@ def UpdateLocalVariableDict(self, existing_dict):
     new_dict['variables'][key] = val
   for key, val in existing_dict['imports'].iteritems():
     new_dict['imports'][key] = val
+  for key, val in existing_dict['setup'].iteritems():
+    new_dict['setup'][key] = val
   return new_dict
 
 class ObjectProcessor(object):
@@ -403,6 +706,7 @@ def storeIFrame(self, html, key):
   self.portal_caches.erp5_pivottable_frame_cache.set(key, html)
   return True
 
+
 # WARNING! 
 # 
 # This is a highly experimental PivotTableJs integration which does not follow
@@ -480,3 +784,4 @@ def erp5PivotTableUI(self, df):
   iframe_host = self.REQUEST['HTTP_X_FORWARDED_HOST'].split(',')[0]
   url = "https://%s/erp5/Base_displayPivotTableFrame?key=%s" % (iframe_host, key)
   return IFrame(src=url, width='100%', height='500')
+

bt5/erp5_data_notebook/TestTemplateItem/portal_components/test.erp5.testExecuteJupyter.py

@@ -126,6 +126,23 @@ portal.%s()
     # Test that calling Base_runJupyter shouldn't change the context Title
     self.assertNotEqual(portal.getTitle(), new_test_title)
     
+  def testJupyterCompileInvalidPythonSyntax(self):
+    """
+    Test how the JupyterCompile extension behaves when it receives Python
+    code to be executed that has invalid syntax. 
+    """
+    self.login('dev_user')
+    jupyter_code = "a = 1\na++"
+    
+    reference = 'Test.Notebook.ErrorHandling.SyntaxError'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    result_json = json.loads(result)
+    
+    self.assertEquals(result_json['ename'], 'SyntaxError')
+
   def testUserCannotAccessBaseExecuteJupyter(self):
     """
     Test if non developer user can't access Base_executeJupyter
@@ -321,26 +338,6 @@ portal.%s()
     expected_result = '11'
     self.assertEquals(json.loads(result)['code_result'].rstrip(), expected_result)
 
-  def testBaseExecuteJupyterWithContextObjectsAsLocalVariables(self):
-    """
-    Test Base_executeJupyter with context objects as local variables
-    """
-    portal = self.portal
-    self.login('dev_user')
-    python_expression = 'a=context.getPortalObject(); print a.getTitle()'
-    reference = 'Test.Notebook.ExecutePythonExpressionWithVariables %s' % time.time()
-    title = 'Test NB Title %s' % time.time()
-
-    result = portal.Base_executeJupyter(
-                                        title=title,
-                                        reference=reference,
-                                        python_expression=python_expression
-                                        )
-    self.tic()
-
-    expected_result = portal.getTitle()
-    self.assertEquals(json.loads(result)['code_result'].rstrip(), expected_result)
-
   def testSavingModuleObjectLocalVariables(self):
     """
     Test to check the saving of module objects in local_variable_dict
@@ -393,7 +390,7 @@ image = context.portal_catalog.getResultValue(portal_type='Image',reference='%s'
 context.Base_renderAsHtml(image)
 """%reference
 
-    local_variable_dict = {'imports' : {}, 'variables' : {}}
+    local_variable_dict = {'setup' : {}, 'variables' : {}}
     result = self.portal.Base_runJupyter(
       jupyter_code=jupyter_code,
       old_local_variable_dict=local_variable_dict
@@ -439,6 +436,187 @@ context.Base_renderAsHtml(image)
       )
     self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys')
     
+  def testEnvironmentObjectWithFunctionAndClass(self):
+    self.login('dev_user')
+    environment_define_code = '''
+def create_sum_machines():
+  def sum_function(x, y):
+    return x + y
+    
+  class Calculator(object):
+  
+    def sum(self, x, y):
+      return x + y
+    
+  return {'sum_function': sum_function, 'Calculator': Calculator}
+
+environment.clearAll()
+environment.define(create_sum_machines, 'creates sum function and class')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Function'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = '''
+print sum_function(1, 1)
+print Calculator().sum(2, 2)
+'''
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    output = result['code_result']
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(output.strip(), '2\n4')
+    
+  def testEnvironmentObjectSimpleVariable(self):
+    self.login('dev_user')
+    environment_define_code = '''
+environment.clearAll()
+environment.define(x='couscous')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Variable'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = 'print x'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(result['code_result'].strip(), 'couscous')
+    
+  def testEnvironmentUndefineFunctionClass(self):
+    self.login('dev_user')
+    environment_define_code = '''
+def create_sum_machines():
+  def sum_function(x, y):
+    return x + y
+    
+  class Calculator(object):
+  
+    def sum(self, x, y):
+      return x + y
+    
+  return {'sum_function': sum_function, 'Calculator': Calculator}
+
+environment.clearAll()
+environment.define(create_sum_machines, 'creates sum function and class')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Function.Undefine'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    undefine_code = '''
+environment.undefine('creates sum function and class')
+'''
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=undefine_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = '''
+print 'sum_function' in locals()
+print 'Calculator' in locals()
+'''
+
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    result = json.loads(result)
+    output = result['code_result']
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(output.strip(), 'False\nFalse')
+    
+  def testEnvironmentUndefineVariable(self):
+    self.login('dev_user')
+    environment_define_code = '''
+environment.clearAll()
+environment.define(x='couscous')
+'''
+    reference = 'Test.Notebook.EnvironmentObject.Variable.Undefine'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=environment_define_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    undefine_code = 'environment.undefine("x")'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=undefine_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = "'x' in locals()"
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(result['code_result'].strip(), 'False')
+    
+  def testImportFixer(self):
+    self.login('dev_user')
+    import_code = '''
+import random
+'''
+
+    reference = 'Test.Notebook.EnvironmentObject.ImportFixer'
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=import_code
+    )
+    
+    self.tic()
+    self.assertEquals(json.loads(result)['status'], 'ok')
+    
+    jupyter_code = '''
+print random.randint(1,1)
+'''
+    result = self.portal.Base_executeJupyter(
+      reference=reference,
+      python_expression=jupyter_code
+    )
+    
+    self.tic()
+    result = json.loads(result)
+    self.assertEquals(result['status'], 'ok')
+    self.assertEquals(result['code_result'].strip(), '1')
+    
   def testPivotTableJsIntegration(self):
     '''
       This test ensures the PivotTableJs user interface is correctly integrated