extension.erp5.JupyterCompile.py 6.36 KB
Newer Older
1 2 3
# -*- coding: utf-8 -*-

from StringIO import StringIO
4
from persistent.list import PersistentList
5
from Products.ERP5Type.Globals import  PersistentMapping
6 7

import sys
8
import ast
9
import types
10 11 12 13

def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
  """
    Function to execute jupyter code and update the local_varibale dictionary.
14 15
    Code execution depends on 'interactivity', a.k.a , if the ast.node object has
    ast.Expr instance(valid for expressions) or not.
16 17 18 19 20 21 22 23 24 25 26
    
    old_local_variable_dict should contain both variables dict and imports list.
    Here, imports list is basically a list of code lines which would be run
    executed separately everytime before execution of jupyter_code to populate
    sys modules beforehand.

    For example :
    old_local_variable_dict = {
                                'imports': ['import numpy as np', 'import sys as sys'],
                                'variables': {'np.split': <function split at 0x7f4e6eb48b90>}
                                }
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43

    The behaviour would be similar to that of jupyter notebook:-
    ( https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py#L2954 )
    Example:

      code1 = '''
      23
      print 23 #Last node not an expression, interactivity = 'last'
      '''
      out1 = '23'

      code2 = '''
      123
      12 #Last node an expression, interactivity = 'none'
      '''
      out2 = '12'

44 45 46 47
  """
  # Other way would be to use all the globals variables instead of just an empty
  # dictionary, but that might hamper the speed of exec or eval.
  # Something like -- g = globals(); g['context'] = self;
48 49 50 51
  g = {}

  # Saving the initial globals dict so as to compare it after code execution
  globals_dict = globals()
52 53 54
  g['context'] = self
  result_string = None
  ename, evalue, tb_list = None, None, None
55
  # Update globals dict and use it while running exec command
56
  g.update(old_local_variable_dict['variables'])
57 58

  # IPython expects 2 status message - 'ok', 'error'
59 60 61 62
  # XXX: The focus is on 'ok' status only, we're letting errors to be raised on
  # erp5 for now, so as not to hinder the transactions while catching them.
  # TODO: This can be refactored by using client side error handling instead of
  # catching errors on server/erp5.
63 64
  status = u'ok'

65 66
  # Execute only if jupyter_code is not empty
  if jupyter_code:
67 68 69 70 71
    # Import all the modules from local_variable_dict['imports']
    import_statement_code = '\n'.join(old_local_variable_dict['imports'])

    exec(import_statement_code, g, g)
  
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
    # Create ast parse tree
    ast_node = ast.parse(jupyter_code)
    # Get the node list from the parsed tree
    nodelist = ast_node.body

    # If the last node is instance of ast.Expr, set its interactivity as 'last'
    # This would be the case if the last node is expression
    if isinstance(nodelist[-1], ast.Expr):
      interactivity = "last"
    else:
      interactivity = "none"

    # Here, we define which nodes to execute with 'single' and which to execute
    # with 'exec' mode.
    if interactivity == 'none':
      to_run_exec, to_run_interactive = nodelist, []
    elif interactivity == 'last':
      to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]

91 92 93 94
    old_stdout = sys.stdout
    result = StringIO()
    sys.stdout = result

95 96 97 98 99 100 101 102 103 104 105 106
    # Execute the nodes with 'exec' mode
    for node in to_run_exec:
      mod = ast.Module([node])
      code = compile(mod, '<string>', "exec")
      exec(code, g, g)

    # Execute the interactive nodes with 'single' mode
    for node in to_run_interactive:
      mod = ast.Interactive([node])
      code = compile(mod, '<string>', "single")
      exec(code, g, g)

107 108 109 110 111 112
    # Letting the code fail in case of error while executing the python script/code
    # XXX: Need to be refactored so to acclimitize transactions failure as well as
    # normal python code failure and show it to user on jupyter frontend.
    # Decided to let this fail silently in backend without letting the frontend
    # user know the error so as to let tranasction or its error be handled by ZODB
    # in uniform way instead of just using half transactions.
113

114 115
    sys.stdout = old_stdout
    result_string = result.getvalue()
116 117
  else:
    result_string = jupyter_code
118

119 120 121 122 123
  # Difference between the globals variable before and after exec/eval so that
  # we don't have to save unnecessary variables in database which might or might
  # not be picklabale
  local_variable_dict = old_local_variable_dict
  local_variable_dict_new = {key: val for key, val in g.items() if key not in globals_dict.keys()}
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
  local_variable_dict['variables'].update(local_variable_dict_new)

  # Differentiate 'module' objects from local_variable_dict and save them as
  # string in the dict as {'imports': ['import numpy as np', 'import matplotlib as mp']}
  if 'variables' in local_variable_dict:
    for key, val in local_variable_dict['variables'].items():
      # Check if the val in the dict is ModuleType and remove it in case it is
      if isinstance(val, types.ModuleType):
        # XXX: The next line is mutating the dict, beware in case any reference
        # is made later on to local_variable_dict['variables'] dictionary
        local_variable_dict['variables'].pop(key)
        # While any execution, in locals() dict, a module is saved as:
        # code : 'from os import path'
        # {'path': <module 'posixpath'>}
        # So, here we would try to get the name 'posixpath' and import it as 'path'
        module_name = val.__name__
        import_statement = 'import %s as %s'%(module_name, key)
        local_variable_dict['imports'].append(import_statement)
142

143 144
  result = {
    'result_string': result_string,
145
    'local_variable_dict': local_variable_dict,
146 147 148
    'status': status,
    'evalue': evalue,
    'ename': ename,
149
    'traceback': tb_list,
150
  }
151

152
  return result
153

154
def AddNewLocalVariableDict(self):
155
  """
156
  Function to add a new Local Variable for a Data Notebook
157 158
  """
  new_dict = PersistentMapping()
159 160 161 162
  variable_dict = PersistentMapping()
  import_list = PersistentList()
  new_dict['variables'] = variable_dict
  new_dict['imports'] = import_list
163 164
  return new_dict

165
def UpdateLocalVariableDict(self, existing_dict):
166
  """
167
  Function to update local_varibale_dict for a Data Notebook
168
  """
169
  new_dict = self.Base_addLocalVariableDict()
170 171 172
  for key, val in existing_dict['variables'].iteritems():
    new_dict['variables'][key] = val
  new_dict['imports'] = PersistentList(existing_dict['imports'])
173
  return new_dict
174