extension.erp5.JupyterCompile.py 34 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3
from cStringIO import StringIO
4
import cPickle
5 6
from erp5.portal_type import Image
from types import ModuleType
7
from ZODB.serialize import ObjectWriter
8

9

10
import sys
11
import traceback
12
import ast
13
import base64
14
import json
15

16
import transaction
17
import Acquisition
18
import astor
19

20 21 22 23
from matplotlib.figure import Figure
from IPython.core.display import DisplayObject
from IPython.lib.display import IFrame

24

25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
def Base_executeJupyter(self, python_expression=None, reference=None, title=None, request_reference=False, **kw):
  # Check permissions for current user and display message to non-authorized user 
  if not self.Base_checkPermission('portal_components', 'Manage Portal'):
    return "You are not authorized to access the script"
  
  # Convert the request_reference argument string to their respeced boolean values
  request_reference = {'True': True, 'False': False}.get(request_reference, False)
  
  # Return python dictionary with title and reference of all notebooks
  # for request_reference=True
  if request_reference:
    data_notebook_list = self.portal_catalog(portal_type='Data Notebook')
    notebook_detail_list = [{'reference': obj.getReference(), 'title': obj.getTitle()} for obj in data_notebook_list]
    return notebook_detail_list
  
  if not reference:
    message = "Please set or use reference for the notebook you want to use"
    return message
  
  # Take python_expression as '' for empty code from jupyter frontend
  if not python_expression:
    python_expression = ''
  
  # Get Data Notebook with the specific reference
  data_notebook = self.portal_catalog.getResultValue(portal_type='Data Notebook',
                        reference=reference)
  
  # Create new Data Notebook if reference doesn't match with any from existing ones
  if not data_notebook:
    notebook_module = self.getDefaultModule(portal_type='Data Notebook')
    data_notebook = notebook_module.DataNotebookModule_addDataNotebook(
      title=title,
      reference=reference,
      batch_mode=True
    )
  
  # Add new Data Notebook Line to the Data Notebook
  data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
    notebook_code=python_expression,
    batch_mode=True
  )
  
  # Gets the context associated to the data notebook being used
  old_notebook_context = data_notebook.getNotebookContext()
  if not old_notebook_context:
    old_notebook_context = self.Base_createNotebookContext()
  
  # Pass all to code Base_runJupyter external function which would execute the code
  # and returns a dict of result
  final_result = Base_runJupyterCode(self, python_expression, old_notebook_context)
    
  new_notebook_context = final_result['notebook_context']
  
  result = {
    u'code_result': final_result['result_string'],
    u'ename': final_result['ename'],
    u'evalue': final_result['evalue'],
    u'traceback': final_result['traceback'],
    u'status': final_result['status'],
    u'mime_type': final_result['mime_type'],
  }
  
  # Updates the context in the notebook with the resulting context of code 
  # execution.
  data_notebook.setNotebookContext(new_notebook_context)
  
  # We try to commit, but the notebook context property may have variables that
  # cannot be serialized into the ZODB and couldn't be captured by our code yet.
  # In this case we abort the transaction and warn the user about it. Unforunately,
  # the exeception raised when this happens doesn't help to know exactly which
  # object caused the problem, so we cannot tell the user what to fix.
  try:
    transaction.commit()
  except transaction.interfaces.TransactionError as e:
    transaction.abort()
    exception_dict = getErrorMessageForException(self, e, new_notebook_context)
    result.update(exception_dict)
    return json.dumps(result)
  
  # Catch exception while seriaizing the result to be passed to jupyter frontend
  # and in case of error put code_result as None and status as 'error' which would
  # be shown by Jupyter frontend
  try:
    serialized_result = json.dumps(result)
  except UnicodeDecodeError:
    result = {
      u'code_result': None,
      u'ename': u'UnicodeDecodeError',
      u'evalue': None,
      u'traceback': None,
      u'status': u'error',
      u'mime_type': result['mime_type']
    }
    serialized_result = json.dumps(result)
  
  data_notebook_line.edit(
    notebook_code_result=result['code_result'], 
    mime_type=result['mime_type']
  )
  
  return serialized_result  


128
def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
129
  """
130
    Function to execute jupyter code and update the context dictionary.
131
    Code execution depends on 'interactivity', a.k.a , if the ast.node object has
132
    ast.Expr instance (valid for expressions) or not.
133
    
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
    old_notebook_context should contain both variables dict and setup functions.
    Here, setup dict is {key: value} pair of setup function names and another dict,
    which contains the function's alias and code, as string. These functions
    should be executed before `jupyter_code` to properly create the required
    environment.

    For example:
    old_notebook_context =  {
      'setup': {
        'numpy setup': {
          'func_name': 'numpy_setup_function',
          'code': ...
        }
      },
      'variables': {
        'my_variable': 1
      }
    }
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168

    The behaviour would be similar to that of jupyter notebook:-
    ( https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py#L2954 )
    Example:

      code1 = '''
      23
      print 23 #Last node not an expression, interactivity = 'last'
      '''
      out1 = '23'

      code2 = '''
      123
      12 #Last node an expression, interactivity = 'none'
      '''
      out2 = '12'

169
  """
170
  mime_type = 'text/plain'
171 172
  status = u'ok'
  ename, evalue, tb_list = None, None, None
173
  
174 175
  # Other way would be to use all the globals variables instead of just an empty
  # dictionary, but that might hamper the speed of exec or eval.
176 177
  # Something like -- user_context = globals(); user_context['context'] = self;
  user_context = {}
178 179 180

  # Saving the initial globals dict so as to compare it after code execution
  globals_dict = globals()
181
  result_string = ''
182
  notebook_context = old_notebook_context
183

184 185 186
  # Execute only if jupyter_code is not empty
  if jupyter_code:
    # Create ast parse tree
187 188 189 190 191
    try:
      ast_node = ast.parse(jupyter_code)
    except Exception as e:
      # It's not necessary to abort the current transaction here 'cause the 
      # user's code wasn't executed at all yet.
192
      return getErrorMessageForException(self, e, notebook_context)
193
    
194 195 196 197 198 199
    # Fixing "normal" imports and detecting environment object usage
    import_fixer = ImportFixer()
    environment_collector = EnvironmentParser()
    ast_node = import_fixer.visit(ast_node)
    ast_node = environment_collector.visit(ast_node)
    
200 201 202
    # Get the node list from the parsed tree
    nodelist = ast_node.body

203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
    # Handle case for empty nodelist(in case of comments as jupyter_code)
    if nodelist:
      # If the last node is instance of ast.Expr, set its interactivity as 'last'
      # This would be the case if the last node is expression
      if isinstance(nodelist[-1], ast.Expr):
        interactivity = "last"
      else:
        interactivity = "none"

      # Here, we define which nodes to execute with 'single' and which to execute
      # with 'exec' mode.
      if interactivity == 'none':
        to_run_exec, to_run_interactive = nodelist, []
      elif interactivity == 'last':
        to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]

219 220 221
      # TODO: fix this global handling by replacing the print statement with
      # a custom print function. Tip: create an ast.NodeTransformer, like the
      # one used to fix imports.
222 223 224
      old_stdout = sys.stdout
      result = StringIO()
      sys.stdout = result
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
      
      # Variables used at the display hook to get the proper form to display
      # the last returning variable of any code cell.
      display_data = {'result': '', 'mime_type': None}
      
      # This is where one part of the  display magic happens. We create an 
      # instance of ProcessorList and add each of the built-in processors.
      # The classes which each of them are responsiblefor rendering are defined
      # in the classes themselves.
      #
      # The customized display hook will automatically use the processor
      # of the matching class to decide how the object should be displayed.
      processor_list = ProcessorList()
      processor_list.addProcessor(IPythonDisplayObjectProcessor)
      processor_list.addProcessor(MatplotlibFigureProcessor)
      processor_list.addProcessor(ERP5ImageProcessor)
      processor_list.addProcessor(IPythonDisplayObjectProcessor)
      
243 244 245 246 247 248 249 250 251
      # Putting necessary variables in the `exec` calls context and storing
      inject_variable_dict = {
        'context': self,
        'environment': Environment(),
        '_display_data': display_data,
        '_processor_list': processor_list,
        '_volatile_variable_list': []
      }
      user_context.update(inject_variable_dict)
252
      user_context.update(notebook_context['variables'])
253 254 255 256 257 258 259 260 261
      
      # Getting the environment setup defined in the current code cell
      current_setup_dict = environment_collector.getEnvironmentSetupDict()
      current_var_dict = environment_collector.getEnvironmentVarDict()

      # Removing old setup from the setup functions
      removed_setup_message_list = []
      for func_alias in environment_collector.getEnvironmentRemoveList():
        found = False
262
        for key, data in notebook_context['setup'].items():
263 264 265
          if key == func_alias:
            found = True
            func_name = data['func_name']
266
            del notebook_context['setup'][func_alias]
267 268 269 270 271 272 273 274 275 276 277 278
            try:
              del user_context[func_alias]
            except KeyError:
              pass
            removed_setup_message = (
              "%s (%s) was removed from the setup list. "
              "Variables it may have added to the context and are not pickleable "
              "were automatically removed.\n"
            ) % (func_name, func_alias)
            removed_setup_message_list.append(removed_setup_message)
            break
        if not found:
279 280
          transaction.abort()
          raise Exception("Trying to remove non existing function/variable from environment: '%s'\nEnvironment: %s" % (func_alias, str(notebook_context['setup'])))
281 282 283
      
      # Removing all the setup functions if user call environment.clearAll()
      if environment_collector.clearAll():
284
        keys = notebook_context ['setup'].keys()
285
        for key in keys:
286
          del notebook_context['setup'][key]
287 288
      
      # Running all the setup functions that we got
289
      for key, value in notebook_context['setup'].iteritems():
290 291 292 293
        try:
          code = compile(value['code'], '<string>', 'exec')
          exec(code, user_context, user_context)
        # An error happened, so we show the user the stacktrace along with a
294
        # note that the exception happened in a setup function's code.
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
        except Exception as e:
          if value['func_name'] in user_context:
            del user_context[value['func_name']]
          error_return_dict =  getErrorMessageForException(self, e, notebook_context)
          additional_information = "An error happened when trying to run the one of your setup functions:"
          error_return_dict['traceback'].insert(0, additional_information)
          # As in any other user's code execution, transaction needs to be 
          # aborted.
          transaction.abort()
          return error_return_dict
      
      # Iterating over envinronment.define calls captured by the environment collector
      # that are functions and saving them as setup functions.
      for func_name, data in current_setup_dict.iteritems():
        setup_string = (
          "%s\n"
          "_result = %s()\n"
          "if _result and isinstance(_result, dict):\n"
          "    globals().update(_result)\n"
          "_volatile_variable_list += _result.keys()\n"
          "del %s, _result\n"
        ) % (data['code'], func_name, func_name)
317
        notebook_context['setup'][data['alias']] = {
318 319 320 321 322 323 324 325
          "func_name": func_name,
          "code": setup_string
        }

      # Iterating over envinronment.define calls captured by the environment collector
      # that are simple variables and saving them in the setup.
      for variable, value, in current_var_dict.iteritems():
        setup_string = "%s = %s\n" % (variable, repr(value))
326
        notebook_context['setup'][variable] = {
327 328 329 330 331 332
          'func_name': variable,
          'code': setup_string
        }
        user_context['_volatile_variable_list'] += variable
        
      if environment_collector.showEnvironmentSetup():
333
        result_string += "%s\n" % str(notebook_context['setup'])
334 335 336 337 338

      # Execute the nodes with 'exec' mode
      for node in to_run_exec:
        mod = ast.Module([node])
        code = compile(mod, '<string>', "exec")
339
        try:
340
          exec(code, user_context, user_context)
341 342 343 344 345 346
        except Exception as e:
          # Abort the current transaction. As a consequence, the notebook lines
          # are not added if an exception occurs.
          #
          # TODO: store which notebook line generated which exception.
          transaction.abort()
347 348
          # Clear the portal cache from previous transaction
          return getErrorMessageForException(self, e, notebook_context)
349 350 351 352

      # Execute the interactive nodes with 'single' mode
      for node in to_run_interactive:
        mod = ast.Interactive([node])
353
        try:
354 355
          code = compile(mod, '<string>', 'single')
          exec(code, user_context, user_context)
356 357 358 359 360 361
        except Exception as e:
          # Abort the current transaction. As a consequence, the notebook lines
          # are not added if an exception occurs.
          #
          # TODO: store which notebook line generated which exception.
          transaction.abort()
362 363
          # Clear the portal cache from previous transaction
          return getErrorMessageForException(self, e, notebook_context)
364 365

      sys.stdout = old_stdout
366
      mime_type = display_data['mime_type'] or mime_type
367 368
      result_string += "\n".join(removed_setup_message_list) + result.getvalue() + display_data['result']
    
369 370
    # Saves a list of all the variables we injected into the user context and
    # shall be deleted before saving the context.
371
    volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context['_volatile_variable_list']
372 373
    volatile_variable_list.append('__builtins__')

374
    for key, val in user_context.items():
375
      if not key in globals_dict.keys() and not isinstance(val, ModuleType) and not key in volatile_variable_list:
376
        if canSerialize(val):
377 378
          notebook_context['variables'][key] = val
        else:
379
          del user_context[key]
380
          result_string += (
381
            "Cannot serialize the variable named %s whose value is %s, "
382 383 384 385
            "thus it will not be stored in the context. "
            "You should move it's definition to a function and " 
            "use the environment object to load it.\n"
          ) % (key, val)
386 387
    
    # Deleting from the variable storage the keys that are not in the user 
388 389
    # context anymore (i.e., variables that are deleted by the user).
    for key in notebook_context['variables'].keys():
390
      if not key in user_context:
391
        del notebook_context['variables'][key]
392
  
393 394
  result = {
    'result_string': result_string,
395
    'notebook_context': notebook_context,
396
    'status': status,
397
    'mime_type': mime_type,
398 399
    'evalue': evalue,
    'ename': ename,
400
    'traceback': tb_list,
401 402
  }
  return result
403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459


def canSerialize(obj):
  result = False
        
  container_type_tuple = (list, tuple, dict, set, frozenset)
  
  # if object is a container, we need to check its elements for presence of
  # objects that cannot be put inside the zodb
  if isinstance(obj, container_type_tuple):
    if isinstance(obj, dict):
      result_list = []
      for key, value in obj.iteritems():
        result_list.append(canSerialize(key))
        result_list.append(canSerialize(value))
    else:
      result_list = [canSerialize(element) for element in obj]
    return all(result_list)
  # if obj is an object and implements __getstate__, ZODB.serialize can check
  # if we can store it
  elif isinstance(obj, object) and hasattr(obj, '__getstate__'):
    # Need to unwrap the variable, otherwise we get a TypeError, because
    # objects cannot be pickled while inside an acquisition wrapper.
    unwrapped_obj = Acquisition.aq_base(obj)
    writer = ObjectWriter(unwrapped_obj)
    for obj in writer:
      try:
        writer.serialize(obj)
      # Because writer.serialize(obj) relies on the implementation of __getstate__
      # of obj, all errors can happen, so the "except all" is necessary here. 
      except:
        return False
    return True
  else:
    # If cannot serialize object with ZODB.serialize, try with cPickle
    # Only a dump of the object is not enough. Dumping and trying to
    # load it will properly raise errors in all possible situations, 
    # for example: if the user defines a dict with an object of a class 
    # that he created the dump will stil work, but the load will fail. 
    try:
      cPickle.loads(cPickle.dumps(obj))
    # By unknowing reasons, trying to catch cPickle.PicklingError in the "normal"
    # way isn't working. This issue might be related to some weirdness in 
    # pickle/cPickle that is reported in this issue: http://bugs.python.org/issue1457119.
    #
    # So, as a temporary fix, we're investigating the exception's class name as
    # string to be able to identify them.
    # 
    # Even though the issue seems complicated, this quickfix should be 
    # properly rewritten in a better way as soon as possible.
    except Exception as e:
      if type(e).__name__ in ('PicklingError', 'TypeError', 'NameError', 'AttributeError'):
        return False
      else:
        raise e
    else:
      return True
460
  
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669

class EnvironmentParser(ast.NodeTransformer):
  """
    EnvironmentParser class is an AST transformer that walks in the abstract
    code syntax tree to find calls to `define` and `undefine`  on a variable
    named `environment`.
    
    The `define` call should receive a function, which will have it's code
    stored as string in `self.environment_setup_dict`. If only kw args are 
    provided, the variables definition will be stored in self.environment_var_dict.
    
    The `undefine` call will removed keys in self.environment_setup_dict.
  """

  def __init__(self):
    self.environment_setup_dict = {}
    self.environment_var_dict = {}
    self.environment_remove_list = []
    self.function_dict = {}
    self.environment_clear_all = False
    self.show_environment_setup = False

  def visit_FunctionDef(self, node):
    """
      Stores all the function nodes in a dictionary to be accesed later when
      we detect they are used as parameters for an `environment.define` call.
    """
    self.function_dict[node.name] = node
    return node

  def visit_Expr(self, node):
    """
      Visits expressions and check if they are in the form of either 
      `environment.define` or `environment.undefine` properly stores the 
      arguments definition as string.
    """
    value = node.value
    if isinstance(value, ast.Call):
      function = value.func
      if isinstance(function, ast.Attribute):
        attribute = function.value
        if isinstance(attribute, ast.Name):
          name = attribute.id
          if name == 'environment' and function.attr == 'define' and not value.keywords:
            if not len(value.args) == 2:
              message = (
                'Not enough arguments for environment definition. Function '
                'name and alias are required.'
              )
              raise Exception(message)
            func_name = value.args[0].id
            func_alias = value.args[1].s
            function_node = self.function_dict[func_name]
            function_string = astor.to_source(function_node)
            self.environment_setup_dict[func_name] = {
              "code": function_string,
              "alias": func_alias
            }
          elif name == 'environment' and function.attr == 'define' and value.keywords:
            for keyword in value.keywords:
              arg_name = keyword.arg
              arg_value_node = keyword.value
              
              # The value can be a number, string or name. We need to handle 
              # them separatedly. This dict trick was used to avoid the very
              # ugly if.
              node_value_dict = {
                ast.Num: lambda node: str(node.n),
                ast.Str: lambda node: node.s,
                ast.Name: lambda node: node.id
              }
              arg_value = node_value_dict[type(arg_value_node)](arg_value_node)
              self.environment_var_dict[arg_name] = arg_value
          elif name == 'environment' and function.attr == 'undefine':
            func_alias = value.args[0].s
            self.environment_remove_list.append(func_alias)
          elif name == 'environment' and function.attr == 'clearAll':
            self.environment_clear_all = True
          elif name == 'environment'and function.attr == 'showSetup':
            self.show_environment_setup = True
    return node
    
  def clearAll(self):
    return self.environment_clear_all
    
  def showEnvironmentSetup(self):
    return self.show_environment_setup

  def getEnvironmentSetupDict(self):
    return self.environment_setup_dict
    
  def getEnvironmentVarDict(self):
    return self.environment_var_dict
    
  def getEnvironmentRemoveList(self):
    return self.environment_remove_list


class Environment(object):
  """
   Dumb object used to receive call on an object named `environment` inside
   user context. These calls will be tracked by the EnvironmentParser calls.
  """
  
  def define(self, *args, **kwargs):
    pass
  
  def undefine(self, name):
    pass
        
  def clearAll(self):
    pass
    
  def showSetup(self):
    pass
  

class ImportFixer(ast.NodeTransformer):
  """
   The ImportFixer class is responsivle for fixing "normal" imports that users
   might try to execute.
   
   It will automatically replace them with the proper usage of the environment
   object using AST manipulation.
  """
  
  def __init__(self):
    self.import_func_dict = {}
  
  def visit_FunctionDef(self, node):
    """
      Processes funcion definition nodes. We want to store a list of all the 
      import that are inside functions, because they do not affect the outter
      user context, thus do not imply in any un-pickleable variable being added
      there.
    """
    for child in node.body:
      if isinstance(child, ast.Import):
        for alias in child.names:
          self.import_func_dict[alias.name] = node.name  
    return self.generic_visit(node)
    
  def visit_ImportFrom(self, node):
    """
     Fixes `import x from y` statements in the same way `import y` is fixed.
    """
    return self.visit_Import(node)

  def visit_Import(self, node):
    """
    This function replaces `normal` imports by creating AST nodes to define
    and environment function which setups the module and return it to be merged
    with the user context.
    """
    module_name = node.names[0].name
    if getattr(node.names[0], 'asname'):
      module_name = node.names[0].asname
    if not self.import_func_dict.get(module_name):
      empty_function = self.newEmptyFunction("%s_setup" % module_name)
      return_dict = self.newReturnDict(module_name)
      empty_function.body = [node, return_dict]
      environment_set = self.newEnvironmentSetCall("%s_setup" % module_name)
      warning = self.newImportWarningCall(module_name)
      return [empty_function, environment_set, warning]
    else:
      return node

  def newEmptyFunction(self, func_name):
    """
      Return a AST.Function object representing a function with name `func_name`
      and an empty body.
    """
    func_body = "def %s(): pass" % func_name
    return ast.parse(func_body).body[0]

  def newReturnDict(self, module_name):
    """
      Return an AST.Expr representing a returned dict with one single key named
      `'module_name'` (as string) which returns the variable `module_name` (as 
      exoression).
    """
    return_dict = "return {'%s': %s}" % (module_name, module_name)
    return ast.parse(return_dict).body[0]

  def newEnvironmentSetCall(self, func_name):
    """
      Return an AST.Expr representaion an `environment.define` call receiving
      `func_name` (as an expression) and `'func_name'` (as string).
    """
    code_string = "environment.define(%s, '%s')" % (func_name, func_name)
    tree = ast.parse(code_string)
    return tree.body[0]

  def newImportWarningCall(self, module_name):
    """
      Return an AST.Expr representanting a print statement with a warning to an
      user about the import of a module named `module_name` and instructs him
      on how to fix it.
    """
    warning = ("print '"
               "WARNING: Your imported the module %s without using "
               "the environment object, which is not recomended. "
               "Your import was automatically converted to use such method."
               "The setup function registered was named %s_setup.\\n" 
               "'") % (module_name, module_name)
    tree = ast.parse(warning)
    return tree.body[0]

  
670 671 672 673 674 675 676 677 678 679 680 681
def renderAsHtml(self, renderable_object):
  '''
    renderAsHtml will render its parameter as HTML by using the matching 
    display processor for that class. Some processors can be found in this
    file. 
  '''
  # Ugly frame hack to access the processor list defined in the body of the
  # kernel's code, where `exec` is called.
  #
  # At this point the stack should be, from top to the bottom:
  #
  #   5. ExternalMethod Patch call
682
  #   4. Base_runJupyterCode frame (where we want to change variable)
683 684 685 686 687 688 689 690 691 692 693 694
  #   3. exec call to run the user's code
  #   2. ExternalMethod Patch call through `context.Base_renderAsHtml` in the notebook
  #   1. renderAsHtml frame (where the function is)
  # 
  # So sys._getframe(3) is enough to get us up into the frame we want.
  #
  compile_jupyter_frame = sys._getframe(3)
  compile_jupyter_locals = compile_jupyter_frame.f_locals
  processor = compile_jupyter_locals['processor_list'].getProcessorFor(renderable_object)
  result, mime_type = processor(renderable_object).process()
  compile_jupyter_locals['result'].write(result)
  compile_jupyter_locals['display_data']['mime_type'] = 'text/html'
695

696
def getErrorMessageForException(self, exception, notebook_context):
697 698
  '''
    getErrorMessageForException receives an Expcetion object and a context for
699
    code execution (notebook_context) and will return a dict as Jupyter
700 701 702 703 704 705 706
    requires for error rendering.
  '''
  etype, value, tb = sys.exc_info()
  traceback_text = traceback.format_exc().split('\n')[:-1]
  return {
    'status': 'error',
    'result_string': None,
707
    'notebook_context': notebook_context,
708 709 710 711 712 713
    'mime_type': 'text/plain',
    'evalue': str(value),
    'ename': exception.__class__.__name__,
    'traceback': traceback_text
  }

714
def createNotebookContext(self):
715
  """
716
  Function to create an empty notebook context.
717
  """
718
  return {'variables': {}, 'setup': {}}
719

720 721 722 723 724 725 726
class ObjectProcessor(object):
  '''
    Basic object processor that stores the first parameters of the constructor
    in the `subject` attribute and store the target classes for that processor.
  '''
  TARGET_CLASSES=None
  TARGET_MODULES=None
727
  
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
  @classmethod
  def getTargetClasses(cls):
    return cls.TARGET_CLASSES
    
  @classmethod
  def getTargetModules(cls):
    return cls.TARGET_MODULES
    
  def __init__(self, something):
    self.subject = something

class MatplotlibFigureProcessor(ObjectProcessor):
  '''
    MatplotlibFigureProcessor handles the rich display of 
    matplotlib.figure.Figure objects. It displays them using an img tag with
    the inline png image encoded as base64.
  '''
  TARGET_CLASSES=[Figure,]
  TARGET_MODULES=['matplotlib.pyplot',]

  def process(self):
    image_io = StringIO()
    self.subject.savefig(image_io, format='png')
    image_io.seek(0)
    return self._getImageHtml(image_io), 'text/html'
753
  
754 755 756 757 758 759 760 761 762 763
  def _getImageHtml(self, image_io):
    return '<img src="data:image/png;base64,%s" /><br />' % base64.b64encode(image_io.getvalue())
    
class ERP5ImageProcessor(ObjectProcessor):
  '''
   ERP5ImageProcessor handles the rich display of ERP5's image_module object.
   It gets the image data and content type and use them to create a proper img
   tag.
  '''
  TARGET_CLASSES=[Image,]
764
  
765 766 767 768 769
  def process(self):
    from base64 import b64encode
    figure_data = b64encode(self.subject.getData())
    mime_type = self.subject.getContentType()
    return '<img src="data:%s;base64,%s" /><br />' % (mime_type, figure_data), 'text/html'
770

771 772 773 774 775 776 777 778 779 780
class IPythonDisplayObjectProcessor(ObjectProcessor):
  '''
    IPythonDisplayObjectProcessor handles the display of all objects from the
    IPython.display module, including: Audio, IFrame, YouTubeVideo, VimeoVideo, 
    ScribdDocument, FileLink, and FileLinks. 
    
    All these objects have the `_repr_html_` method, which is used by the class
    to render them.
  '''
  TARGET_CLASSES=[DisplayObject, IFrame]
781
  
782 783 784 785 786 787 788 789
  def process(self):
    html_repr = self.subject._repr_html_()
    return html_repr + '<br />', 'text/html' 

class GenericProcessor(ObjectProcessor):
  '''
    Generic processor to render objects as string.
  '''
790
  
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
  def process(self):
    return str(self.subject), 'text/plain'
    
class ProcessorList(object):
  '''
    ProcessorList is responsible to store all the processors in a dict using
    the classes they handle as the key. Subclasses of these classes will have
    the same processor of the eigen class. This means that the order of adding
    processors is important, as some classes' processors may be overwritten in
    some situations.
    
    The `getProcessorFor` method uses `something.__class__' and not 
    `type(something)` because using the later onobjects returned by portal 
    catalog queries will return an AcquisitionWrapper type instead of the 
    object's real class.
  '''
807
  
808 809 810
  def __init__(self, default=GenericProcessor):
    self.processors = {}
    self.default_processor = GenericProcessor
811
  
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
  def addProcessor(self, processor):
    classes = processor.getTargetClasses()
    modules = processor.getTargetModules()
    
    if classes and not len(classes) == 0:
      for klass in classes:
        self.processors[klass] = processor
        for subclass in klass.__subclasses__():
          self.processors[subclass] = processor
      
    if modules and not len(modules) == 0:
      for module in modules:
        self.processors[module] = processor
        
  def getProcessorFor(self, something):
    if not isinstance(something, ModuleType):
      return self.processors.get(something.__class__, self.default_processor)
    else:
      return self.processors.get(something.__name__, self.default_processor)
831

832

833
def storeIFrame(self, html, key):
834
  self.portal_caches.erp5_pivottable_frame_cache.set(key, html)
835 836
  return True

837

838 839 840 841 842
# WARNING! 
# 
# This is a highly experimental PivotTableJs integration which does not follow
# ERP5 Javascrpt standards and it will be refactored to use JIO and RenderJS.
#
843
def erp5PivotTableUI(self, df):
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
  from IPython.display import IFrame
  template = """
  <!DOCTYPE html>
  <html>
    <head>
      <title>PivotTable.js</title>

      <!-- external libs from cdnjs -->
      <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.css">
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.11.4/jquery-ui.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery-csv/0.71/jquery.csv-0.71.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.js"></script>

      <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/pivot.min.css">
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/pivot.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/d3_renderers.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/c3_renderers.min.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/export_renderers.min.js"></script>

      <style>
        body {font-family: Verdana;}
        .node {
         border: solid 1px white;
         font: 10px sans-serif;
         line-height: 12px;
         overflow: hidden;
         position: absolute;
         text-indent: 2px;
        }
        .c3-line, .c3-focused {stroke-width: 3px !important;}
        .c3-bar {stroke: white !important; stroke-width: 1;}
        .c3 text { font-size: 12px; color: grey;}
        .tick line {stroke: white;}
        .c3-axis path {stroke: grey;}
        .c3-circle { opacity: 1 !important; }
      </style>
    </head>
    <body>
      <script type="text/javascript">
        $(function(){
          if(window.location != window.parent.location)
            $("<a>", {target:"_blank", href:""})
              .text("[pop out]").prependTo($("body"));

          $("#output").pivotUI( 
            $.csv.toArrays($("#output").text()), 
            { 
              renderers: $.extend(
                $.pivotUtilities.renderers, 
                $.pivotUtilities.c3_renderers, 
                $.pivotUtilities.d3_renderers,
                $.pivotUtilities.export_renderers
                ),
              hiddenAttributes: [""]
            }
          ).show();
         });
      </script>
      <div id="output" style="display: none;">%s</div>
    </body>
  </html>
907
  """
908 909 910 911
  html_string = template % df.to_csv()
  from hashlib import sha512
  key = sha512(html_string).hexdigest()
  storeIFrame(self, html_string, key)
912 913
  iframe_host = self.REQUEST['HTTP_X_FORWARDED_HOST'].split(',')[0]
  url = "https://%s/erp5/Base_displayPivotTableFrame?key=%s" % (iframe_host, key)
914
  return IFrame(src=url, width='100%', height='500')
915