[hal_json_style] Necessary changes to support Accounting Reports

Search mode of getHateoas now is able to facilitate ListBoxes as well
as Related Fields and Reports. If the search is issued by ListBox's
list_method then the method MUST specify all its input arguments and
do not try to obtain them from REQUEST because it is called asynchronously
from the config form submit thus with different request. It will be supplied
only those parameters specified as input parameters (via introspection).

Changes in Base_callDialogMethod ensures compatible Skin Selection
since the skin change happens there.

getHateoas extends its functionality to render ERP5 Report alongside
with ERP5 Form. It unfortunately creates Selections for Report Section
Form's ListBoxes who uses them instead of input parameters. Those
selection hangs in memory because by the time we need to remove them
we don't have the source Report Section to it anymore.

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/Base_callDialogMethod.py

@@ -15,17 +15,20 @@ def isListBox(field):
       return True
   return False
 
-from Products.Formulator.Errors import FormValidationError
+from Products.Formulator.Errors import FormValidationError, ValidationError
 from ZTUtils import make_query
 
 request = REQUEST
 if REQUEST is None:
   request = container.REQUEST
 
+# request.form holds POST data thus containing 'field_' + field.id items
+# such as 'field_your_some_field'
 request_form = request.form
 error_message = ''
+translate = context.Base_translateString
 
-# Make this script work alike wether called from another script or by a request
+# Make this script work alike whether called from another script or by a request
 kw.update(request_form)
 
 # Exceptions for UI
@@ -63,6 +66,7 @@ if dialog_method == 'Base_editRelation':
                                    listbox_uid=kw.get('listbox_uid', None),
                                    saved_form_data=kw['saved_form_data'])
 # Exception for create relation
+# Not used in new UI - relation field implemented using JIO calls from JS
 if dialog_method == 'Base_createRelation':
   return context.Base_createRelation(form_id=kw['form_id'],
                                      selection_name=kw['list_selection_name'],
@@ -80,6 +84,24 @@ if dialog_method == 'Folder_delete':
                                selection_name=kw['selection_name'],
                                md5_object_uid_list=kw['md5_object_uid_list'])
 
+
+def handle_form_error(form, validation_errors):
+  """Return correctly rendered form with all errors assigned to its fields."""
+  field_errors = form.ErrorFields(validation_errors)
+  request.set('field_errors', field_errors)
+  # Make sure editors are pushed back as values into the REQUEST object
+  for f in form.get_fields():
+    field_id = f.id
+    if request.has_key(field_id):
+      value = request.get(field_id)
+      if callable(value):
+        value(request)
+  if silent_mode:
+    return context.ERP5Document_getHateoas(form=form, REQUEST=request, mode='form'), 'form'
+  request.RESPONSE.setStatus(400)
+  return context.ERP5Document_getHateoas(form=form, REQUEST=request, mode='form')
+
+
 form = getattr(context, dialog_id)
 
 # form can be a python script that returns the form
@@ -95,54 +117,54 @@ try:
   request.set('editable_mode', 1)
   form.validate_all_to_request(request)
   request.set('editable_mode', editable_mode)
-except FormValidationError, validation_errors:
-  # Pack errors into the request
-  field_errors = form.ErrorFields(validation_errors)
-  request.set('field_errors', field_errors)
-  # Make sure editors are pushed back as values into the REQUEST object
-  for f in form.get_fields():
-    field_id = f.id
-    if request.has_key(field_id):
-      value = request.get(field_id)
-      if callable(value):
-        value(request)
-  if silent_mode: return context.ERP5Document_getHateoas(form=form, REQUEST=request, mode='form'), 'form'
-  request.RESPONSE.setStatus(400)
-  return context.ERP5Document_getHateoas(form=form, REQUEST=request, mode='form')
 
-# Use REQUEST.redirect if possible. It will not be possible if at least one of these is true :
-#  * we got an import_file,
-#  * we got a listbox
-#  * a value is None or [] or (), because this is not supported by make_query
-can_redirect = 1
+  default_skin = context.getPortalObject().portal_skins.getDefaultSkin()
+  allowed_styles = ("ODT", "ODS", "Hal", "HalRestricted")
+  if getattr(getattr(context, dialog_method), 'pt', None) == "report_view" and \
+     request.get('your_portal_skin', default_skin) not in allowed_styles:
+    # RJS own validation - only ODS/ODT and Hal* skins work for reports
+    # Form is OK, it's just this field - style so we return back form-wide error
+    # for which we don't have support out-of-the-box thus we manually craft it
+    # XXX TODO: Form-wide validation errors
+    return handle_form_error(
+      form,
+      FormValidationError([
+        ValidationError(
+          error_key=None,
+          field=form.get_field('your_portal_skin'),
+          error_text=translate(
+            'Only ODT, ODS, Hal and HalRestricted skins are allowed for reports '\
+            'in Preferences - User Interface - Report Style'))
+      ], {}))
+
+except FormValidationError as validation_errors:
+  return handle_form_error(form, validation_errors)
+
 MARKER = [] # A recognisable default value. Use with 'is', not '=='.
 listbox_id_list = [] # There should not be more than one listbox - but this give us a way to check.
 file_id_list = [] # For uploaded files.
 for field in form.get_fields():
-  k = field.id
-  v = request.get(k, MARKER)
-  if v is not MARKER:
+  field_id = field.id
+  field_value = request.get(field_id, MARKER)
+
+  if field_value is not MARKER:
     if isListBox(field):
-      listbox_id_list.append(k)
-    elif can_redirect and (v in (None, [], ()) or hasattr(v, 'read')) : # If we cannot redirect, useless to test it again
-      can_redirect = 0
-
-    # Cleanup my_ and your_ prefixes
-    splitted = k.split('_', 1)
-    if len(splitted) == 2 and splitted[0] in ('my', 'your'):
-      if hasattr(v, 'as_dict'):
-        # This is an encapsulated editor
-        # convert it
-        kw.update(v.as_dict())
+      listbox_id_list.append(field_id)
+
+    # Cleanup my_ and your_ prefixes if present
+    if field_id.startswith("my_") or field_id.startswith("your_"):
+      field_prefix, field_name = field_id.split('_', 1)
+      if hasattr(field_value, 'as_dict'):
+        # This is an encapsulated editor - convert it
+        kw.update(field_value.as_dict())
       else:
-        kw[splitted[1]] = request_form[splitted[1]] = v
+        kw[field_name] = request_form[field_name] = field_value
 
     else:
-      kw[k] = request_form[k] = v
+      kw[field_id] = request_form[field_id] = field_value
 
 
 if len(listbox_id_list):
-  can_redirect = 0
   # Warn if there are more than one listbox in form ...
   if len(listbox_id_list) > 1:
     log('Base_callDialogMethod', 'There are %s listboxes in form %s.' % (len(listbox_id_list), form.id))
@@ -183,11 +205,8 @@ if listbox_uid is not None and kw.has_key('list_selection_name'):
   selected_uids = context.portal_selections.updateSelectionCheckedUidList(
     kw['list_selection_name'],
     listbox_uid, uids)
-# Remove unused parameter
-clean_kw = {}
-for k, v in kw.items() :
-  if v not in (None, [], ()) :
-    clean_kw[k] = kw[k]
+# Remove empty items
+clean_kw = dict((k, v) for k, v in kw.items() if v not in (None, [], ()))
 
 # Handle deferred style, unless we are executing the update action
 if dialog_method != update_method and clean_kw.get('deferred_style', 0):
@@ -195,8 +214,23 @@ if dialog_method != update_method and clean_kw.get('deferred_style', 0):
   # XXX Hardcoded Deferred style name
   clean_kw['portal_skin'] = 'Deferred'
   
-  dialog_form = getattr(context, dialog_method)
-  page_template = getattr(dialog_form, 'pt', None)
+  page_template = getattr(getattr(context, dialog_method), 'pt', None)
+
+  if page_template == 'report_view':
+    # Limit Reports in Deferred style to known working styles
+    if request_form.get('your_portal_skin', None) not in ("ODT", "ODS"):
+      # RJS own validation - deferred option works here only with ODS/ODT skins
+      return handle_form_error(
+        form,
+        FormValidationError([
+          ValidationError(
+            error_key=None,
+            field=form.get_field('your_deferred_style'),
+            error_text=translate(
+              'Deferred reports are possible only with preference '\
+              '"Report Style" set to "ODT" or "ODS"'))
+        ], {}))
+
   # If the action form has report_view as it's method, it 
   if page_template != 'report_view':
     # use simple wrapper
@@ -205,23 +239,24 @@ if dialog_method != update_method and clean_kw.get('deferred_style', 0):
     request.set('deferred_style_dialog_method', dialog_method)
     dialog_method = 'Base_activateSimpleView'
 
-
 url_params_string = make_query(clean_kw)
 
-# XXX: We always redirect in report mode to make sure portal_skin
-# parameter is taken into account by SkinTool.
-# If url is too long, we do not redirect to avoid crash.
-# XXX: 2000 is an arbitrary value resulted from trial and error.
-if (not(can_redirect) or len(url_params_string) > 2000):
+# Never redirect in JSON style - do as much as possible here.
+# At this point the 'dialog_method' should point to a form (if we are in report)
+# if we are not in Deferred mode - then it points to `Base_activateSimpleView`
+
+if True:
   if dialog_method != update_method:
     # When we are not executing the update action, we have to change the skin
     # manually,
     if 'portal_skin' in clean_kw:
       new_skin_name = clean_kw['portal_skin']
+      context.log("Changing skin to \"{}\"".format(new_skin_name))
       context.getPortalObject().portal_skins.changeSkin(new_skin_name)
       request.set('portal_skin', new_skin_name)
       deferred_portal_skin = clean_kw.get('deferred_portal_skin')
       if deferred_portal_skin:
+        # has to be either ODS or ODT because only those contain `form_list`
         request.set('deferred_portal_skin', deferred_portal_skin)
     # and to cleanup formulator's special key in request
     # XXX unless we are in Folder_modifyWorkflowStatus which validates again !
@@ -230,15 +265,28 @@ if (not(can_redirect) or len(url_params_string) > 2000):
         if str(key).startswith('field') or str(key).startswith('subfield'):
           request.form.pop(key, None)
 
-  # If we cannot redirect, then call the form directly.
+    # now get dialog_method after skin re-selection and dialog_method mingling
     dialog_form = getattr(context, dialog_method)
+
     # XXX: this is a hack that should not be needed anymore with the new listbox.
     # set the URL in request, so that we can immediatly call method
     # that depend on it (eg. Show All). This is really related to
     # current ListBox implementation which edit Selection's last_url
     # with the content of REQUEST.URL
     request.set('URL', '%s/%s' % (context.absolute_url(), dialog_method))
+
+    # RJS: If we are in deferred mode - call the form directly and return
+    # dialog method is now `Base_activateSimpleView` - the only script in
+    # deferred portal_skins folder
+    if clean_kw.get('deferred_style', 0):
+      return dialog_form(**kw)  # deferred form should return redirect with a message
+
+    # RJS: If skin selection is different than Hal* then ERP5Document_getHateoas
+    # does not exist and we call form method directly
+    if clean_kw.get("portal_skin", context.getPortalObject().portal_skins.getDefaultSkin()) not in ("Hal", "HalRestricted"):
       return dialog_form(**kw)
 
-dialog_method = getattr(context, dialog_method)
-return dialog_method(**clean_kw)
+    return context.ERP5Document_getHateoas(REQUEST=request, form=dialog_form, mode="form")
+
+# XXX If somebody knows in which case this gets executed ... please tell me.
+return getattr(context, dialog_method)(**kw)

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.py

+"""Hello. This will be long because this goodness script does almost everything.
+
+In general it always returns a JSON reponse in HATEOAS format specification.
+
+:param REQUEST: HttpRequest holding GET and/or POST data
+:param response:
+:param view: either "view" or absolute URL of an ERP5 Action
+:param mode: {str} help to decide what user wants from us "form" | "search" ...
+:param relative_url: an URL of `traversed_document` to operate on (it must have an object_view)
+
+Only in mode == 'search'
+:param query:
+:param select_list:
+:param limit:
+
+Only in mode == 'form'
+:param form:
+
+TBD.
+"""
 from ZTUtils import make_query
 import json
 from base64 import urlsafe_b64encode, urlsafe_b64decode
@@ -9,6 +29,7 @@ from email.Utils import formatdate
 import re
 from zExceptions import Unauthorized
 from Products.ERP5Type.Utils import UpperCase
+from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery
 
 if REQUEST is None:
   REQUEST = context.REQUEST
@@ -78,19 +99,23 @@ def serialize_DateTime(obj):
 
 
 def getProtectedProperty(document, select):
+  """getProtectedProperty is a security-aware substitution for builtin `getattr`
+
+  It resolves Properties on Products (visible via Zope Formulator), which are
+  accessible as ordinary attributes as well, by following security rules.
+
+  See https://lab.nexedi.com/nexedi/erp5/blob/master/product/ERP5Form/ListBox.py#L2293
+  """
   try:
-    #see https://lab.nexedi.com/nexedi/erp5/blob/master/product/ERP5Form/ListBox.py#L2293
-    try:
+    if "." in select:
       select = select[select.rindex('.') + 1:]
-    except ValueError:
-      pass
-
     return document.getProperty(select, d=None)
   except (ConflictError, RuntimeError):
     raise
   except:
     return None
 
+
 url_template_dict = {
   "form_action": "%(traversed_document_url)s/%(action_id)s",
   "traverse_generator": "%(root_url)s/%(script_id)s?mode=traverse" + \
@@ -133,7 +158,11 @@ def getRealRelativeUrl(document):
 
 def getFormRelativeUrl(form):
   return portal.portal_catalog(
-    portal_type="ERP5 Form",
+    portal_type=ComplexQuery(
+      Query(portal_type="ERP5 Form"),
+      Query(portal_type="ERP5 Report"),
+      logical_operator='or'
+    ),
     uid=form.getUid(),
     id=form.getId(),
     limit=1,
@@ -152,12 +181,14 @@ def getFieldDefault(traversed_document, field, key, value=None):
 def renderField(traversed_document, field, form, value=None, meta_type=None, key=None, key_prefix=None, selection_params=None):
   """Extract important field's attributes into `result` dictionary."""
 
+  if selection_params is None:
+    selection_params = {}
+
   if meta_type is None:
     meta_type = field.meta_type
   if key is None:
     key = field.generate_field_key(key_prefix=key_prefix)
-  context.log("traversed_document={!s}, field={!s}, form={!s}, value={!s}, meta_type={!s}".format(
-    traversed_document, field, form, value, meta_type))
+
   result = {
     "type": meta_type,
     "title": Base_translateString(field.get_value("title")),
@@ -358,11 +389,20 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
     return result
 
   if meta_type == "ListBox":
-    """Display list of objects with optional search/sort capabilities on columns from catalog."""
+    """Display list of objects with optional search/sort capabilities on columns from catalog.
+
+    We might be inside a ReportBox which is inside a parent form BUT we still have access to
+    the original REQUEST with sent POST values from the parent form. We can save those
+    values into our query method and reconstruct them meanwhile calling asynchronous jio.allDocs.
+    """
     _translate = Base_translateString
 
-    column_list = [(name, _translate(title)) for name, title in field.get_value("columns")]
-    editable_column_list = [(name, _translate(title)) for name, title in field.get_value("editable_columns")]
+    # column definition in ListBox own value 'columns' is superseded by dynamic
+    # column definition from Selection for specific Report ListBoxes; the same for editable_columns
+    column_list = [(name, _translate(title)) for name, title in (selection_params.get('selection_columns', [])
+                                                                 or field.get_value("columns"))]
+    editable_column_list = [(name, _translate(title)) for name, title in (selection_params.get('editable_columns', [])
+                                                                          or field.get_value("editable_columns"))]
     catalog_column_list = [(name, title)
                            for name, title in column_list
                            if sql_catalog.isValidColumn(name)]
@@ -374,7 +414,8 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
 
     # try to get specified sortable columns and fail back to searchable fields
     sort_column_list = [(name, _translate(title))
-                        for name, title in field.get_value("sort_columns")
+                        for name, title in (selection_params.get('selection_sort_order', [])
+                                            or field.get_value("sort_columns"))
                         if sql_catalog.isValidColumn(name)] or search_column_list
 
     # requirement: get only sortable/searchable columns which are already displayed in listbox
@@ -392,8 +433,6 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
     
     lines = field.get_value('lines')
     list_method_name = traversed_document.Listbox_getListMethodName(field)
-    context.log("ListBox '{!s}'\n >> selection params: {!s}\n >> default params: {!s} ".format(
-      field.absolute_url(), selection_params, default_params))
 
     # ListBoxes in report view has portal_type defined already in default_params
     # in that case we prefer non_empty version
@@ -415,6 +454,9 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
         # which we will not introspect
         pass
 
+    # Put all ListBox's search method params from REQUEST to `default_param_json`
+    # because old code expects synchronous render thus having all form's values
+    # still in the request which is not our case because we do asynchronous rendering
     if list_method is not None and hasattr(list_method, "ZScriptHTML_tryParams"):
       for list_method_param in list_method.ZScriptHTML_tryParams():
         if list_method_param in REQUEST and list_method_param not in list_method_query_dict:
@@ -422,6 +464,10 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
       # MIDDLE-DANGEROUS!
       # In case of reports (later even exports) substitute None for unknown 
       # parameters. We suppose Python syntax for parameters!
+      # What we do here is literally putting every form field from REQUEST
+      # into search method parameters - this is later put back into REQUEST
+      # this way we can mimic synchronous rendering when all form field values
+      # were available in REQUEST. It is obviously wrong behaviour.
       for list_method_param in list_method.params().split(","):
         if "*" in list_method_param:
           continue
@@ -546,6 +592,12 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
 
 
 def renderForm(traversed_document, form, response_dict, key_prefix=None, selection_params=None):
+  """
+  :param selection_params: holds parameters to construct ERP5Form.Selection instance
+      for underlaying ListBox - since we do not use selections in RenderJS UI
+      we mitigate the functionality here by overriding ListBox's own values
+      for columns, editable columns, and sort with those found in `selection_params`
+  """
   REQUEST.set('here', traversed_document)
   field_errors = REQUEST.get('field_errors', {})
 
@@ -625,16 +677,27 @@ def renderForm(traversed_document, form, response_dict, key_prefix=None, selecti
   }
 
   if (form.pt == 'report_view'):
+    # reports are expected to return list of ReportSection which is a wrapper
+    # around a form - thus we will need to render those forms
     report_item_list = []
     report_result_list = []
     for field in form.get_fields():
       if field.getRecursiveTemplateField().meta_type == 'ReportBox':
+        # ReportBox.render returns a list of ReportSection classes which are
+        # just containers for FormId(s) usually containing one ListBox
+        # and its search/query parameters hidden in `selection_params`
+        # `path` contains relative_url of intended CONTEXT for underlaying ListBox
         report_item_list.extend(field.render())
-    j = 0
-    for report_item in report_item_list:
-      report_context = report_item.getObject(portal)
-      report_prefix = 'x%s' % j
-      j += 1
+    # ERP5 Report document differs from a ERP5 Form in only one thing: it has
+    # `report_method` attached to it - thus we call it right here
+    if hasattr(form, 'report_method'):
+      report_method_name = getattr(form, 'report_method')
+      report_method = getattr(traversed_document, report_method_name)
+      report_item_list.extend(report_method())
+
+    for report_index, report_item in enumerate(report_item_list):
+      report_context = report_item.getObject(traversed_document)
+      report_prefix = 'x%s' % report_index
       report_title = report_item.getTitle()
       # report_class = "report_title_level_%s" % report_item.getLevel()
       report_form = report_item.getFormId()
@@ -643,13 +706,44 @@ def renderForm(traversed_document, form, response_dict, key_prefix=None, selecti
       # key "portal_type" (don't confuse with "portal_types" in ListBox) into
       # report_item.selection_params thus we need to take that into account in
       # ListBox field
-      renderForm(traversed_document,
+      #
+      # Selection Params are parameters for embedded ListBox's List Method
+      # and it must be passed in `default_json_param` field (might contain
+      # unserializable data types thus we need to take care of that
+      # In order not to lose information we put all ReportSection attributes
+      # inside the report selection params
+      selection_name = report_prefix + "_" + report_item.selection_name
+      report_form_params = {
+        "selection_name": selection_name,
+        "selection_columns": report_item.selection_columns,
+        "selection_sort_order": report_item.selection_sort_order,
+      }
+      report_form_params.update(report_item.selection_params)
+      # this should load selections with correct values - since it is modifying
+      # global state in the backend we have nothing more to do here
+      # I could not find where the code stores params in selection with render
+      # prefix - maybe it in some `render` method where it should not be
+      # Of course it is ugly, terrible and should be removed!
+      selection_tool = context.getPortalObject().portal_selections
+      selection_tool.getSelectionFor(selection_name, REQUEST)
+      selection_tool.setSelectionParamsFor(selection_name, report_form_params)
+      selection_tool.setSelectionColumns(selection_name, report_item.selection_columns)
+
+      # Report section is just a wrapper around form thus we render it right
+      # we keep traversed_document because its Portal Type Class should be 
+      # addressable by the user = have actions (object_view) attached to it
+      # BUT! when Report Section defines `path` that is the new context for
+      # form rendering and subsequent searches...
+      renderForm(traversed_document if not report_item.path else report_context,
                  getattr(report_context, report_item.getFormId()),
                  report_result,
                  key_prefix=report_prefix,
-                 selection_params=None)  # report_item.selection_params XXX: does it matter?
+                 selection_params=report_form_params)  # used to be only report_item.selection_params
+      # Report Title is important since there are more section on report page
+      # but often they render the same form with different data so we need to
+      # distinguish by the title at least.
+      report_result['title'] = report_title
       report_result_list.append(report_result)
-
     response_dict['report_section_list'] = report_result_list
 
 # XXX form action update, etc
@@ -693,6 +787,7 @@ def renderRawField(field):
 
 
 def renderFormDefinition(form, response_dict):
+  """Form "definition" is configurable in Zope admin: Form -> Order."""
   group_list = []
   for group in form.Form_getGroupTitleAndId():
 
@@ -814,11 +909,15 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
 
     # Add a link to the portal type if possible
     if not is_portal:
+      # traversed_document should always have its Portal Type in ERP5 Portal Types
+      # thus attached actions to it so it is viewable
+      document_type_name = traversed_document.getPortalType()
+      document_type = getattr(portal.portal_types, document_type_name, None)
+      if document_type is not None:
         result_dict['_links']['type'] = {
           "href": default_document_uri_template % {
             "root_url": site_root.absolute_url(),
-          "relative_url": portal.portal_types[traversed_document.getPortalType()]\
-                            .getRelativeUrl(), 
+            "relative_url": document_type.getRelativeUrl(),
             "script_id": script.id
           },
           "name": Base_translateString(traversed_document.getPortalType())
@@ -933,7 +1032,6 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       # renderer_form = traversed_document.restrictedTraverse(form_id, None)
       # XXX Proxy field are not correctly handled in traversed_document of web site
       renderer_form = getattr(traversed_document, form_id)
-  #     traversed_document.log(form_id)
       if (renderer_form is not None):
         embedded_dict = {
           '_links': {
@@ -943,14 +1041,23 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           }
         }
         # Put all query parameters (?reset:int=1&workflow_action=start_action) in request to mimic usual form display
+        query_param_dict = {}
         query_split = embedded_url.split('?', 1)
         if len(query_split) == 2:
           for query_parameter in query_split[1].split("&"):
-            query_key, query_value = query_parameter.split("=")
-            # since humans are lazy they are using + in URLs instead of %20
-            # so we mitigate here
+            query_key, query_value = query_parameter.split('=')
+            # often + is used instead of %20 so we replace for space here
+            query_param_dict[query_key] = query_value.replace("+", " ")
+
         # set URL params into REQUEST (just like it was sent by form)
-            REQUEST.set(query_key, query_value.replace("+", " "))
+        for query_key, query_value in query_param_dict.items():
+          REQUEST.set(query_key, query_value)
+
+        # unfortunatelly some people use Scripts as targets for Workflow
+        # transactions - thus we need to check and mitigate
+        if "Script" in renderer_form.meta_type:
+          # we suppose that the script takes only what is given in the URL params
+          return renderer_form(**query_param_dict)
   
         renderForm(traversed_document, renderer_form, embedded_dict)
         result_dict['_embedded'] = {
@@ -1040,7 +1147,7 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
   
     else:
       traversed_document_portal_type = traversed_document.getPortalType()
-      if traversed_document_portal_type == "ERP5 Form":
+      if traversed_document_portal_type in ("ERP5 Form", "ERP5 Report"):
         renderFormDefinition(traversed_document, result_dict)
         response.setHeader("Cache-Control", "private, max-age=1800")
         response.setHeader("Vary", "Cookie,Authorization,Accept-Encoding")
@@ -1086,11 +1193,21 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
     # Default Param JSON contains
     #  portal_type: list of Portal Types to include (singular form matches the
     #                                                catalog column name)
+    #
+    # Discussion:
+    #
+    #  Why you didn't use ListBoxRendererLine?
+    #  > Method 'search' is used for getting related objects as well which are
+    #  > not backed up by a ListBox thus the value resolution would have to be
+    #  > there anyway. It is better to use one code for all in this case.
     #################################################
     if REQUEST.other['method'] != "GET":
       response.setStatus(405)
       return ""
 
+    # in case we have custom list method
+    catalog_kw = {}
+
     # hardcoded responses for site and portal objects (which are not Documents!)
     # we let the flow to continue because the result of a list_method call can
     # be similar - they can in practice return anything
@@ -1116,28 +1233,49 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
               json.loads(urlsafe_b64decode(default_param_json)))))
       if query:
         catalog_kw["full_text"] = query
+
       if sort_on is not None:
+        def parse_sort_on(raw_string):
+          """Turn JSON serialized array into a tuple (col_name, order)."""
+          sort_col, sort_order = json.loads(raw_string)
+          sort_col, sort_order = byteify(sort_col), byteify(sort_order)
+          # JIO keeps sort order as whole word 'ascending' resp. 'descending'
+          if sort_order.lower().startswith("asc"):
+            sort_order = "ASC"
+          elif sort_order.lower().startswith("desc"):
+            sort_order = "DESC"
+          else:
+            # should raise an ValueError instead
+            context.log('Wrong sort order "{}" in {}! It must start with "asc" or "desc"'.format(sort_order, form_relative_url),
+                        level=200)  # error
+          return (sort_col, sort_order)
+
         if isinstance(sort_on, list):
-          catalog_kw['sort_on'] = tuple((byteify(sort_col), byteify(sort_order))
-                                         for sort_col, sort_order in map(json.loads, sort_on))
+          # sort_on argument is always a list of tuples(col_name, order)
+          catalog_kw['sort_on'] = list(map(parse_sort_on, sort_on))
         else:
-          sort_col, sort_order = json.loads(sort_on)
-          catalog_kw['sort_on'] = ((byteify(sort_col), byteify(sort_order)), )
+          catalog_kw['sort_on'] = [parse_sort_on(sort_on), ]
+
+      # Some search scripts impertinently grab their arguments from REQUEST
+      # instead of being nice and specify them as their input parameters.
+      #
+      # We expect that wise and mighty ListBox did copy all form field values
+      # from its REQUEST into `default_param_json` so we can put them back.
+      #
+      # XXX Kato: Seems that current scripts are behaving nicely (using only
+      # specified input parameters). In case some list_method does not work
+      # this is the first place to try to uncomment.
+      #
+      # for k, v in catalog_kw.items():
+      #   REQUEST.set(k, v)
 
-      context.log("callable_list_method(**catalog_kw): {}({!s})".format(
-        list_method, catalog_kw))
       search_result_iterable = callable_list_method(**catalog_kw)
-    context.log('search_result_iterable: {!s}\n >> len {:d}\n >> first item: {!s}'.format(
-      search_result_iterable,
-      len(search_result_iterable),
-      (search_result_iterable[0] if len(search_result_iterable) > 0 else "search_result_iterable empty")))
 
     # Cast to list if only one element is provided
     if select_list is None:
       select_list = []
     elif same_type(select_list, ""):
       select_list = [select_list]
-    context.log("listbox: select_list {!s}".format(select_list))
 
     # extract form field definition into `editable_field_dict`
     editable_field_dict = {}
@@ -1148,6 +1286,9 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       listbox_form = getattr(traversed_document, listbox_field.aq_parent.id)
       for select in select_list:
         # See Listbox.py getValueList --> getEditableField & getColumnAliasList method
+        # In short: there are Form Field definitions which names start with
+        # matching ListBox name - those are template fields to be rendered in
+        # cells with actual values defined by row and column
         field_name = "{}_{}".format(listbox_field_id, select.replace(".", "_"))
         if listbox_form.has_field(field_name, include_disabled=1):
           editable_field_dict[select] = listbox_form.get_field(field_name, include_disabled=1)
@@ -1185,14 +1326,22 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       contents_item = {}
       contents_list.append(contents_item)
 
+      # Fields, which are used to render results of search, use TALES to obtain
+      # their other properties. Thus we set up REQUEST.here which is used in TAL
+      #
+      # XXX Kato: This might be wrong since search_result is expected in 'cell' and
+      # 'here' is reserved for a traversed_document
+      # REQUEST.set('here', search_result)
+
       contents_uid = None
       if hasattr(search_result, "getObject"):
-        search_result = search_result.getObject()
+        # search_result = search_result.getObject()
         contents_uid = search_result.uid
         # every document indexed in catalog has to have relativeUrl
         contents_relative_url = getRealRelativeUrl(search_result)
         # get property in secure way from documents
         search_property_getter = getProtectedProperty
+        search_property_hasser = lambda doc, attr: doc.hasProperty(attr)
       elif hasattr(search_result, "aq_self"):
         # Zope products have at least ID thus we work with that
         contents_uid = search_result.uid
@@ -1200,6 +1349,7 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
         contents_relative_url = getRealRelativeUrl(search_result) or search_result.getId()
         # documents and products have the same way of accessing properties
         search_property_getter = getProtectedProperty
+        search_property_hasser = lambda doc, attr: doc.hasProperty(attr)
       else:
         # In case of reports the `search_result` can be list of 
         # PythonScripts.standard._Object - a reimplementation of plain dictionary
@@ -1212,7 +1362,8 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
         # never happen!) thus we provide temporary UID
         contents_relative_url = "{}/{}".format(traversed_document.getRelativeUrl(), contents_uid)
         # property getter must be simple __getattr__ implementation
-        search_property_getter = getattr
+        search_property_getter = lambda obj, attr: getattr(obj, attr, None)
+        search_property_hasser = lambda obj, attr: hasattr(obj, attr)
 
       # _links.self.href is mandatory for JIO so it can create reference to the
       # (listbox) item alone
@@ -1235,14 +1386,15 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       # render whole field in contents_item or at least search result value 
       for select in select_list:
         if editable_field_dict.has_key(select):
+          # cell has a Form Field template thus render it using the field
           REQUEST.set('cell', search_result)
-  
           # if default value is given by evaluating Tales expression then we only
           # put "cell" to request (expected by tales) and let the field evaluate
           default_field_value = None
           if getattr(editable_field_dict[select].tales, "default", "") == "":
-            # if there is no tales expr we extract the value from search result
+            # if there is no tales expr (or is empty) we extract the value from search result
             default_field_value = search_property_getter(search_result, select)
+
           contents_item[select] = renderField(
             traversed_document,
             editable_field_dict[select],
@@ -1252,21 +1404,78 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           REQUEST.other.pop('cell', None)
 
         else:
-          contents_value = search_property_getter(search_result, select)
+          # if the variable does not have a field template we need to find its
+          # value by resolving value in the correct order. The code is copy&pasted
+          # from ListBoxRendererLine.getValueList because it is universal
+          contents_value = None
+          if "." in select:
+            select = select[select.rindex('.') + 1:]
+
+          if len(select) == 0:
+            context.log('There is an empty column name in {!s}!'.format(select_list), level=200)
+            continue
+
+          # 1. resolve attribute on a raw object (all wrappers removed) using
+          # lowest-level secure getattr method given object type
+          raw_search_result = search_result
+          if hasattr(search_result, 'aq_base'):
+            raw_search_result = search_result.aq_base
+          
+          if search_property_hasser(raw_search_result, select):
+            contents_value = search_property_getter(raw_search_result, select)
+
+          # 2. use the fact that wrappers (brain or acquisition wrapper) use
+          # permissioned getters
+          unwrapped_search_result = search_result
+          if hasattr(search_result, 'aq_self'):
+            unwrapped_search_result = search_result.aq_self
+
           if contents_value is None:
+            if not select.startswith('get') and select[0] not in string.ascii_uppercase:
+              # maybe a hidden getter (variable accessible by a getter)
               accessor_name = 'get' + UpperCase(select)
-            # check for attribute getter in the whole chain of acquisition
-            # why? because it does not work otherways (it's most probably wrong)
+            else:
+              accessor_name = select
+            # again we check on a unwrapped object to avoid acquisition resolution
+            # which would certainly find something which we don't want
             try:
-              # this was copied out from ListsBoxHTMLRenderer
-              if hasattr(search_result, accessor_name):
-                contents_value = getattr(search_result, accessor_name)()
-            except (AttributeError, KeyError, Unauthorized):
-              context.log("Could not evaluate {} nor {} on {}".format(
-                select, accessor_name, search_result))
-
-          context.log("search_property_getter({!s}, {}) -> {!s}".format(
-            search_result, select, contents_value))
+              if hasattr(raw_search_result, accessor_name) and callable(getattr(search_result, accessor_name)):
+                # test on raw object but get the actual accessor using wrapper and acquisition
+                # do not call it here - it will be done later in generic call part
+                contents_value = getattr(search_result, accessor_name)
+            except (AttributeError, KeyError, Unauthorized) as error:
+              context.log("Could not evaluate {} nor {} on {} with error {!s}".format(
+                select, accessor_name, search_result, error), level=100)  # WARNING
+
+          if contents_value is None and search_property_hasser(search_result, select):
+            # maybe it is just a attribute
+            contents_value = search_property_getter(search_result, select)
+
+          if contents_value is None:
+            try:
+              contents_value = getattr(search_result, select, None)
+            except (Unauthorized, AttributeError, KeyError) as error:
+              context.log("Cannot resolve {} on {!s} because {!s}".format(
+                select, raw_search_result, error), level=100)
+
+          if callable(contents_value):
+            has_mandatory_param = False
+            has_brain_param = False
+            if hasattr(contents_value, "params"):
+              has_mandatory_param = any(map(lambda param: '=' not in param and '*' not in param,
+                                            contents_value.params().split(",")))
+              has_brain_param = "brain" in contents_value.params()
+            try:
+              if has_mandatory_param:
+                contents_value = contents_value(search_result)
+              elif has_brain_param:
+                contents_value = contents_value(brain=search_result)
+              else:
+                contents_value = contents_value()
+            except (AttributeError, KeyError, Unauthorized) as error:
+              context.log("Could not evaluate {} on {} with error {!s}".format(
+                contents_value, search_result, error), level=100)  # WARNING
+          # make resulting value JSON serializable
           if contents_value is not None:
             if same_type(contents_value, DateTime()):
               # Serialize DateTime
@@ -1278,6 +1487,10 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
 
           contents_item[select] = contents_value
 
+    # We should cleanup the selection if it exists in catalog params BUT
+    # we cannot because it requires escalated Permission.'modifyPortal' so
+    # the correct solution would be to ReportSection.popReport but unfortunately
+    # we don't have it anymore because we are asynchronous
 
   elif mode == 'form':
     #################################################
@@ -1386,6 +1599,7 @@ hateoas = calculateHateoas(is_portal=temp_is_portal, is_site_root=temp_is_site_r
                            restricted=restricted, list_method=list_method,
                            default_param_json=default_param_json,
                            form_relative_url=form_relative_url)
+
 if hateoas == "":
   return hateoas
 else: