# -*- coding: utf-8 -*- ############################################################################## # # Copyright (c) 2017 Nexedi SARL and Contributors. All Rights Reserved. # Ayush-Tiwari <ayush.tiwari@nexedi.com> # # WARNING: This program as such is intended to be used by professional # programmers who take the whole responsability of assessing all potential # consequences resulting from its eventual inadequacies and bugs # End users who are looking for a ready-to-use solution with commercial # garantees and support are strongly adviced to contract a Free Software # Service Company # # This program is Free Software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## import gc import os import posixpath import transaction import imghdr import tarfile import time import hashlib import fnmatch import re import threading import pprint from copy import deepcopy from collections import defaultdict from cStringIO import StringIO from OFS.Image import Pdata from lxml.etree import parse from urllib import quote, unquote from OFS import SimpleItem, XMLExportImport from datetime import datetime from itertools import chain from operator import attrgetter from Products.ERP5Type.XMLObject import XMLObject from Products.CMFCore.utils import getToolByName from Products.PythonScripts.PythonScript import PythonScript from Products.ERP5Type.dynamic.lazy_class import ERP5BaseBroken from Products.ERP5Type.Globals import Persistent from Products.ERP5Type import Permissions, PropertySheet, interfaces from AccessControl import ClassSecurityInfo, Unauthorized, getSecurityManager from Acquisition import Implicit, aq_base, aq_inner, aq_parent from Products.ERP5Type.Globals import InitializeClass from Products.ERP5Type.TransactionalVariable import getTransactionalVariable from zLOG import LOG, INFO, WARNING from Products.ERP5Type.patches.ppml import importXML from Products.ERP5Type.Accessor.Constant import PropertyGetter as ConstantGetter customImporters = { XMLExportImport.magic: importXML, } CACHE_DATABASE_PATH = None try: if int(os.getenv('ERP5_BT5_CACHE', 0)): from App.config import getConfiguration import gdbm instancehome = getConfiguration().instancehome CACHE_DATABASE_PATH = os.path.join(instancehome, 'bt5cache.db') except TypeError: pass cache_database = threading.local() _MARKER = [] SEPARATELY_EXPORTED_PROPERTY_DICT = { # For objects whose class name is 'class_name', the 'property_name' # attribute is removed from the XML export, and the value is exported in a # separate file, with extension specified by 'extension'. # 'extension' must be None for auto-detection. # # class_name: (extension, unicode_data, property_name), "Document Component": ("py", 0, "text_content"), "DTMLDocument": (None, 0, "raw"), "DTMLMethod": (None, 0, "raw"), "Extension Component": ("py", 0, "text_content"), "File": (None, 0, "data"), "Image": (None, 0, "data"), "OOoTemplate": ("oot", 1, "_text"), "PDF": ("pdf", 0, "data"), "PDFForm": ("pdf", 0, "data"), "Python Script": ("py", 0, "_body"), "PythonScript": ("py", 0, "_body"), "Spreadsheet": (None, 0, "data"), "SQL": ("sql", 0, "src"), "SQL Method": ("sql", 0, "src"), "Test Component": ("py", 0, "text_content"), "Test Page": (None, 0, "text_content"), "Web Page": (None, 0, "text_content"), "Web Script": (None, 0, "text_content"), "Web Style": (None, 0, "text_content"), "ZopePageTemplate": ("zpt", 1, "_text"), } def _delObjectWithoutHook(obj, id): """OFS.ObjectManager._delObject without calling manage_beforeDelete.""" ob = obj._getOb(id) if obj._objects: obj._objects = tuple([i for i in obj._objects if i['id'] != id]) obj._delOb(id) try: ob._v__object_deleted__ = 1 except: pass def _recursiveRemoveUid(obj): """Recusivly set uid to None, to prevent (un)indexing. This is used to prevent unindexing real objects when we delete subobjects on a copy of this object. """ if getattr(aq_base(obj), 'uid', _MARKER) is not _MARKER: obj.uid = None for subobj in obj.objectValues(): _recursiveRemoveUid(subobj) class BusinessManager(XMLObject): """Business Manager is responsible for saving objects and properties in an ERP5Site. Everything will be saved just via path""" meta_type = 'ERP5 Business Manager' portal_type = 'Business Manager' add_permission = Permissions.AddPortalContent # Declarative security security = ClassSecurityInfo() security.declareObjectProtected(Permissions.AccessContentsInformation) _properties = ( {'id': 'template_path_list', 'type': 'lines', 'default': 'python: ()', 'acquisition_base_category': (), 'acquisition_portal_type': (), 'acquisition_depends': None, 'acquisition_accessor_id': 'getTemplatePathList', 'override': 1, 'mode': 'w'}, {'id': 'template_format_version', 'type': 'int', 'default': 'python: 3', 'acquisition_base_category': (), 'acquisition_portal_type': (), 'acquisition_depends': None, 'acquisition_accessor_id': 'getTemplateFormatVersion', 'override': 1, 'mode': 'w'}, ) template_path_list = () template_format_version = 3 status = 'uninstalled' # Declarative security security = ClassSecurityInfo() security.declareObjectProtected(Permissions.AccessContentsInformation) # Declarative properties property_sheets = ( PropertySheet.Base, PropertySheet.XMLObject, PropertySheet.SimpleItem, PropertySheet.CategoryCore, PropertySheet.Version, ) def getStatus(self): """ installed :BI(s) are installed in OFS. uninstalled :Values for BI(s) at the current version removed from OFS. reduced :No two BI of same path exist at different layers. flatenned :BI(s) should be at the zeroth layer. built :BI(s) do have values from the OS DB. """ return self.status def setStatus(self, status=None): if not status: raise ValueError, 'No status provided' else: self.status = status def applytoERP5(self, DB): """Apply the flattened/reduced Business Manager to the DB""" portal = self.getPortalObject() pass def edit(self, **kw): """ Explicilty edit the class instance XXX: No need of this class ? as we already have _edit from ERP5Type.Folder """ if 'template_path_list' in kw: path_item_list = kw.pop('template_path_list') self._setTemplatePathList(path_item_list) def _setTemplatePathList(self, path_item_list): self.template_path_list = path_item_list def getTemplatePathList(self): return self.template_path_list security.declareProtected(Permissions.ManagePortal, '_getTemplatePathList') def _getTemplatePathList(self): result = self.getTemplatePathList() if not isinstance(result, tuple): result = tuple(result) return result def getTemplateFormatVersion(self): return self.template_format_version def _setTemplateFormatVersion(self, value): self.template_format_version = int(value) def propertyMap(self): prop_map = super(BusinessManager, self).propertyMap() final_prop_map = prop_map+self._properties return final_prop_map security.declareProtected(Permissions.ManagePortal, 'export') def export(self, path=None, local=0, bma=None, **kw): """ Export the object XXX: Are we planning to use something like archive for saving the exported objects inside a Business Manager """ if not self.getStatus() == 'built': raise ValueError, 'Manager not built properly' return self._export(path, local, bma, **kw) def _export(self, path=None, local=0, bma=None, **kw): if bma is None: if local: # we export into a folder tree bma = BusinessManagerFolder(path, creation=1) else: # We export bm into a tarball file if path is None: path = self.getTitle() bma = BusinessManagerTarball(path, creation=1) # export bt for prop in self.propertyMap(): prop_type = prop['type'] id = prop['id'] if id in ('id', 'uid', 'rid', 'sid', 'id_group', 'last_id', 'revision', 'install_object_list_list', 'id_generator', 'bm_for_diff'): continue value = self.getProperty(id) if not value: continue if prop_type in ('text', 'string', 'int', 'boolean'): bma.addObject(str(value), name=id, path='bm', ext='') elif prop_type in ('lines', 'tokens'): bma.addObject('\n'.join(value), name=id, path='bm', ext='') # Export each part for item in self._path_item_list: item.export(context=self, bma=bma, **kw) return bma.finishCreation() security.declareProtected(Permissions.ManagePortal, 'importFile') def importFile(self, path): """ Import all xml files in Business Manager """ bma = (BusinessManagerFolder if os.path.isdir(path) else BusinessManagerTarball)(path, importing=1) bm_item = bm() bma.importFiles(bm_item, parent=self) prop_dict = {} for prop in self.propertyMap(): pid = prop['id'] if pid != 'id': prop_type = prop['type'] value = bm_item.get(pid) if prop_type in ('text', 'string'): prop_dict[pid] = value or '' elif prop_type in ('int', 'boolean'): prop_dict[pid] = value or 0 elif prop_type in ('lines', 'tokens'): # XXX: Do add pid[:-5] after we switch to proper getters and setters prop_dict[pid] = (value or '').splitlines() # XXX: This is not working, needs to be fixed so as it copies all the # properties from BMA to the newly created Business Manager self._edit(**prop_dict) self.storeTemplateData() #workflow_tool = self.getPortalObject().portal_workflow #workflow_tool.business_package_building_workflow.notifyWorkflowMethod( # self, 'edit', kw={'comment': 'Downloaded'}) for item_object in self._path_item_list: item_object.importFile(bma, parent=self) # Set the status to uninstalled self.setStatus('uninstalled') def __add__(self, other): """ Adds the Business Item objects for the given Business Manager objects """ self._path_item_list.extend(other._path_item_list) template_path_list = list(self.template_path_list)+list(other.template_path_list) self.template_path_list = template_path_list return self __radd__ = __add__ def __sub__(self, other): """ Override subtract to find difference b/w the values in different cases. """ # Create the sha list for all path item list available in current object sha_list = [item._sha for item in self._path_item_list] # Reverse the sign of Business Item objects for the old Business Manager # Trying comparing/subtracting ZODB with old installed object for path_item in other._path_item_list: if path_item._sha in sha_list: self._path_item_list = [item for item in self._path_item_list if item._sha != path_item._sha] else: path_item._sign = -1 self._path_item_list.append(path_item) return self __rsub__ = __sub__ security.declareProtected(Permissions.ManagePortal, 'storeTemplateData') def storeTemplateData(self): """ Store data for objects in the ERP5 """ LOG('Business Manager', INFO, 'Storing Manager Data') self._path_item_list = [] path_item_list = self.getTemplatePathList() if path_item_list: path_item_list = [l.split(' | ') for l in path_item_list] for path_item in path_item_list: try: self._path_item_list.append(BusinessItem(path_item[0], path_item[1], path_item[2])) except IndexError: pass def getPathList(self): path_list = [] for item in self._path_item_list: path_list.append(item._path) return path_list def getPathShaDict(self): path_sha_dict = {} # TODO: Handle error for BM with multiple items at same path for item in self._path_item_list: path_sha_dict[item._path] = item._sha return path_item_dict def getPathItemDict(self): path_item_dict = {} # TODO: Handle error for BM with multiple items at same path for item in self._path_item_list: path_item_dict[item._path] = item return path_item_dict def getBusinessItemByPath(self, path): path_item_dict = self.getPathItemDict() try: return path_item_dict[path] except KeyError: return def build(self, no_action=False, **kw): """Creates new values for business item from the values from OFS Database""" LOG('Business Manager', INFO, 'Building Business Manager') if not no_action: self.storeTemplateData() for path_item in self._path_item_list: path_item.build(self, **kw) self.status = 'built' return self def install(self): """ Installs the Business Manager in steps: 1. Reduction of the BT 2. Flattenning the BT 3. Copying the object at the path mentioned in BT """ if self.status == 'uninstalled': self.reduceBusinessManager() #elif self.status == 'reduced': # self.flattenBusinessManager() self._install() def _install(self): """ Run installation """ if self.status != 'reduced': self.install() else: # Invoke install on every BusinessItem object for path_item in self._path_item_list: path_item.install() def upgrade(self): """Upgrade the Business Manager""" pass def flattenBusinessManager(self): """ Flattening a reduced Business Manager with two path p1 and p2 where p1 <> p2: flatten([(p1, s1, l1, v1), (p2, s2, l2, v2)]) = [(p1, s1, 0, v1), (p2, s2, 0, v2)] A reduced Business Manager BT is said to be flattened if and only if: flatten(BT) = BT """ portal = self.getPortalObject() if self.getStatus() != 'reduced': raise ValueError, 'Please reduce the BT before flatenning' # XXX: Maybe call reduce function on BT by itself here rather than just # raising the error, because there is no other choice else: path_list = self.getTemplatePathList() for path_item in self._path_item_list: path = path_item._path layer = path_item._layer # Flatten the BusinessItem to the lowest layer ?? Why required, no change if layer != 0: path_item._layer = 0 self.status = 'flattened' def reduceBusinessManager(self): """ Reduction is a function that takes a Business Manager as input and returns a smaller Business Manager by taking out values with lower priority layers. After taking out BusinessItem(s) with lower priority layer, we also go through arithmetic in case there are multiple number of BI at the higher layer Two path on different layer are reduced as a single path with the highest layer: If l1 > l2, reduce([(p, s, l1, (a, b, c)), (p, s, l2, (d, e))]) = [(p, s, l1, merge(a, b, c))] A Business Manager BT is said to be reduced if and only if: reduce(BT) = BT """ path_list = list(set([path_item.getBusinessPath() for path_item in self._path_item_list])) reduced_path_item_list = [] # We separate the path list in the ones which are repeated and the ones # which are unique for the installation seen_path_list = set() unique_path_list = [x for x in path_list if x not in seen_path_list and not seen_path_list.add(x)] # Create an extra dict for values on path which are repeated in the path list seen_path_dict = {path: [] for path in seen_path_list} for path_item in self._path_item_list: if path_item._path in seen_path_list: # In case the path is repeated keep the path_item in a separate dict # for further arithmetic seen_path_dict[path_item._path].append(path_item) else: # If the path is unique, add them in the list of reduced Business Item reduced_path_item_list.append(path_item) # Reduce the values and get the merged result out of it for path, path_item_list in seen_path_dict.items(): # Create separate list of list items with highest priority higest_priority_layer = max(path_item_list, key=attrgetter('_layer'))._layer prioritized_path_item = [path_item for path_item in path_item_list if path_item._layer == higest_priority_layer] # Separate the positive and negative sign path_item if len(prioritized_path_item) > 1: path_item_list_add = [item for item in prioritized_path_item if item._sign > 0] path_item_list_subtract = [item for item in prioritized_path_item if item._sign < 0] combined_added_path_item = reduce(lambda x, y: x+y, path_item_list_add) combined_subtracted_path_item = reduce(lambda x, y: x+y, path_item_list_subtract) added_value = combined_added_path_item._value subtracted_value = combined_subtracted_path_item._value if added_value != subtracted_value: # Append the arithmetically combined path_item objects in the final # reduced list after removing the intersection added_value, subtracted_value = \ self._simplifyValueIntersection(added_value, subtracted_value) combined_added_path_item._value = added_value combined_subtracted_path_item._value = subtracted_value # Append the path_item to the final reduced path_item_list after # doing required arithmetic on it. Make sure to first append # subtracted item because while installation, we need to first # uninstall the old object and then install new object at same path reduced_path_item_list.append(combined_subtracted_path_item) reduced_path_item_list.append(combined_added_path_item) else: reduced_path_item_list.append(prioritized_path_item[0]) self._path_item_list = reduced_path_item_list self.setStatus('reduced') def _simplifyValueIntersection(self, added_value, subtracted_value): """ Returns values for the Business Item having same path and layer after removing the intersection of the values Parameters: added_value - Value for the Business Item having sign = +1 subtracted_value - Value for Busienss Item having sign = -1 """ built_in_number_type = (int, long, float, complex) built_in_container_type = (tuple, list, dict, set) built_in_type_list = built_in_number_type + built_in_container_type # For ERP5 objects, we should return the added and subtracted values as it is if type(added_value).__name__ not in built_in_type_list and \ type(subtracted_value).__name__ not in built_in_type_list: return added_value, subtracted_value # For all the values of container type, we remove the intersection added_value = [x for x in added_value if x not in subtracted_value] subtracted_value = [x for x in subtracted_value if x not in added_value] return added_value, subtracted_value class BusinessItem(Implicit, Persistent): """Saves the path and values for objects, properties, etc, the attributes for a path configuration being: - path (similar to an xpath expression) Examples of path : portal_type/Person portal_type/Person#title portal_type/Person#property_sheet?ancestor=DublinCore portal_type/Person#property_sheet?position=2 - sign (+1/-1) - layer (0, 1, 2, 3, etc.) - value (a set of pickable value in python) - hash of the value""" isProperty = False def __init__(self, path, sign=1, layer=0, value=None, *args, **kw): """ Initialize/update the attributes """ self.__dict__.update(kw) self._path = path self._sign = int(sign) self._layer = int(layer) self._value = value if value: # Generate hash of from the value self._sha = self._generateHash() else: self._sha = '' def _generateHash(self): """ Generate hash based on value for the object. Initially, for simplicity, we go on with SHA256 values only """ LOG('Business Manager', INFO, 'Genrating hash') if not self._value: # Raise in case there is no value for the BusinessItem object raise ValueError, "Value not defined for the %s BusinessItem" % self._path else: # Expects to raise error on case the value for the object # is not picklable try: sha256 = hashlib.sha256(self._value).hexdigest() except TypeError: obj_dict = self._value.__dict__.copy() del obj_dict['uid'] sha256 = hash(pprint.pformat(obj_dict)) self._sha = sha256 def build(self, context, **kw): """ Extract value for the given path from the OFS Three different situations to extract value: 1. For paths which point directly to an object in OFS 2. For paths which point to multiple objects inside a folder 3. For paths which point to property of an object in OFS : In this case, we can have URL delimiters like ?, #, = in the path """ LOG('Business Manager', INFO, 'Building Business Item') p = context.getPortalObject() path = self._path if '#' in str(path): self.isProperty = True relative_url, property_id = path.split('#') obj = p.unrestrictedTraverse(relative_url) property_value = obj.getProperty(property_id) self._value = property_value # Generate hash for the property value self._generateHash() else: try: for relative_url in self._resolvePath(p, [], path.split('/')): obj = p.unrestrictedTraverse(relative_url) obj = obj._getCopy(context) obj = obj.__of__(context) _recursiveRemoveUid(obj) self._value = obj # Generate hash for the erp5 object value self._generateHash() except AttributeError: # In case the object doesn't exist, just pass without raising error pass def applyValueToPath(self): """ Apply the value to the path given. 1. If the path doesn't exist, and its a new object, create the object. 2. If the path doesn't exist, and its a new property, apply the property on the object. 3. If the path doesn't exist, and its a new property, raise error. """ pass def _resolvePath(self, folder, relative_url_list, id_list): """ We go through 3 types of paths: 1. General path we find in erp5 for objects Ex: portal_type/Person In this case, we import/export the object on the path 2. Path where we consider saving sub-objects also, in that case we create new BusinessItem for those objects Ex: portal_catalog/erp5_mysql_innodb/** This should create BI for the catalog methods sub-objects present in the erp5_catalog. This method calls itself recursively. The folder is the current object which contains sub-objects. The list of ids are path components. If the list is empty, the current folder is valid. """ if len(id_list) == 0: return ['/'.join(relative_url_list)] id = id_list[0] if re.search('[\*\?\[\]]', id) is None: # If the id has no meta character, do not have to check all objects. obj = folder._getOb(id, None) if obj is None: raise AttributeError, "Could not resolve '%s' during BusinessItem processing." % id return self._resolvePath(obj, relative_url_list + [id], id_list[1:]) path_list = [] for object_id in fnmatch.filter(folder.objectIds(), id): if object_id != "": path_list.extend(self._resolvePath( folder._getOb(object_id), relative_url_list + [object_id], id_list[1:])) return path_list def setPropertyToPath(self, path, property_name, value): """ Set property for the object at given path """ portal = self.getPortalObject() obj = portal.unrestrictedTraverse(path) obj.setProperty(property_name, value) def generateXML(self): """ Generate XML for different objects/type/properties differently. 1. Objects: Use XMLImportExport from ERP5Type 2. For properties, first get the property type, then create XML object for the different property differenty(Use ObjectPropertyItem from BT5) 3. For attributes, we can export part of the object, rather than exporting whole of the object """ pass def install(self, context): """ Set the value to the defined path. """ # In case the path denotes property, we create separate object for # ObjectTemplateItem and handle the installation there. portal = context.getPortalObject() if self.isProperty: realtive_url, property_id = self._path.split('#') object_property_item = ObjectPropertyTemplateItem(id_list) object_property_item.install() else: path_list = self._path.split('/') container_path = path_list[:-1] object_id = path_list[-1] try: container = self.unrestrictedResolveValue(portal, container_path) except KeyError: # parent object can be set to nothing, in this case just go on container_url = '/'.join(container_path) old_obj = container._getOb(object_id, None) # delete the old object before installing a new object if old_obj: container._delObject(object_id) # Create a new object only if sign is +1 # If sign is +1, set the new object on the container if self._sign == 1: # install object obj = self._value container._setObject(object_id, obj) obj = container._getOb(object_id) obj.isIndexable = ConstantGetter('isIndexable', value=False) aq_base(obj).uid = portal.portal_catalog.newUid() del obj.isIndexable if getattr(aq_base(obj), 'reindexObject', None) is not None: obj.reindexObject() def unrestrictedResolveValue(self, context=None, path='', default=_MARKER, restricted=0): """ Get the value without checking the security. This method does not acquire the parent. """ if isinstance(path, basestring): stack = path.split('/') else: stack = list(path) stack.reverse() if stack: if context is None: portal = aq_inner(self.getPortalObject()) container = portal else: container = context if restricted: validate = getSecurityManager().validate while stack: key = stack.pop() try: value = container[key] except KeyError: LOG('BusinessManager', WARNING, 'Could not access object %s' % (path,)) if default is _MARKER: raise return default if restricted: try: if not validate(container, container, key, value): raise Unauthorized('unauthorized access to element %s' % key) except Unauthorized: LOG('BusinessManager', WARNING, 'access to %s is forbidden' % (path,)) if default is _MARKER: raise return default container = value return value else: return context def __add__(self, other): """ Add the values from the path when the path is same for 2 objects """ if self._path != other._path: raise ValueError, "BusinessItem are incommensurable, have different path" elif self._sign != other._sign: raise ValueError, "BusinessItem are incommensurable, have different sign" else: self._value = self._mergeValue(value_list=[self._value, other._value]) return self __radd__ = __add__ def _mergeValue(self, value_list): """ Merge value in value list merge(a, b, c) : A monotonic commutative function that depends on the type of a, b and c: if a, b and c are sets, merge = union if a, b and c are lists, merge = ordered concatenation if a, b and c are objects, merge = the object created the last else merge = MAX """ builtin_number_type = (int, long, float, complex) # Now, consider the type of both values if all(isinstance(x, builtin_number_type) for x in value_list): merged_value = max(value_list) elif all(isinstance(x, set) for x in value_list): merged_value = set(chain.from_iterable(value_list)) elif all(isinstance(x, list) for x in value_list): merged_value = list(chain.from_iterable(value_list)) elif all(isinstance(x, tuple) for x in value_list): merged_value = tuple(chain.from_iterable(value_list)) else: # In all other case, check if the values are objects and then take the # objects created last. # XXX: Should we go with creation date or modification_date ?? # TODO: # 1. Add check that the values are ERP5 objects # 2. In case 2 maximum values are created at same time, prefer one with # higher priority layer merged_value = max([max(value, key=attrgetter('creation_date')) for value in value_list], key=attrgetter('creation_date')) return merged_value def _guessFilename(self, document, key, data): """ Try to guess the extension based on the id of the document """ yield key document_base = aq_base(document) # Try to guess the extension based on the reference of the document if hasattr(document_base, 'getReference'): yield document.getReference() elif isinstance(document_base, ERP5BaseBroken): yield getattr(document_base, "reference", None) # Try to guess the extension based on the title of the document yield getattr(document_base, "title", None) # Try to guess from content if data: for test in imghdr.tests: extension = test(data, None) if extension: yield 'x.' + extension def guessExtensionOfDocument(self, document, key, data=None): """ Guesses and returns the extension of an ERP5 document. The process followed is: 1. Try to guess extension by the id of the document 2. Try to guess extension by the title of the document 3. Try to guess extension by the reference of the document 4. Try to guess from content (only image data is tested) If there's a content type, we only return an extension that matches. In case everything fails then: - '.bin' is returned for binary files - '.txt' is returned for text """ document_base = aq_base(document) # XXX Zope items like DTMLMethod would not implement getContentType method mime = None if hasattr(document_base, 'getContentType'): content_type = document.getContentType() elif isinstance(document_base, ERP5BaseBroken): content_type = getattr(document_base, "content_type", None) else: content_type = None # For stable export, people must have a MimeTypes Registry, so do not # fallback on mimetypes. We prefer the mimetypes_registry because there # are more extensions and we can have preferred extensions. # See also https://bugs.python.org/issue1043134 mimetypes_registry = self.getPortalObject()['mimetypes_registry'] if content_type: try: mime = mimetypes_registry.lookup(content_type)[0] except (IndexError, MimeTypeException): pass for key in self._guessFilename(document, key, data): if key: ext = os.path.splitext(key)[1][1:].lower() if ext and (mimetypes_registry.lookupExtension(ext) is mime if mime else mimetypes_registry.lookupExtension(ext)): return ext if mime: # return first registered extension (if any) if mime.extensions: return mime.extensions[0] for ext in mime.globs: if ext[0] == "*" and ext.count(".") == 1: return ext[2:].encode("utf-8") # in case we could not read binary flag from mimetypes_registry then return # '.bin' for all the Portal Types where exported_property_type is data # (File, Image, Spreadsheet). Otherwise, return .bin if binary was returned # as 1. binary = getattr(mime, 'binary', None) if binary or binary is None is not data: return 'bin' # in all other cases return .txt return 'txt' def removeProperties(self, obj, export, properties=[], keep_workflow_history=False, keep_workflow_history_last_history_only=False): """ Remove unneeded properties for export """ obj._p_activate() klass = obj.__class__ classname = klass.__name__ attr_set = {'_dav_writelocks', '_filepath', '_owner', '_related_index', 'last_id', 'uid', '__ac_local_roles__', '__ac_local_roles_group_id_dict__'} if properties: for prop in properties: if prop.endswith('_list'): prop = prop[:-5] attr_set.add(prop) if export: if keep_workflow_history_last_history_only: self._removeAllButLastWorkflowHistory(obj) elif not keep_workflow_history: attr_set.add('workflow_history') # PythonScript covers both Zope Python scripts # and ERP5 Python Scripts if isinstance(obj, PythonScript): attr_set.update(('func_code', 'func_defaults', '_code', '_lazy_compilation', 'Python_magic')) for attr in 'errors', 'warnings', '_proxy_roles': if not obj.__dict__.get(attr, 1): delattr(obj, attr) elif classname in ('File', 'Image'): attr_set.update(('_EtagSupport__etag', 'size')) elif classname == 'SQL' and klass.__module__ == 'Products.ZSQLMethods.SQL': attr_set.update(('_arg', 'template')) elif interfaces.IIdGenerator.providedBy(obj): attr_set.update(('last_max_id_dict', 'last_id_dict')) elif classname == 'Types Tool' and klass.__module__ == 'erp5.portal_type': attr_set.add('type_provider_list') for attr in obj.__dict__.keys(): if attr in attr_set or attr.startswith('_cache_cookie_'): delattr(obj, attr) if classname == 'PDFForm': if not obj.getProperty('business_template_include_content', 1): obj.deletePdfContent() return obj def export(self, context, bma, **kw): """ Export the business item object : fill the BusinessManagerArchive with objects exported as XML, hierarchicaly organised. """ if not self._value: return path = self.__class__.__name__ + '/' # We now will add the XML object and its sha hash while exporting the object # to Business Manager itself # Back compatibility with filesystem Documents key = self._path obj = self._value if isinstance(obj, str): if not key.startswith(path): key = path + key bma.addObject(obj, name=key, ext='.py') else: try: extension, unicode_data, record_id = \ SEPARATELY_EXPORTED_PROPERTY_DICT[obj.__class__.__name__] except KeyError: pass else: while 1: # not a loop obj = obj._getCopy(context) data = getattr(aq_base(obj), record_id, None) if unicode_data: if type(data) is not unicode: break try: data = data.encode(aq_base(obj).output_encoding) except (AttributeError, UnicodeEncodeError): break elif type(data) is not bytes: if not isinstance(data, Pdata): break data = bytes(data) try: # Delete this attribute from the object. # in case the related Portal Type does not exist, the object may be broken. # So we cannot delattr, but we can delete the key of its its broken state if isinstance(obj, ERP5BaseBroken): del obj.__Broken_state__[record_id] obj._p_changed = 1 else: delattr(obj, record_id) except (AttributeError, KeyError): # property was acquired on a class, # do nothing, only .xml metadata will be exported break # export a separate file with the data if not extension: extension = self.guessExtensionOfDocument(obj, key, data if record_id == 'data' else None) bma.addObject(StringIO(data), key, path=path, ext='._xml' if extension == 'xml' else '.' + extension) break # since we get the obj from context we should # again remove useless properties obj = self.removeProperties(obj, 1, keep_workflow_history=True) transaction.savepoint(optimistic=True) f = StringIO() XMLExportImport.exportXML(obj._p_jar, obj._p_oid, f) bma.addObject(f, key, path=path) def importFile(self, bma, parent, **kw): bma.importFiles(self, parent) def _importFile(self, file_name, file_obj, parent): obj_key, file_ext = os.path.splitext(file_name) # id() for installing several bt5 in the same transaction transactional_variable_obj_key = "%s-%s" % (id(self), obj_key) if file_ext != '.xml': # For ZODB Components: if .xml have been processed before, set the # source code property, otherwise store it in a transactional variable # so that it can be set once the .xml has been processed data = file_obj.read() try: obj = self._objects[obj_key] except KeyError: getTransactionalVariable()[transactional_variable_obj_key] = data else: self._restoreSeparatelyExportedProperty(obj, data) else: connection = self.getConnection(parent) __traceback_info__ = 'Importing %s' % file_name if hasattr(cache_database, 'db') and isinstance(file_obj, file): obj = connection.importFile(self._compileXML(file_obj)) else: # FIXME: Why not use the importXML function directly? Are there any BT5s # with actual .zexp files on the wild? obj = connection.importFile(file_obj, customImporters=customImporters) self._value = obj data = getTransactionalVariable().get(transactional_variable_obj_key) if data is not None: self._restoreSeparatelyExportedProperty(obj, data) def _restoreSeparatelyExportedProperty(self, obj, data): unicode_data, property_name = SEPARATELY_EXPORTED_PROPERTY_DICT[ obj.__class__.__name__][1:] if unicode_data: data = data.decode(obj.output_encoding) try: setattr(obj, property_name, data) except BrokenModified: obj.__Broken_state__[property_name] = data obj._p_changed = 1 else: # Revert any work done by __setstate__. # XXX: This is enough for all objects we currently split in 2 files, # but __setstate__ could behave badly with the missing attribute # and newly added types may require more than this. self.removeProperties(obj, 1, keep_workflow_history=True) def getConnection(self, obj): while True: connection = obj._p_jar if connection is not None: return connection obj = obj.aq_parent def _compileXML(self, file): # This method converts XML to ZEXP. Because the conversion # is quite heavy, a persistent cache database is used to # store ZEXP, so the second run wouldn't have to re-generate # identical data again. # # For now, a pair of the path to an XML file and its modification time # are used as a unique key. In theory, a checksum of the content could # be used instead, and it could be more reliable, as modification time # might not be updated in some insane filesystems correctly. However, # in practice, checksums consume a lot of CPU time, so when the cache # does not hit, the increased overhead is significant. In addition, it # does rarely happen that two XML files in Business Manager contain # the same data, so it may not be expected to have more cache hits # with this approach. # # The disadvantage is that this wouldn't work with the archive format, # because each entry in an archive does not have a mtime in itself. # However, the plan is to have an archive to retain ZEXP directly # instead of XML, so the idea of caching would be completely useless # with the archive format. name = file.name mtime = os.path.getmtime(file.name) key = '%s:%s' % (name, mtime) try: return StringIO(cache_database.db[key]) except: pass from Shared.DC.xml import ppml from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata import xml.parsers.expat outfile=StringIO() try: data=file.read() F=ppml.xmlPickler() F.end_handlers['record'] = save_record F.end_handlers['ZopeData'] = save_zopedata F.start_handlers['ZopeData'] = start_zopedata F.binary=1 F.file=outfile p=xml.parsers.expat.ParserCreate('utf-8') p.returns_unicode = False p.CharacterDataHandler=F.handle_data p.StartElementHandler=F.unknown_starttag p.EndElementHandler=F.unknown_endtag p.Parse(data) try: cache_database.db[key] = outfile.getvalue() except: pass outfile.seek(0) return outfile except: outfile.close() raise def getBusinessPath(self): return self._path def getBusinessPathSign(self): return self._sign def getBusinessPathLayer(self): return self._layer def getBusinessPathValue(self): return self._value def setBusinessPathValue(self, value): self._value = value def getBusinessPathSha(self): return self._sha def getParentBusinessManager(self): return self.aq_parent class BusinessManagerArchive(object): """ This is the base class for all Business Manager archives """ def __init__(self, path, **kw): self.path = path def addObject(self, obj, name, path=None, ext='.xml'): if path: name = posixpath.join(path, name) # XXX required due to overuse of os.path name = name.replace('\\', '/').replace(':', '/') name = quote(name + ext) path = name.replace('/', os.sep) try: write = self._writeFile except AttributeError: if not isinstance(obj, str): obj.seek(0) obj = obj.read() self._writeString(obj, path) else: if isinstance(obj, str): obj = StringIO(obj) else: obj.seek(0) write(obj, path) def finishCreation(self): pass class BusinessManagerFolder(BusinessManagerArchive): """ Class archiving business manager into a folder tree """ def _writeString(self, obj, path): object_path = os.path.join(self.path, path) path = os.path.dirname(object_path) os.path.exists(path) or os.makedirs(path) f = open(object_path, 'wb') try: f.write(obj) finally: f.close() def importFiles(self, item, parent): """ Import file from a local folder """ join = os.path.join item_name = item.__class__.__name__ root = join(os.path.normpath(self.path), item_name, '') root_path_len = len(root) if CACHE_DATABASE_PATH: try: cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'cf') except gdbm.error: cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'nf') try: for root, dirs, files in os.walk(root): for file_name in files: file_name = join(root, file_name) with open(file_name, 'rb') as f: file_name = posixpath.normpath(file_name[root_path_len:]) if '%' in file_name: file_name = unquote(file_name) elif item_name == 'bm' and file_name == 'revision': continue # self.revision.hash(item_name + '/' + file_name, f.read()) f.seek(0) item._importFile(file_name, f, parent) finally: if hasattr(cache_database, 'db'): cache_database.db.close() del cache_database.db class BusinessManagerTarball(BusinessManagerArchive): """ Class archiving business manager into a tarball file """ def __init__(self, path, creation=0, importing=0, **kw): super(BusinessManagerTarball, self).__init__(path, **kw) if creation: self.fobj = StringIO() self.tar = tarfile.open('', 'w:gz', self.fobj) self.time = time.time() elif importing: self.tar = tarfile.open(path, 'r:gz') self.item_dict = item_dict = defaultdict(list) for info in self.tar.getmembers(): if info.isreg(): path = info.name.split('/') if path[0] == '.': del path[0] item_dict[path[1]].append(('/'.join(path[2:]), info)) def _writeFile(self, obj, path): if self.path: path = posixpath.join(self.path, path) info = tarfile.TarInfo(path) info.mtime = self.time obj.seek(0, 2) info.size = obj.tell() obj.seek(0) self.tar.addfile(info, obj) def finishCreation(self): self.tar.close() return self.fobj def importFiles(self, item, parent): """ Import all file from the archive to the site """ extractfile = self.tar.extractfile item_name = item.__class__.__name__ for file_name, info in self.item_dict.get(item_name, ()): if '%' in file_name: file_name = unquote(file_name) elif item_name == 'bm' and file_name == 'revision': continue f = extractfile(info) self.revision.hash(item_name + '/' + file_name, f.read()) f.seek(0) item._importFile(file_name, f, parent) class bm(dict): """ Fake 'bm' item to read bm/* files through BusinessManagerArchive """ def _importFile(self, file_name, file, parent): self[file_name] = file.read() #InitializeClass(BusinessManager)