Skip to content

erp5
erp5
Commit c72c3740 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

erp5_email_reader: use transaction cache instead of read-only transaction cache

parent 22887cde
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 138 deletions
bt5/erp5_email_reader/DocumentTemplateItem/portal_components/document.erp5.EmailReader.py
View file @ c72c3740
...@@ -32,7 +32,7 @@ from Products.CMFCore.utils import getToolByName ...@@ -32,7 +32,7 @@ from Products.CMFCore.utils import getToolByName
from Products.ERP5Type import Permissions, PropertySheet from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5.Document.ExternalSource import ExternalSource from Products.ERP5.Document.ExternalSource import ExternalSource
from Products.ERP5Type.XMLObject import XMLObject from Products.ERP5Type.XMLObject import XMLObject
from Products.ERP5Type.Cache import getReadOnlyTransactionCache from Products.ERP5Type.Cache import transactional_cached
# IMAP imports # IMAP imports
import imaplib import imaplib
...@@ -140,10 +140,10 @@ class IMAPSServer(MailServer): ...@@ -140,10 +140,10 @@ class IMAPSServer(MailServer):
LOG('server.select folder %s' % message_folder, INFO, "Response: %s" % response) LOG('server.select folder %s' % message_folder, INFO, "Response: %s" % response)
if response in ('OK', 'EXISTS'): # is this right status list XXX if response in ('OK', 'EXISTS'): # is this right status list XXX
self.successful_select = True self.successful_select = True
self.message_count = message_count[0] self.message_count = message_count[0]
LOG('server.select folder %s' % message_folder, INFO, "Count: %s" % self.message_count) LOG('server.select folder %s' % message_folder, INFO, "Count: %s" % self.message_count)
else: else:
raise ValueError(message_count[0]) # Use a better exception here XXX raise ValueError(message_count[0]) # Use a better exception here XXX
return self.message_count return self.message_count
def getMessageUIDList(self, message_folder=None): def getMessageUIDList(self, message_folder=None):
...@@ -215,7 +215,7 @@ class IMAPServer(IMAPSServer): ...@@ -215,7 +215,7 @@ class IMAPServer(IMAPSServer):
are closed and logged out. are closed and logged out.
""" """
def __init__(self, host, user, password, port=143): def __init__(self, host, user, password, port=143):
return IMAPSServer.__init__(self, host, user, password, port=port) IMAPSServer.__init__(self, host, user, password, port=port)
class POPSServer(MailServer): class POPSServer(MailServer):
""" """
...@@ -227,7 +227,7 @@ class POPSServer(MailServer): ...@@ -227,7 +227,7 @@ class POPSServer(MailServer):
class POPServer(POPSServer): class POPServer(POPSServer):
def __init__(self, host, user, password, port=143): def __init__(self, host, user, password, port=143):
return POPSServer.__init__(self, host, user, password, port=port) POPSServer.__init__(self, host, user, password, port=port)
class EmailReader(ExternalSource): class EmailReader(ExternalSource):
...@@ -274,7 +274,7 @@ class EmailReader(ExternalSource): ...@@ -274,7 +274,7 @@ class EmailReader(ExternalSource):
, PropertySheet.DublinCore , PropertySheet.DublinCore
, PropertySheet.Url , PropertySheet.Url
, PropertySheet.Login , PropertySheet.Login
, PropertySheet.ExternalDocument , PropertySheet.ExternalDocument
) )
# Global values # Global values
...@@ -405,8 +405,8 @@ class EmailReader(ExternalSource): ...@@ -405,8 +405,8 @@ class EmailReader(ExternalSource):
# Ingest MAX_UID_LIST_SIZE emails # Ingest MAX_UID_LIST_SIZE emails
for uid in uid_list[0:min(uid_len, self.MAX_UID_LIST_SIZE)]: for uid in uid_list[0:min(uid_len, self.MAX_UID_LIST_SIZE)]:
#if not self.hasContent(self.getMessageID(uid, message_folder)): #if not self.hasContent(self.getMessageID(uid, message_folder)):
# Only ingest new messages # Only ingest new messages
#self.activate(activity='SQLQueue', tag=message_activity_tag, #self.activate(activity='SQLQueue', tag=message_activity_tag,
# priority=2).ingestMessage(uid, # priority=2).ingestMessage(uid,
# message_folder=message_folder) # message_folder=message_folder)
self.ingestMessage(uid, message_folder=message_folder) self.ingestMessage(uid, message_folder=message_folder)
...@@ -437,28 +437,17 @@ class EmailReader(ExternalSource): ...@@ -437,28 +437,17 @@ class EmailReader(ExternalSource):
return "%s-%s" % (message_folder.replace('/','.'), uid) # Can this be configurable, based on what ? date? return "%s-%s" % (message_folder.replace('/','.'), uid) # Can this be configurable, based on what ? date?
security.declareProtected(Permissions.AccessContentsInformation, 'getMessageFolderList') security.declareProtected(Permissions.AccessContentsInformation, 'getMessageFolderList')
@transactional_cached()
def getMessageFolderList(self): def getMessageFolderList(self):
""" """
Returns the list of folders of the current server Returns the list of folders of the current server
XXX Add read only transaction cache XXX Add read only transaction cache
""" """
cache = getReadOnlyTransactionCache()
if cache is not None:
key = ('getMessageFolderList', self)
try:
return cache[key]
except KeyError:
pass
server = self._getMailServer() server = self._getMailServer()
if server is None: return () return () if server is None else server.getMessageFolderList()
result = server.getMessageFolderList()
if cache is not None:
cache[key] = result
return result
### Implementation - Private methods ### Implementation - Private methods
@transactional_cached()
def _getMailServer(self): def _getMailServer(self):
""" """
A private method to retrieve a mail server A private method to retrieve a mail server
...@@ -468,32 +457,23 @@ class EmailReader(ExternalSource): ...@@ -468,32 +457,23 @@ class EmailReader(ExternalSource):
break things. An interactor is required to clear break things. An interactor is required to clear
the variable the variable
""" """
cache = getReadOnlyTransactionCache()
if cache is not None:
key = ('_getMailServer', self)
try:
return cache[key]
except KeyError:
pass
# No server defined # No server defined
if not self.getURLServer(): return None server_url = self.getURLServer()
if not server_url:
return
# XXX - Here we need to add a switch (POP vs. IMAP vs. IMAPS etc.) # XXX - Here we need to add a switch (POP vs. IMAP vs. IMAPS etc.)
url_protocol = self.getUrlProtocol('imaps') # Default to IMAP url_protocol = self.getUrlProtocol('imaps') # Default to IMAP
if url_protocol == 'imaps': if url_protocol == 'imaps':
result = IMAPSServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort()) server_class = IMAPSServer
elif url_protocol == 'imap': elif url_protocol == 'imap':
result = IMAPServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort()) server_class = IMAPServer
elif url_protocol == 'pops': elif url_protocol == 'pops':
result = POPSServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort()) server_class = POPSServer
elif url_protocol == 'pop': elif url_protocol == 'pop':
result = POPServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort()) server_class = POPServer
else: else:
raise NotImplementedError raise NotImplementedError
if cache is not None: return server_class(server_url, self.getUserId(), self.getPassword(),
cache[key] = result port=self.getURLPort())
return result
bt5/erp5_email_reader/DocumentTemplateItem/portal_components/document.erp5.EmailReader.xml
View file @ c72c3740
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item> <item>
<key> <string>default_reference</string> </key> <key> <string>default_reference</string> </key>
<value> <string>EmailReader</string> </value> <value> <string>EmailReader</string> </value>
...@@ -27,79 +33,31 @@ ...@@ -27,79 +33,31 @@
<item> <item>
<key> <string>text_content_error_message</string> </key> <key> <string>text_content_error_message</string> </key>
<value> <value>
<tuple> <tuple/>
<string>E: 43, 0: No name \'ZRDB\' in module \'Shared.DC\' (no-name-in-module)</string>
<string>F: 43, 0: Unable to import \'Shared.DC.ZRDB.TM\' (import-error)</string>
<string>E:217, 2: Explicit return in __init__ (return-in-init)</string>
<string>E:229, 2: Explicit return in __init__ (return-in-init)</string>
<string>E:381, 6: Instance of \'list\' has no \'newContent\' member (but some types could not be inferred) (maybe-no-member)</string>
</tuple>
</value> </value>
</item> </item>
<item> <item>
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple>
<string>W: 73, 0: XXX - Rewrite everything to use UID! (fixme)</string> <string>W: 79, 2: __init__ method from base class \'MailServer\' is not called (super-init-not-called)</string>
<string>W: 89, 0: XXX What about port ? (fixme)</string> <string>W: 91, 14: Unused variable \'message_count\' (unused-variable)</string>
<string>W: 91, 0: XXX response not taken into account (fixme)</string> <string>W: 91, 4: Unused variable \'response\' (unused-variable)</string>
<string>W: 92, 0: XXX What about failures ? (fixme)</string> <string>W:157, 4: Unused variable \'response\' (unused-variable)</string>
<string>W: 94, 0: XXX - This leaves a small probability of failure (to be researched) (fixme)</string> <string>W:179, 8: Unused variable \'folder_delimiter\' (unused-variable)</string>
<string>W:137, 0: XXX response not taken into account (fixme)</string> <string>W:172, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:139, 0: XXX response not taken into account (fixme)</string> <string>W:220, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:141, 0: XXX (fixme)</string> <string>W:220, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:146, 0: XXX (fixme)</string> <string>W:220, 0: Method \'__init__\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:157, 0: XXX - reponse not taken into account (fixme)</string> <string>W:220, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:166, 0: XXX - Danger - because we are in multithreaded environment (fixme)</string> <string>W:228, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:172, 0: XXX - reponse not taken into account (fixme)</string> <string>W:228, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:176, 0: XXX - Here we consider that (fixme)</string> <string>W:228, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:193, 0: XXX - reponse not taken into account (fixme)</string> <string>W:298, 20: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:226, 0: XXX - Not implemented yet (fixme)</string> <string>W:339, 23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:246, 0: XXX) (fixme)</string> <string>W:348, 23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:248, 0: TODO: (fixme)</string> <string>W:335, 23: Unused variable \'error_message\' (unused-variable)</string>
<string>W:310, 0: XXX - Start with default one and only use filtered mailboxes if defined (fixme)</string> <string>W: 34, 0: Unused import XMLObject (unused-import)</string>
<string>W:319, 0: XXX - TODO: crawl 10 folders at once (fixme)</string>
<string>W:335, 0: XXX (fixme)</string>
<string>W:342, 0: XXX-JPS (fixme)</string>
<string>W:345, 0: XXX maybe there is a faster way to compute this (fixme)</string>
<string>W:370, 0: XXX - double check whether this is consistent with (fixme)</string>
<string>W:374, 0: XXX Is this a good way of processing exceptions (fixme)</string>
<string>W:443, 0: XXX Add read only transaction cache (fixme)</string>
<string>W:467, 0: XXX - Danger: if the server Url is changed, we (fixme)</string>
<string>W:482, 0: XXX - Here we need to add a switch (POP vs. IMAP vs. IMAPS etc.) (fixme)</string>
<string>W:143, 0: Found indentation with tabs instead of spaces (mixed-indentation)</string>
<string>W:144, 0: Found indentation with tabs instead of spaces (mixed-indentation)</string>
<string>W:146, 0: Found indentation with tabs instead of spaces (mixed-indentation)</string>
<string>R: 67, 0: Too many instance attributes (8/7) (too-many-instance-attributes)</string>
<string>W: 79, 2: __init__ method from base class \'MailServer\' is not called (super-init-not-called)</string>
<string>W: 93, 4: Access to a protected member _register of a client class (protected-access)</string>
<string>W: 91,14: Unused variable \'message_count\' (unused-variable)</string>
<string>W: 91, 4: Unused variable \'response\' (unused-variable)</string>
<string>W: 97, 0: Unused argument \'ignored\' (unused-argument)</string>
<string>W:113, 0: Unused argument \'ignored\' (unused-argument)</string>
<string>W:157, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:179, 8: Unused variable \'folder_type\' (unused-variable)</string>
<string>W:172, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:143, 1: Attribute \'message_count\' defined outside __init__ (attribute-defined-outside-init)</string>
<string>W:220, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'__init__\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>R:233, 0: Too many ancestors (45/7) (too-many-ancestors)</string>
<string>W:298,20: Used builtin function \'filter\' (bad-builtin)</string>
<string>W:298,20: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:339,23: Used builtin function \'filter\' (bad-builtin)</string>
<string>W:339,23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:348,23: Used builtin function \'filter\' (bad-builtin)</string>
<string>W:348,23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:335,23: Unused variable \'error_message\' (unused-variable)</string>
<string>R:430, 2: Method could be a function (no-self-use)</string>
<string>R:233, 0: Too many public methods (398/20) (too-many-public-methods)</string>
<string>W:427, 4: Attribute \'_latest_uid\' defined outside __init__ (attribute-defined-outside-init)</string>
<string>W: 34, 0: Unused import XMLObject (unused-import)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
...@@ -110,13 +68,28 @@ ...@@ -110,13 +68,28 @@
<item> <item>
<key> <string>workflow_history</string> </key> <key> <string>workflow_history</string> </key>
<value> <value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent> <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value> </value>
</item> </item>
</dictionary> </dictionary>
</pickle> </pickle>
</record> </record>
<record id="2" aka="AAAAAAAAAAI="> <record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle> <pickle>
<global name="PersistentMapping" module="Persistence.mapping"/> <global name="PersistentMapping" module="Persistence.mapping"/>
</pickle> </pickle>
...@@ -129,7 +102,7 @@ ...@@ -129,7 +102,7 @@
<item> <item>
<key> <string>component_validation_workflow</string> </key> <key> <string>component_validation_workflow</string> </key>
<value> <value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent> <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value> </value>
</item> </item>
</dictionary> </dictionary>
...@@ -138,7 +111,7 @@ ...@@ -138,7 +111,7 @@
</dictionary> </dictionary>
</pickle> </pickle>
</record> </record>
<record id="3" aka="AAAAAAAAAAM="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle> </pickle>
...@@ -151,33 +124,6 @@ ...@@ -151,33 +124,6 @@
<key> <string>action</string> </key> <key> <string>action</string> </key>
<value> <string>validate</string> </value> <value> <string>validate</string> </value>
</item> </item>
<item>
<key> <string>actor</string> </key>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1377850660.74</float>
<string>GMT+9</string>
</tuple>
</state>
</object>
</value>
</item>
<item> <item>
<key> <string>validation_state</string> </key> <key> <string>validation_state</string> </key>
<value> <string>validated</string> </value> <value> <string>validated</string> </value>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7