Skip to content

erp5
erp5
Commit c72c3740 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

erp5_email_reader: use transaction cache instead of read-only transaction cache

parent 22887cde
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 138 deletions
bt5/erp5_email_reader/DocumentTemplateItem/portal_components/document.erp5.EmailReader.py
View file @ c72c3740
......@@ -32,7 +32,7 @@ from Products.CMFCore.utils import getToolByName
from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5.Document.ExternalSource import ExternalSource
from Products.ERP5Type.XMLObject import XMLObject
from Products.ERP5Type.Cache import getReadOnlyTransactionCache
from Products.ERP5Type.Cache import transactional_cached
# IMAP imports
import imaplib
......@@ -140,10 +140,10 @@ class IMAPSServer(MailServer):
LOG('server.select folder %s' % message_folder, INFO, "Response: %s" % response)
if response in ('OK', 'EXISTS'): # is this right status list XXX
self.successful_select = True
self.message_count = message_count[0]
LOG('server.select folder %s' % message_folder, INFO, "Count: %s" % self.message_count)
self.message_count = message_count[0]
LOG('server.select folder %s' % message_folder, INFO, "Count: %s" % self.message_count)
else:
raise ValueError(message_count[0]) # Use a better exception here XXX
raise ValueError(message_count[0]) # Use a better exception here XXX
return self.message_count
def getMessageUIDList(self, message_folder=None):
......@@ -215,7 +215,7 @@ class IMAPServer(IMAPSServer):
are closed and logged out.
"""
def __init__(self, host, user, password, port=143):
return IMAPSServer.__init__(self, host, user, password, port=port)
IMAPSServer.__init__(self, host, user, password, port=port)
class POPSServer(MailServer):
"""
......@@ -227,7 +227,7 @@ class POPSServer(MailServer):
class POPServer(POPSServer):
def __init__(self, host, user, password, port=143):
return POPSServer.__init__(self, host, user, password, port=port)
POPSServer.__init__(self, host, user, password, port=port)
class EmailReader(ExternalSource):
......@@ -274,7 +274,7 @@ class EmailReader(ExternalSource):
, PropertySheet.DublinCore
, PropertySheet.Url
, PropertySheet.Login
, PropertySheet.ExternalDocument
, PropertySheet.ExternalDocument
)
# Global values
......@@ -405,8 +405,8 @@ class EmailReader(ExternalSource):
# Ingest MAX_UID_LIST_SIZE emails
for uid in uid_list[0:min(uid_len, self.MAX_UID_LIST_SIZE)]:
#if not self.hasContent(self.getMessageID(uid, message_folder)):
# Only ingest new messages
#self.activate(activity='SQLQueue', tag=message_activity_tag,
# Only ingest new messages
#self.activate(activity='SQLQueue', tag=message_activity_tag,
# priority=2).ingestMessage(uid,
# message_folder=message_folder)
self.ingestMessage(uid, message_folder=message_folder)
......@@ -437,28 +437,17 @@ class EmailReader(ExternalSource):
return "%s-%s" % (message_folder.replace('/','.'), uid) # Can this be configurable, based on what ? date?
security.declareProtected(Permissions.AccessContentsInformation, 'getMessageFolderList')
@transactional_cached()
def getMessageFolderList(self):
"""
Returns the list of folders of the current server
XXX Add read only transaction cache
"""
cache = getReadOnlyTransactionCache()
if cache is not None:
key = ('getMessageFolderList', self)
try:
return cache[key]
except KeyError:
pass
server = self._getMailServer()
if server is None: return ()
result = server.getMessageFolderList()
if cache is not None:
cache[key] = result
return result
return () if server is None else server.getMessageFolderList()
### Implementation - Private methods
@transactional_cached()
def _getMailServer(self):
"""
A private method to retrieve a mail server
......@@ -468,32 +457,23 @@ class EmailReader(ExternalSource):
break things. An interactor is required to clear
the variable
"""
cache = getReadOnlyTransactionCache()
if cache is not None:
key = ('_getMailServer', self)
try:
return cache[key]
except KeyError:
pass
# No server defined
if not self.getURLServer(): return None
server_url = self.getURLServer()
if not server_url:
return
# XXX - Here we need to add a switch (POP vs. IMAP vs. IMAPS etc.)
url_protocol = self.getUrlProtocol('imaps') # Default to IMAP
if url_protocol == 'imaps':
result = IMAPSServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort())
server_class = IMAPSServer
elif url_protocol == 'imap':
result = IMAPServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort())
server_class = IMAPServer
elif url_protocol == 'pops':
result = POPSServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort())
server_class = POPSServer
elif url_protocol == 'pop':
result = POPServer(self.getURLServer(), self.getUserId(), self.getPassword(), port=self.getURLPort())
server_class = POPServer
else:
raise NotImplementedError
if cache is not None:
cache[key] = result
return result
return server_class(server_url, self.getUserId(), self.getPassword(),
port=self.getURLPort())
bt5/erp5_email_reader/DocumentTemplateItem/portal_components/document.erp5.EmailReader.xml
View file @ c72c3740
......@@ -6,6 +6,12 @@
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>EmailReader</string> </value>
......@@ -27,79 +33,31 @@
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple>
<string>E: 43, 0: No name \'ZRDB\' in module \'Shared.DC\' (no-name-in-module)</string>
<string>F: 43, 0: Unable to import \'Shared.DC.ZRDB.TM\' (import-error)</string>
<string>E:217, 2: Explicit return in __init__ (return-in-init)</string>
<string>E:229, 2: Explicit return in __init__ (return-in-init)</string>
<string>E:381, 6: Instance of \'list\' has no \'newContent\' member (but some types could not be inferred) (maybe-no-member)</string>
</tuple>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 73, 0: XXX - Rewrite everything to use UID! (fixme)</string>
<string>W: 89, 0: XXX What about port ? (fixme)</string>
<string>W: 91, 0: XXX response not taken into account (fixme)</string>
<string>W: 92, 0: XXX What about failures ? (fixme)</string>
<string>W: 94, 0: XXX - This leaves a small probability of failure (to be researched) (fixme)</string>
<string>W:137, 0: XXX response not taken into account (fixme)</string>
<string>W:139, 0: XXX response not taken into account (fixme)</string>
<string>W:141, 0: XXX (fixme)</string>
<string>W:146, 0: XXX (fixme)</string>
<string>W:157, 0: XXX - reponse not taken into account (fixme)</string>
<string>W:166, 0: XXX - Danger - because we are in multithreaded environment (fixme)</string>
<string>W:172, 0: XXX - reponse not taken into account (fixme)</string>
<string>W:176, 0: XXX - Here we consider that (fixme)</string>
<string>W:193, 0: XXX - reponse not taken into account (fixme)</string>
<string>W:226, 0: XXX - Not implemented yet (fixme)</string>
<string>W:246, 0: XXX) (fixme)</string>
<string>W:248, 0: TODO: (fixme)</string>
<string>W:310, 0: XXX - Start with default one and only use filtered mailboxes if defined (fixme)</string>
<string>W:319, 0: XXX - TODO: crawl 10 folders at once (fixme)</string>
<string>W:335, 0: XXX (fixme)</string>
<string>W:342, 0: XXX-JPS (fixme)</string>
<string>W:345, 0: XXX maybe there is a faster way to compute this (fixme)</string>
<string>W:370, 0: XXX - double check whether this is consistent with (fixme)</string>
<string>W:374, 0: XXX Is this a good way of processing exceptions (fixme)</string>
<string>W:443, 0: XXX Add read only transaction cache (fixme)</string>
<string>W:467, 0: XXX - Danger: if the server Url is changed, we (fixme)</string>
<string>W:482, 0: XXX - Here we need to add a switch (POP vs. IMAP vs. IMAPS etc.) (fixme)</string>
<string>W:143, 0: Found indentation with tabs instead of spaces (mixed-indentation)</string>
<string>W:144, 0: Found indentation with tabs instead of spaces (mixed-indentation)</string>
<string>W:146, 0: Found indentation with tabs instead of spaces (mixed-indentation)</string>
<string>R: 67, 0: Too many instance attributes (8/7) (too-many-instance-attributes)</string>
<string>W: 79, 2: __init__ method from base class \'MailServer\' is not called (super-init-not-called)</string>
<string>W: 93, 4: Access to a protected member _register of a client class (protected-access)</string>
<string>W: 91,14: Unused variable \'message_count\' (unused-variable)</string>
<string>W: 91, 4: Unused variable \'response\' (unused-variable)</string>
<string>W: 97, 0: Unused argument \'ignored\' (unused-argument)</string>
<string>W:113, 0: Unused argument \'ignored\' (unused-argument)</string>
<string>W:157, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:179, 8: Unused variable \'folder_type\' (unused-variable)</string>
<string>W:172, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:143, 1: Attribute \'message_count\' defined outside __init__ (attribute-defined-outside-init)</string>
<string>W:220, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'__init__\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>R:233, 0: Too many ancestors (45/7) (too-many-ancestors)</string>
<string>W:298,20: Used builtin function \'filter\' (bad-builtin)</string>
<string>W:298,20: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:339,23: Used builtin function \'filter\' (bad-builtin)</string>
<string>W:339,23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:348,23: Used builtin function \'filter\' (bad-builtin)</string>
<string>W:348,23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:335,23: Unused variable \'error_message\' (unused-variable)</string>
<string>R:430, 2: Method could be a function (no-self-use)</string>
<string>R:233, 0: Too many public methods (398/20) (too-many-public-methods)</string>
<string>W:427, 4: Attribute \'_latest_uid\' defined outside __init__ (attribute-defined-outside-init)</string>
<string>W: 34, 0: Unused import XMLObject (unused-import)</string>
<string>W: 79, 2: __init__ method from base class \'MailServer\' is not called (super-init-not-called)</string>
<string>W: 91, 14: Unused variable \'message_count\' (unused-variable)</string>
<string>W: 91, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:157, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:179, 8: Unused variable \'folder_delimiter\' (unused-variable)</string>
<string>W:172, 4: Unused variable \'response\' (unused-variable)</string>
<string>W:220, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'__init__\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:220, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageFolderList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageUIDList\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:228, 0: Method \'getMessageData\' is abstract in class \'MailServer\' but is not overridden (abstract-method)</string>
<string>W:298, 20: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:339, 23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:348, 23: map/filter on lambda could be replaced by comprehension (deprecated-lambda)</string>
<string>W:335, 23: Unused variable \'error_message\' (unused-variable)</string>
<string>W: 34, 0: Unused import XMLObject (unused-import)</string>
</tuple>
</value>
</item>
......@@ -110,13 +68,28 @@
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
......@@ -129,7 +102,7 @@
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
......@@ -138,7 +111,7 @@
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
......@@ -151,33 +124,6 @@
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1377850660.74</float>
<string>GMT+9</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7