ContributionTool.py 26.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
##############################################################################
#
# Copyright (c) 2007 Nexedi SARL and Contributors. All Rights Reserved.
#                    Jean-Paul Smets <jp@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

29
import cStringIO
30 31
import re
import string
32 33
import socket
import md5
Jean-Paul Smets's avatar
Jean-Paul Smets committed
34
import urllib2, urllib
35

Bartek Górny's avatar
Bartek Górny committed
36
from AccessControl import ClassSecurityInfo, getSecurityManager
37 38 39 40
from Globals import InitializeClass, DTMLFile
from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5Type import Permissions
from Products.ERP5 import _dtmldir
Jean-Paul Smets's avatar
Jean-Paul Smets committed
41 42
from Products.ERP5.Document.Url import no_crawl_protocol_list, no_host_protocol_list

43 44 45 46
from zLOG import LOG
from DateTime import DateTime
from Acquisition import aq_base

47 48 49 50 51 52 53 54 55 56
# Install openers
import ContributionOpener
opener = urllib2.build_opener(ContributionOpener.DirectoryFileHandler)
urllib2.install_opener(opener)

# A temporary hack until urllib2 supports timeout setting - XXX
import socket
socket.setdefaulttimeout(60) # 1 minute timeout

# Global parameters
57
TEMP_NEW_OBJECT_KEY = '_v_new_object'
58
MAX_REPEAT = 10
59 60

_marker = []  # Create a new marker object.
61 62 63 64

class ContributionTool(BaseTool):
  """
    ContributionTool provides an abstraction layer to unify the contribution
65
    of documents into an ERP5 Site.
66

67 68
    ContributionTool needs to be configured in portal_types (allowed contents) so
    that it can store Text, Spreadsheet, PDF, etc. 
69

70 71 72
    The main method of ContributionTool is newContent. This method can
    be provided various parameters from which the portal type and document
    metadata can be derived. 
73 74

    Configuration Scripts:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
75

76 77 78
      - ContributionTool_getPropertyDictFromFileName: receives file name and a 
        dict derived from filename by regular expression, and does any necesary
        operations (e.g. mapping document type id onto a real portal_type).
Jean-Paul Smets's avatar
Jean-Paul Smets committed
79 80 81 82 83 84

    Problems which are not solved

      - handling of relative links in HTML contents (or others...)
        some text rewriting is necessary.

85 86 87 88 89 90
  """
  title = 'Contribution Tool'
  id = 'portal_contributions'
  meta_type = 'ERP5 Contribution Tool'
  portal_type = 'Contribution Tool'

Jean-Paul Smets's avatar
Jean-Paul Smets committed
91 92 93
  # Regular expressions
  simple_normaliser = re.compile('#.*')

94 95 96 97 98 99 100
  # Declarative Security
  security = ClassSecurityInfo()

  security.declareProtected(Permissions.ManagePortal, 'manage_overview' )
  manage_overview = DTMLFile( 'explainContributionTool', _dtmldir )

  security.declarePrivate('findTypeName')
101
  def findTypeName(self, file_name, document):
102 103
    """
      Finds the appropriate portal type based on the file name
104
      or if necessary the content of the document.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
105 106 107 108

      NOTE: XXX This implementation can be greatly accelerated by
      caching a dict resulting which combines getContentTypeRegistryTypeDict
      and valid_portal_type_list
109
    """
Jean-Paul Smets's avatar
Jean-Paul Smets committed
110 111 112 113 114 115 116
    def getContentTypeRegistryTypeDict():
      result = {}
      for id, pred in self.content_type_registry.listPredicates():
        (p, type) = pred
        result[type] = None
      return result

117
    portal_type = None
118
    # We should only consider those portal_types which share the
Jean-Paul Smets's avatar
Jean-Paul Smets committed
119 120 121 122 123 124 125 126
    # same constructor with the current object and which are not
    # part of the definitions of content_type_registry. For
    # example if content type registry has a definition for
    # RSS feed, then there is no reason to consider this type
    # whenever receiving some text/html content although both
    # types share the same constructor. However, if Memo has
    # same constructor as Text and Memo is not in content_type_registry
    # then it should be considered.
127
    extra_valid_portal_type_list = []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
128 129 130 131 132
    content_registry_type_dict = getContentTypeRegistryTypeDict()
    portal_type_tool = self.portal_types
    for pt in portal_type_tool.objectValues():
      if hasattr(pt, 'factory') and pt.factory == portal_type_tool[document.getPortalType()].factory:
        if not content_registry_type_dict.has_key(pt.id):
133 134 135 136 137 138 139 140 141 142
          extra_valid_portal_type_list.append(pt.id)

    if not extra_valid_portal_type_list:
      # There is really no ambiguity here
      # The portal_type set by PUT_factory is appropriate
      # This is the best case we can get
      # LOG('findTypeName no ambiguity', 0, document.portal_type)
      return document.portal_type

    valid_portal_type_list = [document.portal_type] + extra_valid_portal_type_list
143 144

    # Check if the filename tells which portal_type this is
145
    portal_type_list = self.getPropertyDictFromFileName(file_name).get('portal_type', [])
146 147 148 149
    if isinstance(portal_type_list, str): portal_type_list = [portal_type_list]
    portal_type_list = filter(lambda x: x in valid_portal_type_list, portal_type_list)
    if not portal_type_list:
      portal_type_list = valid_portal_type_list
150 151
    if len(portal_type_list) == 1:
      # if we have only one, then this is it
152
      # LOG('findTypeName single portal_type_list', 0, portal_type_list[0])
153
      return portal_type_list[0]
Ivan Tyagov's avatar
Ivan Tyagov committed
154
      
155 156
    # If it is still None, we need to read the document
    # to check which of the candidates is suitable
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
    # Let us give a chance to getPropertyDictFromContent to
    # tell us what is the portal type of this document
    content_portal_type_list = document.getPropertyDictFromContent().get('portal_type', None)
    if content_portal_type_list:
      if isinstance(portal_type, str):
        content_portal_type_list = [content_portal_type_list]
      # Filter valid candidates
      content_portal_type_list = filter(lambda x: x in portal_type_list, content_portal_type_list)
      if content_portal_type_list:
        # if we have more than one, then return the first one
        # LOG('findTypeName from content', 0, content_portal_type_list[0])
        return content_portal_type_list[0]

    # If portal_type_list is not empty, return the first one
    # LOG('findTypeName from first portal_type_list', 0, portal_type_list[0])
    return portal_type_list[0]
173 174

  security.declareProtected(Permissions.AddPortalContent, 'newContent')
175
  def newContent(self, id=None, portal_type=None, url=None, container=None,
Jean-Paul Smets's avatar
Jean-Paul Smets committed
176
                       container_path=None,
177 178 179 180 181 182 183 184 185
                       discover_metadata=1, temp_object=0,
                       user_login=None, **kw):
    """
      The newContent method is overriden to implement smart content
      creation by detecting the portal type based on whatever information
      was provided and finding out the most appropriate module to store
      the content.

      user_login is the name under which the content will be created
186 187
      XXX - this is a security hole which needs to be fixed by
      making sure only Manager can use this parameter
188

189 190 191 192
      container -- if specified, it is possible to define
      where to contribute the content. Else, ContributionTool
      tries to guess.

Jean-Paul Smets's avatar
Jean-Paul Smets committed
193 194 195
      container_path -- if specified, defines the container path
      and has precedence over container

196 197
      url -- if specified, content is download from the URL.

198 199 200 201 202 203 204 205 206 207
      NOTE:
        We always generate ID. So, we must prevent using the one
        which we were provided.
    """
    # Temp objects use the standard newContent from Folder
    if temp_object:
      # For temp_object creation, use the standard method
      return BaseTool.newContent(self, id=id, portal_type=portal_type, temp_object=1, **kw)

    # Try to find the file_name
208
    file_name = None
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
    mime_type = None
    if url is None:
      # check if file was provided
      file = kw.get('file', None)
      if file is not None:
        file_name = file.filename
      else:
        # some channels supply data and file-name separately
        # this is the case for example for email ingestion
        # in this case, we build a file wrapper for it
        data = kw.get('data', None)
        if data is not None:
          file_name = kw.get('file_name', None)
          if file_name is not None:
            file = cStringIO.StringIO()
            file.write(data)
            file.seek(0)
226 227 228
            kw['file'] = file
            del kw['data']
            del kw['file_name']
Jean-Paul Smets's avatar
Jean-Paul Smets committed
229
    else:
230
      # build a new file from the url
231 232
      url_file = urllib2.urlopen(url)
      data = url_file.read() # time out must be set or ... too long XXX
233 234 235
      file = cStringIO.StringIO()
      file.write(data)
      file.seek(0)
236
      # Create a file name based on the URL and quote it
Jean-Paul Smets's avatar
Jean-Paul Smets committed
237
      file_name = url.split('/')[-1] or url.split('/')[-2]
238 239 240 241 242 243
      file_name = urllib.quote(file_name, safe='')
      file_name = file_name.replace('%', '')
      # For URLs, we want an id by default equal to the encoded URL 
      if id is None: id = self._encodeURL(url)
      if hasattr(url_file, 'headers'):
        headers = url_file.headers
244 245 246
        if hasattr(headers, 'type'):
          mime_type = headers.type
      kw['file'] = file
247 248

    # If the portal_type was provided, we can go faster
249 250 251 252
    if portal_type and container is None:
      # We know the portal_type, let us find the default module
      # and use it as container
      container = self.getDefaultModule(portal_type)
253

254 255
    if portal_type and container is not None:
      # We could simplify things here and return a document immediately
256
      # NOTE: we use the module ID generator rather than the provided ID
257 258 259 260 261
      #document = module.newContent(portal_type=portal_type, **kw)
      #if discover_metadata:
      #  document.activate().discoverMetadata(file_name=file_name, user_login=user_login)
      #return document
      pass # XXX - This needs to be implemented once the rest is stable
262 263 264 265 266 267

    # From here, there is no hope unless a file was provided    
    if file is None:
      raise ValueError, "could not determine portal type"

    # So we will simulate WebDAV to get an empty object
Jean-Paul Smets's avatar
Jean-Paul Smets committed
268
    # with PUT_factory - we provide the mime_type as
269
    # parameter
270
    # LOG('new content', 0, "%s -- %s" % (file_name, mime_type))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
271
    ob = self.PUT_factory(file_name, mime_type, None)
272

Jean-Paul Smets's avatar
Jean-Paul Smets committed
273 274 275 276
    # Raise an error if we could not guess the portal type
    if ob is None:
      raise ValueError, "Could not determine the document type"

277
    # Then put the file inside ourselves for a short while
Jean-Paul Smets's avatar
Jean-Paul Smets committed
278
    BaseTool._setObject(self, file_name, ob)
279
    document = BaseTool._getOb(self, file_name)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
280 281 282

    # Then edit the document contents (so that upload can happen)
    document._edit(**kw)
283
    if url: document.fromURL(url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
284

285
    # Remove the object from ourselves
Jean-Paul Smets's avatar
Jean-Paul Smets committed
286
    BaseTool._delObject(self, file_name)
287

Jean-Paul Smets's avatar
Jean-Paul Smets committed
288
    # Move the document to where it belongs
Jean-Paul Smets's avatar
Jean-Paul Smets committed
289 290
    if container_path is not None:
      container = self.getPortalObject().restrictedTraverse(container_path)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
291 292 293
    document = self._setObject(file_name, ob, user_login=user_login, id=id,
                               container=container, discover_metadata=discover_metadata,
                               )
294
    document = self._getOb(file_name) # Call _getOb to purge cache
Jean-Paul Smets's avatar
Jean-Paul Smets committed
295

Jean-Paul Smets's avatar
Jean-Paul Smets committed
296
    # Notify workflows
297
    #document.notifyWorkflowCreated()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
298

Jean-Paul Smets's avatar
Jean-Paul Smets committed
299
    # Reindex it and return the document
300
    document.reindexObject()
301 302
    return document

303
  security.declareProtected( Permissions.AddPortalContent, 'newXML' )
304 305 306 307 308 309 310 311
  def newXML(self, xml):
    """
      Create a new content based on XML data. This is intended for contributing
      to ERP5 from another application.
    """
    pass

  security.declareProtected(Permissions.ModifyPortalContent,'getPropertyDictFromFileName')
312
  def getPropertyDictFromFileName(self, file_name):
313 314 315 316
    """
      Gets properties from filename. File name is parsed with a regular expression
      set in preferences. The regexp should contain named groups.
    """
317 318 319
    if file_name is None:
      return {}
    property_dict = {}
320
    rx_src = self.portal_preferences.getPreferredDocumentFileNameRegularExpression()
321
    if rx_src:
322
      rx_parse = re.compile(rx_src)
323 324 325 326 327
      if rx_parse is not None:
        try:
          property_dict = rx_parse.match(file_name).groupdict()
        except AttributeError: # no match
          pass
328 329
    method = self._getTypeBasedMethod('getPropertyDictFromFileName', 
        fallback_script_id = 'ContributionTool_getPropertyDictFromFileName')
330
    property_dict = method(file_name, property_dict)
Ivan Tyagov's avatar
Ivan Tyagov committed
331
    if property_dict.get('portal_type', None) is not None:
332
      # we have to return portal_type as a tuple
333
      # because we should allow for having multiple candidate types
334 335
      property_dict['portal_type'] = (property_dict['portal_type'],)
    else:
336
      # we have to find candidates by file extenstion
Ivan Tyagov's avatar
Ivan Tyagov committed
337 338 339
      if file_name.rfind('.')!= -1:
        ext = file_name.split('.')[-1]
        property_dict['portal_type'] = self.ContributionTool_getCandidateTypeListByExtension(ext)
340 341
    return property_dict

342
  # WebDAV virtual folder support
Jean-Paul Smets's avatar
Jean-Paul Smets committed
343 344
  def _setObject(self, name, ob, user_login=None, container=None,
                       id=None, discover_metadata=1):
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
    """
      The strategy is to let NullResource.PUT do everything as
      usual and at the last minute put the object in a different
      location with a different portal type. This means that
      NullResource.PUT creates an empty document with PUT_factory
      then upload document data by invoking PUT on the empty
      document and finally sets the object. By overriding _setObject
      we get a chance to fix the portal_type of the document
      (as long as the one we find is compatible) and move the
      document to the appropriate module.

      content_type_registry must be set up so that an appropriate
      portal_type with appropriate meta_type is found for every
      kind of document. However, a different portal_type might
      be used in the end.

      The ContributionTool instance must be configured in such
      way that _verifyObjectPaste will return TRUE.

      Refer to: NullResource.PUT
    """
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
    # _setObject is called by constructInstance at a time
    # when the object has no portal_type defined yet. It
    # will be removed later on. We can safely store the
    # document inside us at this stage. Else we
    # must find out where to store it.
    if not ob.__dict__.has_key('portal_type'):
      BaseTool._setObject(self, name, ob)
      document = self[name]
    else:
      # We give the system a last chance to analyse the
      # portal_type based on the document content
      # (ex. a Memo is a kind of Text which can be identified
      # by the fact it includes some specific content)
      portal_type = self.findTypeName(name, ob.__of__(self))
      if portal_type is None: portal_type = ob.portal_type
      ob._setPortalTypeName(portal_type) # This is redundant with finishConstruction
                                       # but necessary to move objects to appropriate
                                       # location based on their content. Since the
                                       # object is already constructed here, we
                                       # can safely change its portal_type
      # Now we know the portal_type, let us find the module
      # to which we should move the document to
388 389 390 391
      if container is None:
        module = self.getDefaultModule(ob.portal_type)
      else:
        module = container
Jean-Paul Smets's avatar
Jean-Paul Smets committed
392 393 394 395
      if id is None:
        new_id = module.generateNewId()
      else:
        new_id = id
396
      ob.id = new_id
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
      existing_document = module.get(new_id, None)
      if existing_document is None:
        # There is no preexisting document - we can therefore
        # set the new object
        module._setObject(new_id, ob)
        # We can now discover metadata
        document = module[new_id]
        if discover_metadata:
          # Metadata disovery is done as an activity by default
          # If we need to discoverMetadata synchronously, it must
          # be for user interface and should thus be handled by
          # ZODB scripts
          document.activate().discoverMetadata(file_name=name, user_login=user_login)
      else:
        if document.isExternalDocument():
Jean-Paul Smets's avatar
Jean-Paul Smets committed
412
          document = existing_document 
413 414
          # If this is an external document, update its content
          document.activate().updateContentFromURL()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
415 416 417 418
          # XXX - Make sure this does not increase ZODB
          # XXX - what to do also with parameters (put again edit_kw) ?
          # Providing some information to the use about the fact
          # this was an existing document would also be great
419
        else:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
420 421 422
          # We may have to implement additional revision support
          # to support in place contribution (ie. for a given ID)
          # but is this really useful ? 
423
          raise NotImplementedError
424

425 426
      # Keep the document close to us - this is only useful for
      # file upload from webdav
427 428 429
      if not hasattr(self, '_v_document_cache'):
        self._v_document_cache = {}
      self._v_document_cache[name] = document.getRelativeUrl()
430 431 432

    # Return document to newContent method
    return document
433

434 435 436 437 438
  def _getOb(self, id, default=_marker):
    """
    Check for volatile temp object info first
    and try to find it
    """
439 440
    # Use the document cache if possible and return result immediately
    # this is only useful for webdav
Jean-Paul Smets's avatar
Jean-Paul Smets committed
441 442 443
    if hasattr(self, '_v_document_cache'):
      document_url = self._v_document_cache.get(id, None)
      if document_url is not None:
444
        del self._v_document_cache[id]
Jean-Paul Smets's avatar
Jean-Paul Smets committed
445 446
        return self.getPortalObject().unrestrictedTraverse(document_url)

Jean-Paul Smets's avatar
Jean-Paul Smets committed
447
    # Try first to return an object listed by listDAVObjects
448 449 450
    uid = str(id).split('-')[-1]
    object = self.getPortalObject().portal_catalog.unrestrictedGetResultValue(uid=uid)
    if object is not None:
451
      return object.getObject() # Make sure this does not break security. XXX
452 453

    # Fallback to default method
Jean-Paul Smets's avatar
Jean-Paul Smets committed
454 455 456 457
    if default is _marker:
      return BaseTool._getOb(self, id)
    else:
      return BaseTool._getOb(self, id, default=default)
458

Bartek Górny's avatar
Bartek Górny committed
459
  def listDAVObjects(self):
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
    """
      Get all contents contributed by the current user. This is
      delegated to a script in order to help customisation.
    """
    method = getattr(self, 'ContributionTool_getMyContentList', None)
    if method is not None:
      object_list = method()
    else:
      sm = getSecurityManager()
      user = sm.getUser()
      object_list = self.portal_catalog(portal_type=self.getPortalMyDocumentTypeList(),
                                        owner=str(user))

    def wrapper(o_list):
      for o in o_list:
        o = o.getObject()
        reference = o.getReference()
        if reference:
          id = '%s-%s' % (reference, o.getUid())
        else:
          id = '%s' % o.getUid()
        yield o.getObject().asContext(id=id)

    return wrapper(object_list)
Bartek Górny's avatar
Bartek Górny committed
484

Jean-Paul Smets's avatar
Jean-Paul Smets committed
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
  # Crawling methods
  def _normaliseURL(self, url, base_url=None):
    """
      Returns a normalised version of the url so
      that we do not download twice the same content.
      URL normalisation is an important part in crawlers.
      The current implementation is obviously simplistic.
      Refer to http://en.wikipedia.org/wiki/Web_crawler
      and study Harvestman for more ideas.
    """
    url = self.simple_normaliser.sub('', url)
    url_split = url.split(':')
    url_protocol = url_split[0]
    if url_protocol in no_host_protocol_list:
      return url
    if base_url and len(url_split) == 1:
      # Make relative URL absolute
      url = '%s/%s' % (base_url, url)
    return url

  def _encodeURL(self, url):
    """
    Returns the URL as an ID. ID should be chosen in such
    way that it is optimal with HBTreeFolder (ie. so that
    distribution of access time on a cluster is possible)

    NOTE: alternate approach is based on a url table
    and catalog lookup. It is faster ? Not sure. Since
    we must anyway insert objects in btrees and this
    is simimar in cost to accessing them.
    """
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
    # Produce an MD5 from the URL
    hex_md5 = md5.md5(url).hexdigest()
    # Take the first part in the URL which is not empty
    # LOG("_encodeURL", 0, url)
    url_segment = url.split(':')[1]
    url_segment_list = url_segment.split('/')
    url_domain = None
    for url_part in url_segment_list:
      if url_part:
        url_domain = url_part
        break
    # Return encoded url
    if url_domain:
      url_domain = urllib.quote(url_domain, safe='')
      url_domain = url_domain.replace('%', '')
      return "%s-%s" % (url_domain, hex_md5)
    return hex_md5
Jean-Paul Smets's avatar
Jean-Paul Smets committed
533 534 535 536 537 538
    url = urllib.quote(url, safe='')
    url = url.replace('_', '__')
    url = url.replace('%', '_')
    return url

  security.declareProtected(Permissions.AddPortalContent, 'crawlContent')
539
  def crawlContent(self, content, container=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
540 541 542 543 544 545 546 547 548 549 550 551 552
    """
      Analyses content and download linked pages

      XXX: missing is the conversion of content local href to something
      valid.
    """
    depth = content.getCrawlingDepth()
    if depth <= 0:
      # Do nothing if crawling depth is reached
      return
    base_url = content.getContentBaseURL()
    url_list = map(lambda url: self._normaliseURL(url, base_url), set(content.getContentURLList()))
    for url in set(url_list):
553
      # LOG('trying to crawl', 0, url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
554 555 556
      # Some url protocols should not be crawled
      if url.split(':')[0] in no_crawl_protocol_list:
        continue
557 558 559 560
      if container is None:
        #if content.getParentValue()
        # in place of not ?
        container = content.getParentValue()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
561 562 563 564 565 566 567 568
      # Calculate the id under which content will be stored
      id = self._encodeURL(url)
      # Try to access the document if it already exists
      document = container.get(id, None)
      if document is None:
        # XXX - This call is not working due to missing group_method_id
        # therefore, multiple call happen in parallel and eventually fail
        # (the same URL is created multiple times)
569
        # LOG('activate newContentFromURL', 0, url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
570 571 572 573 574 575 576
        self.activate(activity="SQLQueue").newContentFromURL(container_path=container.getRelativeUrl(),
                                                      id=id, url=url, crawling_depth=depth - 1)
      else:
        # Update depth to the max. of the two values
        new_depth = max(depth - 1, document.getCrawlingDepth())
        document._setCrawlingDepth(new_depth)
        # And activate updateContentFromURL on existing document
577 578 579
        next_date = document.getNextAlarmDate() # This should prevent doing the update too often
        # LOG('activate updateContentFromURL', 0, url)
        document.activate(at_date=next_date).updateContentFromURL(crawling_depth=depth - 1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
580 581

  security.declareProtected(Permissions.AddPortalContent, 'updateContentFromURL')
582
  def updateContentFromURL(self, content, repeat=MAX_REPEAT, crawling_depth=0):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
583 584 585
    """
      Updates an existing content.
    """
586 587 588
    # Step 0: update crawling_depth if required
    if crawling_depth > content.getCrawlingDepth():
      content._setCrawlingDepth(crawling_depth)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
589
    # Step 1: download new content
590 591 592 593 594 595 596 597
    try:
      url = content.asURL()
      data = urllib2.urlopen(url).read()
      file = cStringIO.StringIO()
      file.write(data)
      file.seek(0)
    except socket.error, msg: # repeat multiple times in case of socket error
      content.updateContentFromURL(repeat=repeat - 1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
598 599 600
    # Step 2: compare and update if necessary (md5)
    # do here some md5 stuff to compare contents...
    if 1:
601
      # content._edit(file=file) # Commented for testing
Jean-Paul Smets's avatar
Jean-Paul Smets committed
602
      # Step 3: convert to base format
603
      # content.convertToBaseFormat() # Commented for testing
Jean-Paul Smets's avatar
Jean-Paul Smets committed
604
      # Step 4: activate populate (unless interaction workflow does it)
605
      # content.activate().populateContent() # Commented for testing
Jean-Paul Smets's avatar
Jean-Paul Smets committed
606 607 608 609 610 611 612 613 614 615 616
      # Step 5: activate crawlContent
      content.activate().crawlContent()
    else:
      # XXX
      # We must handle the case for which content type has changed in between
      pass
    # Step 6: activate updateContentFromURL at next period
    next_date = content.getNextAlarmDate()
    content.activate(at_date=next_date).updateContentFromURL()

  security.declareProtected(Permissions.AddPortalContent, 'newContentFromURL')
617
  def newContentFromURL(self, container_path=None, id=None, repeat=MAX_REPEAT, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
618 619 620 621 622 623 624 625 626 627
    """
      A wrapper method for newContent which provides extra safety
      in case or errors (ie. download, access, conflict, etc.).
      The method is able to handle a certain number of exceptions
      and can postpone itself through an activity based on
      the type of exception (ex. for a 404, postpone 1 day), using
      the at_date parameter and some standard values.

      NOTE: implementation needs to be done.
    """
628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
    # First of all, make sure do not try to create an existing document
    if container_path is not None and id is not None:
      container = self.restrictedTraverse(container_path)
      document = container.get(id, None)
      if document is not None:
        # Document aleardy exists: no need to keep on crawling
        return
    try:
      document = self.newContent(container_path=container_path, id=id, **kw)
      if document.getCrawlingDepth() > 0: document.activate().crawlContent()
      document.activate(at_date=document.getNextAlarmDate()).updateContentFromURL()
    except urllib2.HTTPError, error:
      # Catch any HTTP error
      self.activate(at_date=DateTime() + 1).newContentFromURL(
                        container_path=container_path, id=id,
                        repeat=repeat - 1, **kw)
    except urllib2.URLError, error:
      if error.reason.args[0] == -3:
        # Temporary failure in name resolution - try again in 1 day
647 648
        self.activate(at_date=DateTime() + 1,
                      activity="SQLQueue").newContentFromURL(
649 650 651 652 653 654 655 656
                        container_path=container_path, id=id,
                        repeat=repeat - 1, **kw)
      else:
        # Unknown errror - to be extended
        raise
    except:
      # Pass exception to Zope (ex. conflict errors)
      raise
Jean-Paul Smets's avatar
Jean-Paul Smets committed
657

Ivan Tyagov's avatar
Ivan Tyagov committed
658
InitializeClass(ContributionTool)