OOoUtils.py 18.9 KB
Newer Older
Kevin Deldycke's avatar
Kevin Deldycke committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
##############################################################################
#
# Copyright (c) 2003-2005 Nexedi SARL and Contributors. All Rights Reserved.
#                         Kevin DELDYCKE    <kevin@nexedi.com>
#                         Guillaume MICHON  <guillaume@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

30 31
import sys

32 33
from Acquisition import Implicit

Kevin Deldycke's avatar
Kevin Deldycke committed
34 35 36 37 38
from Products.PythonScripts.Utility import allow_class
from ZPublisher.HTTPRequest import FileUpload
from xml.dom.ext.reader import PyExpat
from xml.dom import Node
from AccessControl import ClassSecurityInfo
39 40
from Globals import InitializeClass, get_request
from zipfile import ZipFile, ZIP_DEFLATED
41 42 43 44
try:
  from cStringIO import StringIO
except ImportError:
  from StringIO import StringIO
Kevin Deldycke's avatar
Kevin Deldycke committed
45
import imghdr
46
import random
Bartek Górny's avatar
Bartek Górny committed
47 48
from Products.ERP5Type import Permissions
from zLOG import LOG
49
from zLOG import PROBLEM
Kevin Deldycke's avatar
Kevin Deldycke committed
50

51
from OFS.Image import Pdata
Kevin Deldycke's avatar
Kevin Deldycke committed
52 53 54

class CorruptedOOoFile(Exception): pass

55 56 57 58 59 60 61 62 63 64 65 66
OOo_mimeType_dict = {
  'sxw' : 'application/vnd.sun.xml.writer',
  'stw' : 'application/vnd.sun.xml.writer.template',
  'sxg' : 'application/vnd.sun.xml.writer.global',
  'sxc' : 'application/vnd.sun.xml.calc',
  'stc' : 'application/vnd.sun.xml.calc.template',
  'sxi' : 'application/vnd.sun.xml.impress',
  'sti' : 'application/vnd.sun.xml.impress.template',
  'sxd' : 'application/vnd.sun.xml.draw',
  'std' : 'application/vnd.sun.xml.draw.template',
  'sxm' : 'application/vnd.sun.xml.math',
}
Kevin Deldycke's avatar
Kevin Deldycke committed
67

68
class OOoBuilder(Implicit):
69 70 71
  """
  Tool that allows to reinject new files in a ZODB OOo document.
  """
72
  __allow_access_to_unprotected_subobjects__ = 1
73 74

  def __init__(self, document):
75
    if hasattr(document, 'data') :
76
      self._document = StringIO()
77 78 79 80 81 82 83 84 85 86 87

      if isinstance(document.data, Pdata):
        # Handle image included in the style
        dat = document.data
        while dat is not None:
          self._document.write(dat.data)
          dat = dat.next
      else:
        # Default behaviour
        self._document.write(document.data)
          
88 89 90
    elif hasattr(document, 'read') :
      self._document = document
    else :
91 92
      self._document = StringIO()
      self._document.write(document)
93
    self._image_count = 0    
94
    self._manifest_additions_list = []
95 96 97 98 99 100 101 102 103 104

  def replace(self, filename, stream):
    """
    Replaces the content of filename by stream in the archive.
    Creates a new file if filename was not already there.
    """
    try:
      zf = ZipFile(self._document, mode='a', compression=ZIP_DEFLATED)
    except RuntimeError:
      zf = ZipFile(self._document, mode='a')
105
    try:
106 107 108
      # remove the file first if it exists
      fi = zf.getinfo(filename)
      zf.filelist.remove( fi )
109
    except KeyError:
110 111
      # This is a new file
      pass
112 113
    zf.writestr(filename, stream)
    zf.close()
Bartek Górny's avatar
Bartek Górny committed
114

115 116 117 118 119 120 121 122 123
  def extract(self, filename):
    """
    Extracts a file from the archive
    """
    try:
      zf = ZipFile(self._document, mode='r', compression=ZIP_DEFLATED)
    except RuntimeError:
      zf = ZipFile(self._document, mode='r')
    return zf.read(filename)
Bartek Górny's avatar
Bartek Górny committed
124

125 126 127 128 129 130 131 132 133
  def getNameList(self):
    try:
      zf = ZipFile(self._document, mode='r', compression=ZIP_DEFLATED)
    except RuntimeError:
      zf = ZipFile(self._document, mode='r')
    li = zf.namelist()
    zf.close()
    return li

134 135 136
  def getMimeType(self):
    return self.extract('mimetype')

137
  def prepareContentXml(self, xsl_content=None):
138 139 140 141 142 143 144
    """
      extracts content.xml text and prepare it :
        - add tal namespace
        - indent the xml
    """
    content_xml = self.extract('content.xml')
    output = StringIO()
145 146 147 148 149 150 151 152 153
    try:
      import libxml2
      import libxslt
      if xsl_content is None:
        raise ImportError
      stylesheet_doc = libxml2.parseDoc(xsl_content)
      stylesheet = libxslt.parseStylesheetDoc(stylesheet_doc)
      content_doc = libxml2.parseDoc(content_xml)
      result_doc = stylesheet.applyStylesheet(content_doc, None)
154 155 156 157 158 159 160 161 162
      #Declare zope namespaces
      root = result_doc.getRootElement()
      tal = root.newNs('http://xml.zope.org/namespaces/tal', 'tal')
      i18n = root.newNs('http://xml.zope.org/namespaces/i18n', 'i18n')
      metal = root.newNs('http://xml.zope.org/namespaces/metal', 'metal')
      root.setNs(tal)
      root.setNs(i18n)
      root.setNs(metal)
      root.setNsProp(tal, 'attributes', 'dummy python:request.RESPONSE.setHeader(\'Content-Type\', \'text/html;; charset=utf-8\')')
163 164 165
      buff = libxml2.createOutputBuffer(output, 'utf-8')
      result_doc.saveFormatFileTo(buff, 'utf-8', 1)
      stylesheet_doc.freeDoc(); content_doc.freeDoc(); result_doc.freeDoc()
166
      return output.getvalue()
167 168 169 170 171 172 173 174 175 176 177 178 179
    except ImportError:
      reader = PyExpat.Reader()
      document = reader.fromString(content_xml)
      document_element = document.documentElement
      from xml.dom.ext import PrettyPrint
      PrettyPrint(document_element, output)
      return output.getvalue().replace(
        "office:version='1.0'",
        """ xmlns:tal='http://xml.zope.org/namespaces/tal'
            xmlns:i18n='http://xml.zope.org/namespaces/i18n'
            xmlns:metal='http://xml.zope.org/namespaces/metal'
            tal:attributes='dummy python:request.RESPONSE.setHeader("Content-Type", "text/html;; charset=utf-8")'
          office:version='1.0'""")
180

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
  def addFileEntry(self, full_path, media_type, content=None):
      """ Add a file entry to the manifest and possibly is content """
      self.addManifest(full_path, media_type)
      if content:
          self.replace(full_path, content)

  def addManifest(self, full_path, media_type):
    """ Add a path to the manifest """
    li = '<manifest:file-entry manifest:media-type="%s" manifest:full-path="%s"/>'%(media_type, full_path)
    self._manifest_additions_list.append(li)

  def updateManifest(self):
    """ Add a path to the manifest """
    MANIFEST_FILENAME = 'META-INF/manifest.xml'
    meta_infos = self.extract(MANIFEST_FILENAME)
    # prevent some duplicates
    for meta_line in meta_infos.split('\n'):
        for new_meta_line in self._manifest_additions_list:
            if meta_line.strip() == new_meta_line:
                self._manifest_additions_list.remove(new_meta_line)

    # add the new lines
    self._manifest_additions_list.append('</manifest:manifest>')
    meta_infos = meta_infos.replace( self._manifest_additions_list[-1], '\n'.join(self._manifest_additions_list) )
    self.replace(MANIFEST_FILENAME, meta_infos)
    self._manifest_additions_list = []

208 209 210 211 212 213 214 215
  def addImage(self, image, format='png'):
    """
    Add an image to the current document and return its id
    """
    count = self._image_count
    self._image_count += 1
    name = "Picture/%s.%s" % (count, format)
    self.replace(name, image)
216
    is_legacy = ('oasis.opendocument' not in self.getMimeType())
Yoshinori Okuji's avatar
Yoshinori Okuji committed
217
    return "%s%s" % (is_legacy and '#' or '', name,)
218

219 220 221 222 223 224
  def render(self, name='', extension='sxw'):
    """
    returns the OOo document
    """
    request = get_request()
    if name:
225 226
      request.response.setHeader('Content-Disposition', 'inline; filename=%s.%s' % (name, extension))

227 228
    self._document.seek(0)
    return self._document.read()
Bartek Górny's avatar
Bartek Górny committed
229

230
allow_class(OOoBuilder)
Kevin Deldycke's avatar
Kevin Deldycke committed
231

232
class OOoParser(Implicit):
Kevin Deldycke's avatar
Kevin Deldycke committed
233 234 235
  """
    General purpose tools to parse and handle OpenOffice v1.x documents.
  """
236
  __allow_access_to_unprotected_subobjects__ = 1 
Kevin Deldycke's avatar
Kevin Deldycke committed
237 238 239 240 241 242 243 244
  def __init__(self):
    # Create the PyExpat reader
    self.reader = PyExpat.Reader()
    self.oo_content_dom = None
    self.oo_styles_dom  = None
    self.oo_files = {}
    self.pictures = {}
    self.ns = {}
Kevin Deldycke's avatar
Kevin Deldycke committed
245
    self.filename = None
Kevin Deldycke's avatar
Kevin Deldycke committed
246

247 248 249
  def openFromString(self, text_content):
    return self.openFile(StringIO(text_content))

250
  def openFile(self, file_descriptor):
Kevin Deldycke's avatar
Kevin Deldycke committed
251 252 253 254 255
    """
      Load all files in the zipped OpenOffice document
    """
    # Try to unzip the Open Office doc
    try:
256
      oo_unzipped = ZipFile(file_descriptor, mode="r")
Kevin Deldycke's avatar
Kevin Deldycke committed
257
    except:
258
      LOG('ERP5OOo', PROBLEM, 'Error in openFile', error=sys.exc_info())
259
      raise CorruptedOOoFile()
Kevin Deldycke's avatar
Kevin Deldycke committed
260 261
    # Test the integrity of the file
    if oo_unzipped.testzip() != None:
262
      raise CorruptedOOoFile()
Kevin Deldycke's avatar
Kevin Deldycke committed
263

Kevin Deldycke's avatar
Kevin Deldycke committed
264
    # Get the filename
265
    self.filename = getattr(file_descriptor, 'filename', 'default_filename')
Kevin Deldycke's avatar
Kevin Deldycke committed
266

Kevin Deldycke's avatar
Kevin Deldycke committed
267 268 269
    # List and load the content of the zip file
    for name in oo_unzipped.namelist():
      self.oo_files[name] = oo_unzipped.read(name)
270
    oo_unzipped.close()
Kevin Deldycke's avatar
Kevin Deldycke committed
271 272 273 274 275 276 277 278 279 280 281 282 283

    # Get the main content and style definitions
    self.oo_content_dom = self.reader.fromString(self.oo_files["content.xml"])
    self.oo_styles_dom  = self.reader.fromString(self.oo_files["styles.xml"])

    # Create a namespace table
    doc_ns = self.oo_styles_dom.getElementsByTagName("office:document-styles")
    for i in range(doc_ns[0].attributes.length):
        if doc_ns[0].attributes.item(i).nodeType == Node.ATTRIBUTE_NODE:
            name = doc_ns[0].attributes.item(i).name
            if name[:5] == "xmlns":
                self.ns[name[6:]] = doc_ns[0].attributes.item(i).value

Kevin Deldycke's avatar
Kevin Deldycke committed
284 285 286 287 288 289
  def getFilename(self):
    """
      Return the name of the OpenOffice file
    """
    return self.filename

290
  def getPicturesMapping(self):
Kevin Deldycke's avatar
Kevin Deldycke committed
291 292 293 294 295 296 297 298 299 300 301
    """
      Return a dictionnary of all pictures in the document
    """
    if len(self.pictures) <= 0:
      for file_name in self.oo_files:
        raw_data = self.oo_files[file_name]
        pict_type = imghdr.what(None, raw_data)
        if pict_type != None:
          self.pictures[file_name] = raw_data
    return self.pictures

302
  def getContentDom(self):
Kevin Deldycke's avatar
Kevin Deldycke committed
303 304 305 306 307
    """
      Return the DOM tree of the main OpenOffice content
    """
    return self.oo_content_dom

308
  def getSpreadsheetsDom(self, include_embedded=False):
309 310 311 312
    """
      Return a list of DOM tree spreadsheets (optionnaly included embedded ones)
    """
    spreadsheets = []
313
    spreadsheets = self.getPlainSpreadsheetsDom()
314
    if include_embedded == True:
315
      spreadsheets += self.getEmbeddedSpreadsheetsDom()
316 317
    return spreadsheets

318
  def getSpreadsheetsMapping(self, include_embedded=False, no_empty_lines=False, normalize=True):
319 320 321
    """
      Return a list of table-like spreadsheets (optionnaly included embedded ones)
    """
322
    tables = {}
323
    tables = self.getPlainSpreadsheetsMapping(no_empty_lines, normalize)
324
    if include_embedded == True:
325
      embedded_tables = self.getEmbeddedSpreadsheetsMapping(no_empty_lines, normalize)
326 327
      tables = self._getTableListUnion(tables, embedded_tables)
    return tables
328

329
  def getPlainSpreadsheetsDom(self):
330 331 332 333 334 335 336 337 338
    """
      Retrieve every spreadsheets from the document and get they DOM tree
    """
    spreadsheets = []
    # List all spreadsheets
    for table in self.oo_content_dom.getElementsByTagName("table:table"):
      spreadsheets.append(table)
    return spreadsheets

339
  def getPlainSpreadsheetsMapping(self, no_empty_lines=False, normalize=True):
340 341 342
    """
      Return a list of plain spreadsheets from the document and transform them as table
    """
343
    tables = {}
344
    for spreadsheet in self.getPlainSpreadsheetsDom():
345
      new_table = self.getSpreadsheetMapping(spreadsheet, no_empty_lines, normalize)
346
      if new_table != None:
347
        tables = self._getTableListUnion(tables, new_table)
348 349
    return tables

350
  def getEmbeddedSpreadsheetsDom(self):
Kevin Deldycke's avatar
Kevin Deldycke committed
351 352 353 354 355 356 357
    """
      Return a list of existing embedded spreadsheets in the file as DOM tree
    """
    spreadsheets = []
    # List all embedded spreadsheets
    emb_objects = self.oo_content_dom.getElementsByTagName("draw:object")
    for embedded in emb_objects:
358 359 360 361 362 363 364 365 366 367 368 369
        document = embedded.getAttributeNS(self.ns["xlink"], "href")
        if document:
            try:
                object_content = self.reader.fromString(self.oo_files[document[3:] + '/content.xml'])
                tables = object_content.getElementsByTagName("table:table")
                if tables:
                    for table in tables:
                        spreadsheets.append(table)
                else: # XXX: insert the link to OLE document ?
                    pass
            except:
                pass
Kevin Deldycke's avatar
Kevin Deldycke committed
370 371
    return spreadsheets

372
  def getEmbeddedSpreadsheetsMapping(self, no_empty_lines=False, normalize=True):
Kevin Deldycke's avatar
Kevin Deldycke committed
373
    """
374
      Return a list of embedded spreadsheets in the document as table
Kevin Deldycke's avatar
Kevin Deldycke committed
375
    """
376
    tables = {}
377
    for spreadsheet in self.getEmbeddedSpreadsheetsDom():
378
      new_table = self.getSpreadsheetMapping(spreadsheet, no_empty_lines, normalize)
Kevin Deldycke's avatar
Kevin Deldycke committed
379
      if new_table != None:
380
        tables = self._getTableListUnion(tables, new_table)
Kevin Deldycke's avatar
Kevin Deldycke committed
381 382
    return tables

383
  def getSpreadsheetMapping(self, spreadsheet=None, no_empty_lines=False, normalize=True):
Kevin Deldycke's avatar
Kevin Deldycke committed
384 385
    """
      This method convert an OpenOffice spreadsheet to a simple table.
386
      This code is based on the oo2pt tool (http://cvs.sourceforge.net/viewcvs.py/collective/CMFReportTool/oo2pt).
Kevin Deldycke's avatar
Kevin Deldycke committed
387
    """
388
    if spreadsheet == None or spreadsheet.nodeName != 'table:table':
Kevin Deldycke's avatar
Kevin Deldycke committed
389 390
      return None

391
    table = []
Kevin Deldycke's avatar
Kevin Deldycke committed
392

393 394 395
    # Get the table name
    table_name = spreadsheet.getAttributeNS(self.ns["table"], "name")

396 397
    # Scan table and store usable informations
    for line in spreadsheet.getElementsByTagName("table:table-row"):
398 399 400 401 402 403

      # TODO : to the same as cell about abusive repeated lines

      line_group_found = line.getAttributeNS(self.ns["table"], "number-rows-repeated")
      if not line_group_found:
        lines_to_repeat = 1
404
      else:
405
        lines_to_repeat = int(line_group_found)
406

407
      for i in range(lines_to_repeat):
408 409
        table_line = []

410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
        # Get all cells
        cells = line.getElementsByTagName("table:table-cell")
        cell_index_range = range(len(cells))

        for cell_index in cell_index_range:
          cell = cells[cell_index]

          # If the cell as no child, cells have no content
          # And if the cell is the last of the row, we don't need to add it to the line
          # So we can go to the next line (= exit this cells loop)
          #
          # I must do this test because sometimes the following cell group
          #   can be found in OOo documents : <table:table-cell table:number-columns-repeated='246'/>
          # This is bad because it create too much irrevelent content that slow down the process
          # So it's a good idea to break the loop in this case
          if cell.childNodes.length == 0 and cell_index == cell_index_range[-1]:
            break

          # Handle cells group
          cell_group_found = cell.getAttributeNS(self.ns["table"], "number-columns-repeated")
          if not cell_group_found:
            cells_to_repeat = 1
432
          else:
433
            cells_to_repeat = int(cell_group_found)
434

435 436 437
          # Ungroup repeated cells
          for j in range(cells_to_repeat):
            # Get the cell content
438 439 440 441 442 443 444 445 446 447
            cell_text = None
            text_tags = cell.getElementsByTagName("text:p")
            for text in text_tags:
              for k in range(text.childNodes.length):
                child = text.childNodes[k]
                if child.nodeType == Node.TEXT_NODE:
                  if cell_text == None:
                    cell_text = ''
                  cell_text += child.nodeValue

448
            # Add the cell to the line
449
            table_line.append(cell_text)
450

Kevin Deldycke's avatar
Kevin Deldycke committed
451 452 453 454 455 456 457 458 459
        # Delete empty lines if needed
        if no_empty_lines:
          empty_cell = 0
          for table_cell in table_line:
            if table_cell == None:
              empty_cell += 1
          if empty_cell == len(table_line):
            table_line = None

460
        # Add the line to the table
Kevin Deldycke's avatar
Kevin Deldycke committed
461 462
        if table_line != None:
          table.append(table_line)
463

464
    # Reduce the table to the minimum
465 466 467 468 469 470 471 472 473 474
    new_table = self._getReducedTable(table)

    # Get a homogenized table
    if normalize:
      table_size = self._getTableSizeDict(new_table)
      new_table = self._getNormalizedBoundsTable( table  = new_table
                                                , width  = table_size['width']
                                                , height = table_size['height']
                                                )
    return {table_name: new_table}
Kevin Deldycke's avatar
Kevin Deldycke committed
475

476
  def _getReducedTable(self, table):
Kevin Deldycke's avatar
Kevin Deldycke committed
477
    """
478
      Reduce the table to its minimum size
Kevin Deldycke's avatar
Kevin Deldycke committed
479 480 481 482 483
    """
    empty_lines = 0
    no_more_empty_lines = 0

    # Eliminate all empty cells at the ends of lines and columns
484
    # Browse the table starting from the bottom for easy empty lines count
485
    for line in range(len(table)-1, -1, -1):
Kevin Deldycke's avatar
Kevin Deldycke committed
486
      empty_cells = 0
487
      line_content = table[line]
Kevin Deldycke's avatar
Kevin Deldycke committed
488
      for cell in range(len(line_content)-1, -1, -1):
489
        if line_content[cell] in ('', None):
Kevin Deldycke's avatar
Kevin Deldycke committed
490 491 492
          empty_cells += 1
        else:
          break
493

Kevin Deldycke's avatar
Kevin Deldycke committed
494 495 496 497
      if (not no_more_empty_lines) and (empty_cells == len(line_content)):
        empty_lines += 1
      else:
        line_size = len(line_content) - empty_cells
498
        table[line] = line_content[:line_size]
Kevin Deldycke's avatar
Kevin Deldycke committed
499 500
        no_more_empty_lines = 1

501 502 503
    table_height = len(table) - empty_lines

    return table[:table_height]
Kevin Deldycke's avatar
Kevin Deldycke committed
504

505 506 507 508
  def _getTableSizeDict(self, table):
    """
      Get table dimension as dictionnary contain both height and width
    """
Kevin Deldycke's avatar
Kevin Deldycke committed
509
    max_cols = 0
510 511 512 513
    for line_index in range(len(table)):
      line = table[line_index]
      if len(line) > max_cols:
        max_cols = len(line)
Kevin Deldycke's avatar
Kevin Deldycke committed
514

515 516 517
    return { 'width' : max_cols
           , 'height': len(table)
           }
Kevin Deldycke's avatar
Kevin Deldycke committed
518

519
  def _getNormalizedBoundsTable(self, table, width=0, height=0):
Kevin Deldycke's avatar
Kevin Deldycke committed
520
    """
521
      Add necessary cells and lines to obtain given bounds
Kevin Deldycke's avatar
Kevin Deldycke committed
522
    """
523 524
    while height > len(table):
      table.append([])
Kevin Deldycke's avatar
Kevin Deldycke committed
525
    for line in range(height):
526 527 528 529
      while width > len(table[line]):
        table[line].append(None)
    return table

530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
  def _getTableListUnion(self, list1, list2):
    """
      Coerce two dict containing tables structures.
      We need to use this method because a OpenOffice document can hold
        several embedded spreadsheets with the same id. This explain the
        use of random suffix in such extreme case.
    """
    for list2_key in list2.keys():
      # Generate a new table ID if needed
      new_key = list2_key
      while new_key in list1.keys():
        new_key = list2_key + '_' + str(random.randint(1000,9999))
      list1[new_key] = list2[list2_key]
    return list1

Kevin Deldycke's avatar
Kevin Deldycke committed
545
allow_class(OOoParser)
Nicolas Delaby's avatar
Nicolas Delaby committed
546
allow_class(CorruptedOOoFile)
547 548 549 550

def newOOoParser(container):
  return OOoParser().__of__(container)