downloadunpacked.py 7.65 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
import os
import logging
import shutil
30
import subprocess
31
import tarfile
32 33 34 35
import zc.buildout
import tempfile
import setuptools

36
is_true = ('false', 'true').index
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59

class Recipe:

  def __init__(self, buildout, name, options):
    self.buildout = buildout
    self.name = name
    self.options = options
    self.logger = logging.getLogger(self.name)
    if 'filename' in self.options and 'destination' in self.options:
      raise zc.buildout.UserError('Parameters filename and destination are '
          'exclusive.')
    self.parts = None
    self.destination = self.options.get('destination', None)
    if self.destination is None:
      self.parts = os.path.join(self.buildout['buildout']['parts-directory'],
          self.name)
      self.destination = self.parts
      # backward compatibility with other recipes -- expose location
      options['location'] = os.path.join(self.buildout['buildout'][
        'parts-directory'], self.name)
    options['target'] = self.destination
    options.setdefault('extract-directory', '')

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
    self.environ = {}
    self.original_environment = os.environ.copy()
    environment_section = self.options.get('environment-section', '').strip()
    if environment_section and environment_section in buildout:
      # Use environment variables from the designated config section.
      self.environ.update(buildout[environment_section])
    for variable in self.options.get('environment', '').splitlines():
      if variable.strip():
        try:
          key, value = variable.split('=', 1)
          self.environ[key.strip()] = value
        except ValueError:
          raise zc.buildout.UserError('Invalid environment variable definition: %s', variable)
    # Extrapolate the environment variables using values from the current
    # environment.
    for key in self.environ:
      self.environ[key] = self.environ[key] % os.environ

78 79 80 81
  def install(self):
    if self.parts is not None:
      if not os.path.isdir(self.parts):
        os.mkdir(self.parts)
82

83 84 85
    download = zc.buildout.download.Download(self.buildout['buildout'],
        hash_name=True, cache=self.buildout['buildout'].get('download-cache'))
    extract_dir = tempfile.mkdtemp(self.name)
86
    try:
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
      self.logger.debug('Created working directory %r', extract_dir)
      path, is_temp = download(self.options['url'],
          md5sum=self.options.get('md5sum'))
      try:
        patch_archive_util()
        # ad-hoc support for .xz and .lz archive
        hdr = open(path, 'rb').read(6)
        for magic, cmd in (('\xfd7zXZ\x00', ('xzcat',)),
                           ('LZIP', ('lunzip', '-c'))):
          if hdr.startswith(magic):
            new_path = os.path.join(extract_dir, os.path.basename(path))
            with open(new_path, 'wb') as stdout:
              subprocess.check_call(cmd + (path,),
                stdout=stdout, env=self.environ)
            setuptools.archive_util.unpack_archive(new_path, extract_dir)
            os.unlink(new_path)
            break
        else:
          setuptools.archive_util.unpack_archive(path, extract_dir)
      finally:
        unpatch_archive_util()
        if is_temp:
          os.unlink(path)
110

111 112
      if os.path.exists(self.destination):
        shutil.rmtree(self.destination)
113 114
      os.makedirs(self.destination)

115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
      strip = self.options.get('strip-top-level-dir')
      if strip:
        if is_true(strip.lower()):
          base_dir, = os.listdir(extract_dir)
          base_dir = os.path.join(extract_dir, base_dir)
        else:
          base_dir = extract_dir
      else:
        directories = os.listdir(extract_dir)
        if len(directories) == 1:
          base_dir = os.path.join(extract_dir, directories[0])
          if not os.path.isdir(base_dir):
            base_dir = extract_dir
      base_dir = os.path.join(base_dir, self.options['extract-directory'])
      for filename in os.listdir(base_dir):
        shutil.move(os.path.join(base_dir, filename), self.destination)
    finally:
      shutil.rmtree(extract_dir)
    self.logger.debug('Downloaded %r and saved to %r.',
      self.options['url'], self.destination)
135 136 137 138 139 140 141
    if self.parts is not None:
      return [self.parts]
    else:
      return []
  
  def update(self):
    pass
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166

# Monkey patch to keep symlinks in tarfile
def unpack_tarfile_patched(filename, extract_dir, progress_filter=setuptools.archive_util.default_filter):
    """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir`

    Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined
    by ``tarfile.open()``).  See ``unpack_archive()`` for an explanation
    of the `progress_filter` argument.
    """
    try:
        tarobj = tarfile.open(filename)
    except tarfile.TarError:
        raise setuptools.archive_util.UnrecognizedFormat(
            "%s is not a compressed or uncompressed tar file" % (filename,)
        )
    with setuptools.archive_util.contextlib.closing(tarobj):
        # don't do any chowning!
        tarobj.chown = lambda *args: None
        for member in tarobj:
            name = member.name
            # don't extract absolute paths or ones with .. in them
            if not name.startswith('/') and '..' not in name.split('/'):
                prelim_dst = os.path.join(extract_dir, *name.split('/'))

                if member is not None and (member.isfile() or member.isdir() or member.islnk() or member.issym()):
167 168 169
                    # Prepare the link target for makelink().
                    if member.islnk():
                        member._link_target = os.path.join(extract_dir, member.linkname)
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
                    final_dst = progress_filter(name, prelim_dst)
                    if final_dst:
                        if final_dst.endswith(os.sep):
                            final_dst = final_dst[:-1]
                        try:
                            # XXX Ugh
                            tarobj._extract_member(member, final_dst)
                        except tarfile.ExtractError:
                            # chown/chmod/mkfifo/mknode/makedev failed
                            pass
        return True

def patch_archive_util():
  setuptools.archive_util.extraction_drivers = (
    setuptools.archive_util.unpack_directory,
    setuptools.archive_util.unpack_zipfile,
    unpack_tarfile_patched,
  )

def unpatch_archive_util():
  setuptools.archive_util.extraction_drivers = (
    setuptools.archive_util.unpack_directory,
    setuptools.archive_util.unpack_zipfile,
    setuptools.archive_util.unpack_tarfile,
  )