Implement SSL auth usage in libnetworkcache.

Squashed commit of the following:

commit 2c3b7f2f
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Wed Aug 31 16:36:18 2011 +0200

    Prove that keys are passed to libnetworkcache.

commit c380b1ea
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Wed Aug 31 15:42:05 2011 +0200

    Support shacache and dir SSL based auth.

src/zc/buildout/buildout.py

@@ -355,6 +355,10 @@ class Buildout(UserDict.DictMixin):
             self.upload_dir_url = None
             self.signature_private_key_file = None
             self.signature_certificate_list = None
+            self.shacache_cert_file = None
+            self.shacache_key_file = None
+            self.shadir_cert_file = None
+            self.shadir_key_file = None
         else:
             # support networkcache
             networkcache_section_name = options.get('networkcache-section')
@@ -372,6 +376,14 @@ class Buildout(UserDict.DictMixin):
                     'signature-certificate-list', '')
                 self.signature_private_key_file = networkcache_section.get(
                     'signature-private-key-file', '')
+                self.shacache_cert_file = networkcache_section.get(
+                    'shacache-cert-file', '')
+                self.shacache_key_file = networkcache_section.get(
+                    'shacache-key-file', '')
+                self.shadir_cert_file = networkcache_section.get(
+                    'shadir-cert-file', '')
+                self.shadir_key_file = networkcache_section.get(
+                    'shadir-key-file', '')
 
                 # parse signature list
                 cert_marker = '-----BEGIN CERTIFICATE-----'
@@ -391,6 +403,10 @@ class Buildout(UserDict.DictMixin):
                 options['__networkcache__upload-dir-url'] = self.upload_dir_url
                 options['__networkcache__signature-certificate-list'] = signature_certificate_list
                 options['__networkcache__signature-private-key-file'] = self.signature_private_key_file
+                options['__networkcache__shacache-cert-file'] = self.shacache_cert_file
+                options['__networkcache__shacache-key-file'] = self.shacache_key_file
+                options['__networkcache__shadir-cert-file'] = self.shadir_cert_file
+                options['__networkcache__shadir-key-file'] = self.shadir_key_file
 
             else:
                 self._logger.debug('Networkcache functionality not enabled. '
@@ -401,6 +417,10 @@ class Buildout(UserDict.DictMixin):
                 self.upload_dir_url = None
                 self.signature_private_key_file = None
                 self.signature_certificate_list = None
+                self.shacache_cert_file = None
+                self.shacache_key_file = None
+                self.shadir_cert_file = None
+                self.shadir_key_file = None
 
     def _buildout_path(self, name):
         if '${' in name:
@@ -442,6 +462,10 @@ class Buildout(UserDict.DictMixin):
                 upload_dir_url=self.upload_dir_url,
                 signature_private_key_file=self.signature_private_key_file,
                 signature_certificate_list=self.signature_certificate_list,
+                shacache_cert_file=self.shacache_cert_file,
+                shacache_key_file=self.shacache_key_file,
+                shadir_cert_file=self.shadir_cert_file,
+                shadir_key_file=self.shadir_key_file,
                 )
 
         # Now copy buildout and setuptools eggs, and record destination eggs:
@@ -954,6 +978,10 @@ class Buildout(UserDict.DictMixin):
             upload_dir_url=self.upload_dir_url,
             signature_certificate_list=self.signature_certificate_list,
             signature_private_key_file=self.signature_private_key_file,
+            shacache_cert_file=self.shacache_cert_file,
+            shacache_key_file=self.shacache_key_file,
+            shadir_cert_file=self.shadir_cert_file,
+            shadir_key_file=self.shadir_key_file,
             )
 
         upgraded = []
@@ -1193,6 +1221,14 @@ def _install_and_load(spec, group, entry, buildout):
                     '__networkcache__signature-private-key-file'),
                 signature_certificate_list=buildout_options.get(
                     '__networkcache__signature-certificate-list'),
+                shacache_cert_file=buildout_options.get(
+                    '__networkcache__shacache-cert-file'),
+                shacache_key_file=buildout_options.get(
+                    '__networkcache__shacache-key-file'),
+                shadir_cert_file=buildout_options.get(
+                    '__networkcache__shadir-cert-file'),
+                shadir_key_file=buildout_options.get(
+                    '__networkcache__shadir-key-file'),
                 )
 
         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry

src/zc/buildout/download.py

@@ -86,6 +86,14 @@ class Download(object):
             if q.strip()]
         self.signature_private_key_file = options.get(
             '__networkcache__signature-private-key-file')
+        self.shacache_cert_file = options.get(
+            '__networkcache__shacache-cert-file')
+        self.shacache_key_file = options.get(
+            '__networkcache__shacache-key-file')
+        self.shadir_cert_file = options.get(
+            '__networkcache__shadir-cert-file')
+        self.shadir_key_file = options.get(
+            '__networkcache__shadir-key-file')
 
     @property
     def download_cache(self):
@@ -203,7 +211,11 @@ class Download(object):
                     if self.upload_cache_url and self.upload_dir_url:
                         upload_network_cached(self.upload_dir_url,
                             self.upload_cache_url, url, tmp_path, self.logger,
-                            self.signature_private_key_file)
+                            self.signature_private_key_file,
+                            self.shacache_cert_file,
+                            self.shacache_key_file,
+                            self.shadir_cert_file,
+                            self.shadir_key_file)
             finally:
                 os.close(handle)
         except:

src/zc/buildout/easy_install.py

@@ -346,7 +346,11 @@ class Installer:
                  upload_dir_url=None,
                  upload_cache_url=None,
                  signature_certificate_list=None,
-                 signature_private_key_file=None
+                 signature_private_key_file=None,
+                 shacache_cert_file=None,
+                 shacache_key_file=None,
+                 shadir_cert_file=None,
+                 shadir_key_file=None,
                  ):
         self._dest = dest
         self._allow_hosts = allow_hosts
@@ -421,6 +425,10 @@ class Installer:
         self._upload_cache_url = upload_cache_url
         self._signature_certificate_list = signature_certificate_list
         self._signature_private_key_file = signature_private_key_file
+        self._shacache_cert_file = shacache_cert_file
+        self._shacache_key_file = shacache_key_file
+        self._shadir_cert_file = shadir_cert_file
+        self._shadir_key_file = shadir_key_file
 
     _allowed_eggs_from_site_packages_regex = None
     def allow_site_package_egg(self, name):
@@ -730,7 +738,9 @@ class Installer:
             if self._upload_cache_url and self._upload_dir_url:
                 upload_network_cached(self._upload_dir_url,
                    self._upload_cache_url, dist.location, new_location, logger,
-                   self._signature_private_key_file)
+                   self._signature_private_key_file,
+                   self._shacache_cert_file, self._shacache_key_file,
+                   self._shadir_cert_file, self._shadir_key_file)
 
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
@@ -1111,7 +1121,9 @@ def install(specs, dest,
             include_site_packages=None, allowed_eggs_from_site_packages=None,
             prefer_final=None, download_dir_url=None, download_cache_url=None,
             upload_dir_url=None, upload_cache_url=None,
-            signature_certificate_list=None, signature_private_key_file=None):
+            signature_certificate_list=None, signature_private_key_file=None,
+            shacache_cert_file=None, shacache_key_file=None,
+            shadir_cert_file=None, shadir_key_file=None):
     installer = Installer(
         dest, links, index, executable, always_unzip, path, newest,
         versions, use_dependency_links, allow_hosts=allow_hosts,
@@ -1121,7 +1133,11 @@ def install(specs, dest,
         download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
         upload_cache_url=upload_cache_url,
         signature_certificate_list=signature_certificate_list,
-        signature_private_key_file=signature_private_key_file)
+        signature_private_key_file=signature_private_key_file,
+        shacache_cert_file=shacache_cert_file,
+        shacache_key_file=shacache_key_file,
+        shadir_cert_file=shadir_cert_file,
+        shadir_key_file=shadir_key_file)
     return installer.install(specs, working_set)
 
 

src/zc/buildout/networkcache.py

@@ -105,7 +105,8 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
 
 
 def upload_network_cached(dir_url, cache_url, external_url, path, logger,
-   signature_private_key_file):
+   signature_private_key_file, shacache_cert_file, shacache_key_file,
+   shadir_cert_file, shadir_key_file):
     """Upload file to a network cache server"""
     if not LIBNETWORKCACHE_ENABLED:
         return False
@@ -122,12 +123,24 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
               urlmd5=hashlib.md5(external_url).hexdigest())
 
     f = open(path, 'r')
+    # convert '' into None in order to call nc nicely
     if not signature_private_key_file:
-        # convert '' into None in order to call nc nicely
         signature_private_key_file = None
+    if not shacache_cert_file:
+        shacache_cert_file = None
+    if not shacache_key_file:
+        shacache_key_file = None
+    if not shadir_cert_file:
+        shadir_cert_file = None
+    if not shadir_key_file:
+        shadir_key_file = None
     try:
         nc = NetworkcacheClient(cache_url, dir_url,
-            signature_private_key_file=signature_private_key_file)
+            signature_private_key_file=signature_private_key_file,
+            shacache_cert_file=shacache_cert_file,
+            shacache_key_file=shacache_key_file,
+            shadir_cert_file=shadir_cert_file,
+            shadir_key_file=shadir_key_file)
     except TypeError:
         logger.warning('Incompatible version of networkcache, not using it.')
         return False

src/zc/buildout/networkcache.txt

@@ -693,6 +693,97 @@ Of course eggs are also supported by networkcache:
    Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
    Got demoneeded 1.2c1.
 
+It is possible to pass SSL certificates and key to be used by authentication.
+In order to do it NetworkcacheClient.upload method have to be patched to show
+that proper keys are used.
+
+   >>> mkdir(sample_buildout, 'ncpatch')
+   >>> write(sample_buildout, 'ncpatch', 'ncpatch.py',
+   ... """
+   ... def patched(self, *args, **kwargs):
+   ...     print 'shacache_cert_file', self.shacache_cert_file
+   ...     print 'shacache_key_file', self.shacache_key_file
+   ...     print 'shadir_cert_file', self.shadir_cert_file
+   ...     print 'shadir_key_file', self.shadir_key_file
+   ...     return False
+   ...
+   ... def ext(buildout):
+   ...     print 'Patching slapos.libnetworkcache'
+   ...     import slapos.libnetworkcache
+   ...     slapos.libnetworkcache.NetworkcacheClient.upload = patched
+   ... """)
+   >>> write(sample_buildout, 'ncpatch', 'setup.py',
+   ... """
+   ... from setuptools import setup
+   ...
+   ... setup(
+   ...     name = "ncpatch",
+   ...     entry_points = {'zc.buildout.extension': ['ncpatch = ncpatch:ext']},
+   ...     )
+   ... """)
+   >>> write(sample_buildout, 'ncpatch', 'README.txt', " ")
+
+Lets activiate this extension:
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... develop = ncpatch
+    ... parts =
+    ... """)
+   >>> print system(buildout)
+   Develop: '/sample-buildout/ncpatch'
+   Uninstalling eggs.
+
+Lets clean networkcache:
+
+   >>> rmdir(nc_data)
+   >>> mkdir(nc_data)
+
+And see what happens:
+
+   >>> write(sample_buildout, 'buildout.cfg',
+   ... '''
+   ... [buildout]
+   ...
+   ... networkcache-section = networkcache
+   ... find-links = %(link_server)s
+   ... develop = ncpatch download
+   ... extensions = ncpatch
+   ... parts = download
+   ...
+   ... [download]
+   ... recipe = download
+   ... url = %(remote_server_url)shello.txt
+   ...
+   ... [networkcache]
+   ... download-cache-url = %(nc_url)sshacache
+   ... download-dir-url = %(nc_url)sshadir
+   ... upload-cache-url = %(nc_url)sshacache
+   ... upload-dir-url = %(nc_url)sshadir
+   ... shacache-cert-file = /path/to/shacache/cert/file
+   ... shacache-key-file = /path/to/shacache/key/file
+   ... shadir-cert-file = /path/to/shadir/cert/file
+   ... shadir-key-file = /path/to/shadir/key/file
+   ... ''' % globals())
+   >>> print system(buildout)
+   Networkcache enabled.
+   Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
+   Patching slapos.libnetworkcache
+   Develop: '/sample-buildout/ncpatch'
+   Develop: '/sample-buildout/download'
+   Installing download.
+   Downloading http://localhost/hello.txt
+   Downloading hello.txt from network cache.
+   Failed to download from network cache hello.txt:...404...
+   Uploading http://localhost/hello.txt into network cache.
+   shacache_cert_file /path/to/shacache/cert/file
+   shacache_key_file /path/to/shacache/key/file
+   shadir_cert_file /path/to/shadir/cert/file
+   shadir_key_file /path/to/shadir/key/file
+   download: Downloaded http://localhost/hello.txt
+
 Lets clean networkcache:
 
    >>> rmdir(nc_data)

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -94,7 +94,12 @@ class Eggs(object):
                 ('__networkcache__upload-cache-url', 'upload_cache_url'),
                 ('__networkcache__upload-dir-url', 'upload_dir_url'),
                 ('__networkcache__signature-private-key-file', 'signature_private_key_file'),
-                ('__networkcache__signature-certificate-list', 'signature_certificate_list')):
+                ('__networkcache__signature-certificate-list', 'signature_certificate_list'),
+                ('__networkcache__shacache-cert-file', 'shacache_cert_file'),
+                ('__networkcache__shacache-key-file', 'shacache_key_file'),
+                ('__networkcache__shadir-cert-file', 'shadir_cert_file'),
+                ('__networkcache__shadir-key-file', 'shadir_key_file'),
+                ):
                 if option_key in b_options:
                     if option_key == '__networkcache__signature-certificate-list':
                         cert_marker = '-----BEGIN CERTIFICATE-----'