Remove M2Crypto dependency.

Squashed commit of the following:

commit 5c74964a312fb743f88a115ffb4d5af44dd3d2d1
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Fri Sep 2 10:22:48 2011 +0200

    No more M2Crypto is required.

commit 71ddc285850bd1c5761b95cff8616c6023f6eadf
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Fri Sep 2 10:22:03 2011 +0200

    Drop M2Crypto dependency.

    Simplify, generate only common name in key.

commit 9e058070bc66fc91a590d90adf379a899b00d124
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 18:05:14 2011 +0200

    Cover cases when openssl is not compatible.

    It seems that python executable can be seen as openssl begin not compatible.

commit 69901f159392db65597b97482439bea4d002e9c4
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 18:00:27 2011 +0200

    Raise correctly in case of openssl unavailabilty during upload.

    More: raise DirectoryNotFound in more cases.

commit c17c46053385b586e7ccdb4ebf3ecb9ab46d901b
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 18:00:08 2011 +0200

    Cover case when no openssl is available during upload.

commit d88d9e2b892000fa64c28b74182777fd03c93556
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 17:55:23 2011 +0200

    Covert case when empty shadir entry is in shadir.

commit 6671546db794a5ea8e7025de115a613940e4af14
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 17:45:08 2011 +0200

    Add no openssl during content selection test.

    Add another test skeleton.

commit ad6b3b5abbbb2d8267938661946acf1bd4d02b43
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 17:32:09 2011 +0200

    Close another temporary file.

commit 34d15b585e52810908323cf846af5a64171f62b2
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 17:23:23 2011 +0200

    Expose openssl and avoid dying in case if not exists.

commit 0eb456fc6ccd2d36644c47abc1ef79b7b51187c6
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 17:15:54 2011 +0200

    Pep8isation.

commit 7179acf759a1d18bff367025a965d020443f93e2
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 1 17:12:25 2011 +0200

    Drop M2Crypto dependency.

    M2Crypto is unmantained library. It seems safer to use openssl directly to
    sign and verify content.

setup.py

@@ -32,7 +32,6 @@ setup(
     keywords="slapos networkcache shadir shacache",
     install_requires=[
       'setuptools', # for namespace
-      'M2Crypto', # required for certificate checking
     ] + additional_install_requires,
     classifiers=[
       'Development Status :: 4 - Beta',

slapos/libnetworkcache.py

@@ -18,11 +18,12 @@ import hashlib
 import httplib
 import json
 import os
+import socket
+import subprocess
 import tempfile
+import traceback
 import urllib2
 import urlparse
-import M2Crypto
-import socket
 
 
 class NetworkcacheClient(object):
@@ -33,6 +34,7 @@ class NetworkcacheClient(object):
      - SHACACHE
   '''
 
+  openssl = 'openssl'
   def parseUrl(self, url):
     return_dict = {}
     parsed_url = urlparse.urlparse(url)
@@ -161,7 +163,11 @@ class NetworkcacheClient(object):
         kw['architecture'] = architecture
 
       sha_entry = json.dumps(kw)
-      signature = self._getSignatureString(sha_entry)
+      try:
+        signature = self._getSignatureString(sha_entry)
+      except Exception:
+        raise UploadError('Impossible to sign content, error:\n%s' %
+          traceback.format_exc())
       data = [sha_entry, signature]
 
       if self.shadir_scheme == 'https':
@@ -190,7 +196,6 @@ class NetworkcacheClient(object):
     It uses http GET request method.
     '''
     sha_cache_url = os.path.join(self.shacache_url, sha512sum)
-    file_descriptor = tempfile.NamedTemporaryFile()
     request = urllib2.Request(url=sha_cache_url, data=None,
       headers=self.shadir_header_dict)
     return urllib2.urlopen(request)
@@ -200,7 +205,6 @@ class NetworkcacheClient(object):
     Raise DirectoryNotFound if multiple files are found.
     '''
     url = os.path.join(self.shadir_url, key)
-    file_descriptor = tempfile.NamedTemporaryFile()
     request = urllib2.Request(url=url, data=None,
       headers=self.shadir_header_dict)
     data = urllib2.urlopen(request).read()
@@ -211,11 +215,12 @@ class NetworkcacheClient(object):
       for data in data_list:
         if self._verifySignatureInCertificateList(data[0], data[1]):
           filtered_data_list.append(data)
-      if not filtered_data_list:
-        raise DirectoryNotFound('Could not find a trustable entry.')
     else:
       filtered_data_list = data_list
 
+    if len(filtered_data_list) == 0:
+      raise DirectoryNotFound('Could not find a trustable entry.')
+
     if len(filtered_data_list) > 1:
       raise DirectoryNotFound('Too many entries for a given key %r. ' \
                'Entries: %s.' % (key, str(data_list)))
@@ -232,12 +237,16 @@ class NetworkcacheClient(object):
     if self.signature_private_key_file is None:
       return ''
 
-    SignEVP = M2Crypto.EVP.load_key(self.signature_private_key_file)
-    SignEVP.sign_init()
-    SignEVP.sign_update(content)
-    StringSignature = SignEVP.sign_final()
-    signature = StringSignature.encode('base64')
-    return signature
+    content_file = tempfile.NamedTemporaryFile()
+    content_file.write(content)
+    content_file.flush()
+    content_file.seek(0)
+    try:
+      signature = subprocess.check_output([self.openssl, "dgst", "-sha1",
+            "-sign", self.signature_private_key_file, content_file.name])
+      return signature.encode('base64')
+    finally:
+      content_file.close()
 
   def _verifySignatureInCertificateList(self, content, signature_string):
     """
@@ -246,25 +255,55 @@ class NetworkcacheClient(object):
     """
     if self.signature_certificate_list is not None:
       for certificate in self.signature_certificate_list:
-        if self._verifySignatureCertificate(content, signature_string, certificate):
+        if self._verifySignatureCertificate(content, signature_string,
+            certificate):
           return True
     return False
 
-  def _verifySignatureCertificate(self, content, signature_string, certificate):
+  def _verifySignatureCertificate(self, content, signature_string,
+      certificate):
     """ verify if the signature is valid for a given certificate. """
     certificate_file = tempfile.NamedTemporaryFile()
     certificate_file.write(certificate)
     certificate_file.flush()
     certificate_file.seek(0)
+    signature_file = tempfile.NamedTemporaryFile()
+    signature_file.write(signature_string.decode('base64'))
+    signature_file.flush()
+    signature_file.seek(0)
+    content_file = tempfile.NamedTemporaryFile()
+    content_file.write(content)
+    content_file.flush()
+    content_file.seek(0)
+    pubkey_file = tempfile.NamedTemporaryFile()
     try:
-      PubKey = M2Crypto.X509.load_cert(certificate_file.name)
-      VerifyEVP = M2Crypto.EVP.PKey()
-      VerifyEVP.assign_rsa(PubKey.get_pubkey().get_rsa())
-      VerifyEVP.verify_init()
-      VerifyEVP.verify_update(str(content))
-      return VerifyEVP.verify_final(signature_string.decode('base64'))
+      last_output = ''
+      try:
+        last_output = subprocess.check_output([self.openssl, "x509", "-pubkey",
+            "-noout", "-in", certificate_file.name])
+        pubkey_file.write(last_output)
+        pubkey_file.flush()
+        pubkey_file.seek(0)
+        try:
+          last_output = subprocess.check_output([self.openssl, "dgst", "-sha1",
+              "-verify", pubkey_file.name, "-signature", signature_file.name,
+              content_file.name])
+        except subprocess.CalledProcessError, e:
+          # in case if verification failed
+          last_output = e.output
+        if last_output.startswith('Verified OK'):
+          return True
+      except Exception:
+        # in case of failure, emit *anything*, but swallow all what possible
+        print last_output
+        print traceback.format_exc()
+      return False
     finally:
       certificate_file.close()
+      signature_file.close()
+      content_file.close()
+      pubkey_file.close()
+
 
 class DirectoryNotFound(Exception):
   pass

slapos/libnetworkcachetests.py

@@ -15,6 +15,7 @@ import time
 import unittest
 import slapos.libnetworkcache
 import slapos.signature
+import sys
 
 
 class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler):
@@ -434,6 +435,128 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     result = signed_nc.select(key)
     self.assertEqual(result.read(), self.test_string)
 
+  def test_upload_signed_content_openssl_not_available(self):
+    key = 'somekey' + str(random.random())
+    urlmd5 = str(random.random())
+    file_name = 'my file'
+    test_data = tempfile.TemporaryFile()
+    test_string = str(random.random())
+    test_data.write(test_string)
+    test_data.seek(0)
+    key_file = tempfile.NamedTemporaryFile()
+    key_file.write(self.key)
+    key_file.flush()
+    key_file.seek(0)
+
+    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
+      self.shacache, self.shadir, key_file.name, [self.certificate])
+
+    signed_nc.openssl = '/doesnotexists'
+    try:
+      signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
+    except slapos.libnetworkcache.UploadError, e:
+      self.assertTrue(str(e).startswith("Impossible to sign content, error"))
+
+  def test_upload_signed_content_openssl_non_functional(self):
+    key = 'somekey' + str(random.random())
+    urlmd5 = str(random.random())
+    file_name = 'my file'
+    test_data = tempfile.TemporaryFile()
+    test_string = str(random.random())
+    test_data.write(test_string)
+    test_data.seek(0)
+    key_file = tempfile.NamedTemporaryFile()
+    key_file.write(self.key)
+    key_file.flush()
+    key_file.seek(0)
+
+    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
+      self.shacache, self.shadir, key_file.name, [self.certificate])
+
+    signed_nc.openssl = sys.executable
+    try:
+      signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
+    except slapos.libnetworkcache.UploadError, e:
+      self.assertTrue(str(e).startswith("Impossible to sign content, error"))
+
+  def test_select_signed_content_openssl_not_available(self):
+    key = 'somekey' + str(random.random())
+    urlmd5 = str(random.random())
+    file_name = 'my file'
+    test_data = tempfile.TemporaryFile()
+    test_string = str(random.random())
+    test_data.write(test_string)
+    test_data.seek(0)
+    key_file = tempfile.NamedTemporaryFile()
+    key_file.write(self.key)
+    key_file.flush()
+    key_file.seek(0)
+
+    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
+      self.shacache, self.shadir, key_file.name, [self.certificate])
+
+    self.assertEqual(True, signed_nc.upload(self.test_data, key,
+        urlmd5=urlmd5, file_name=file_name))
+    try:
+      # disable openssl during download
+      signed_nc.openssl = '/doesnotexists'
+      result = signed_nc.select(key)
+    except slapos.libnetworkcache.DirectoryNotFound, msg:
+      # No way to download content from shadir w/o openssl
+      self.assertEqual(str(msg), 'Could not find a trustable entry.')
+
+  def test_select_signed_content_openssl_non_functional(self):
+    key = 'somekey' + str(random.random())
+    urlmd5 = str(random.random())
+    file_name = 'my file'
+    test_data = tempfile.TemporaryFile()
+    test_string = str(random.random())
+    test_data.write(test_string)
+    test_data.seek(0)
+    key_file = tempfile.NamedTemporaryFile()
+    key_file.write(self.key)
+    key_file.flush()
+    key_file.seek(0)
+
+    signed_nc = slapos.libnetworkcache.NetworkcacheClient(
+      self.shacache, self.shadir, key_file.name, [self.certificate])
+
+    self.assertEqual(True, signed_nc.upload(self.test_data, key,
+        urlmd5=urlmd5, file_name=file_name))
+    try:
+      # disable openssl during download
+      signed_nc.openssl = sys.executable
+      result = signed_nc.select(key)
+    except slapos.libnetworkcache.DirectoryNotFound, msg:
+      # No way to download content from shadir w/o openssl
+      self.assertEqual(str(msg), 'Could not find a trustable entry.')
+
+  def test_select_no_entries(self):
+    key = 'somekey' + str(random.random())
+    urlmd5 = str(random.random())
+    file_name = 'my file'
+    test_data = tempfile.TemporaryFile()
+    test_string = str(random.random())
+    test_data.write(test_string)
+    test_data.seek(0)
+    key_file = tempfile.NamedTemporaryFile()
+    key_file.write(self.key)
+    key_file.flush()
+    key_file.seek(0)
+
+    nc = slapos.libnetworkcache.NetworkcacheClient(
+      self.shacache, self.shadir)
+    self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
+        file_name=file_name))
+    f = os.path.join(self.tree, 'shadir', key)
+    # now remove the entry from shacache
+    open(f, 'w').write(json.dumps([]))
+    try:
+      nc.select(key)
+    except slapos.libnetworkcache.DirectoryNotFound, msg:
+      # empty content in shadir so usual exception is raised
+      self.assertEqual(str(msg), 'Could not find a trustable entry.')
+
   def test_select_signed_content_server_hacked(self):
     key = 'somekey' + str(random.random())
     urlmd5 = str(random.random())

slapos/signature.py

@@ -13,100 +13,35 @@
 ##############################################################################
 
 
-import M2Crypto
-import time
 import argparse
+import os
+import subprocess
 import sys
 
 
-def createPrivateKeyAndCertificateFile(signature_certificate_file, signature_private_key_file, **kw):
-  ''' It must create a private key and a certificate file. '''
-  cur_time = M2Crypto.ASN1.ASN1_UTCTIME()
-  cur_time.set_time(int(time.time()) - 60*60*24)
-  expire_time = M2Crypto.ASN1.ASN1_UTCTIME()
-  expire_time.set_time(int(time.time()) + 60 * 60 * 24 * 365)
+def generateCertificate(certificate_file, key_file, common_name):
+  if os.path.lexists(certificate_file):
+    raise ValueError("Certificate %r exists, will not overwrite." %
+      certificate_file)
+  if os.path.lexists(key_file):
+    raise ValueError("Key %r exists, will not overwrite." %
+      key_file)
 
-  cs_rsa = M2Crypto.RSA.gen_key(1024, 65537, lambda: None)
-  cs_pk = M2Crypto.EVP.PKey()
-  cs_pk.assign_rsa(cs_rsa)
+  subj = '/CN=%s' % common_name
+  subprocess.check_call(["openssl", "req", "-x509", "-nodes", "-days", "365",
+    "-subj", subj, "-newkey", "rsa:1024", "-keyout", key_file, "-out",
+    certificate_file])
 
-  cs_cert = M2Crypto.X509.X509()
-  cs_cert.set_not_before(cur_time)
-  cs_cert.set_not_after(expire_time)
-  cs_cert.set_pubkey(cs_pk)
 
-  cs_name = M2Crypto.X509.X509_Name()
-  cs_name.C = kw.get('country', '')
-  cs_name.SP = kw.get('state_name', '')
-  cs_name.L = kw.get('locality', '')
-  cs_name.O = kw.get('organization_name', '')
-  cs_name.OU = kw.get('organization_unit_name', '')
-  cs_name.CN = kw.get('common_name', '')
-  cs_name.Email = kw.get('common_name', '')
-
-  cs_cert.set_subject(cs_name)
-  cs_cert.set_issuer_name(cs_name)
-  cs_cert.sign(cs_pk, md="sha256")
-
-  # Saving...
-  cs_cert.save_pem(signature_certificate_file)
-  cs_rsa.save_pem(signature_private_key_file, None)
-  return cs_cert.as_text()
-
-
-def parseArgument(*args):
-  ''' Parses command line arguments. '''
+def run(*args):
   parser = argparse.ArgumentParser()
-  parser.add_argument('--signature-certificate-file',
-                      default='public.pem',
-                      help='X509 Cetification file.')
-
-  parser.add_argument('--signature-private-key-file',
-                      default='private.pem',
-                      help='Signature private key file.')
-
-  parser.add_argument('-c','--country',
-                      default='XX',
-                      help='Country Name (2 letter code).')
-
-  parser.add_argument('-s', '--state-name',
-                      default='Default Province',
-                      help='State or Province Name (full name).')
-
-  parser.add_argument('-l', '--locality-name',
-                      default='Default City',
-                      help='Locality Name (eg, city).')
-
-  parser.add_argument('-o', '--organization-name',
-                      default='Default Company Ltd',
-                      help='Organization Name (eg, company).')
-
-  parser.add_argument('-ou', '--organization-unit-name',
-                      default='',
-                      help='Organizational Unit Name (eg, section).')
-
-  parser.add_argument('-cn', '--common-name',
-                      default='',
-                      help="Common Name (eg, your name or your server's " \
-                           "hostname).")
-
-  parser.add_argument('-e', '--email',
-                      default='',
-                      help='Email Address.')
-
-  argument_option_instance = parser.parse_args(list(args))
-  option_dict = {}
-  for argument_key, argument_value in vars(argument_option_instance
-      ).iteritems():
-    if argument_value is not None:
-      option_dict.update({argument_key: argument_value})
-
-  return option_dict
+  parser.add_argument('certificate_file', type=str,
+    help='Certificate file to generate.')
 
+  parser.add_argument('key_file', type=str,
+    help='Key file to generate.')
+  parser.add_argument('common_name', type=str, help="Common Name")
 
-def run():
-  ''' Validate the parameters and call the function to create
-    private key and certificate file.
-  '''
-  option_dict = parseArgument(*sys.argv[1:])
-  print createPrivateKeyAndCertificateFile(**option_dict)
+  option = parser.parse_args(list(args) or sys.argv[1:])
+  generateCertificate(option.certificate_file,
+    option.key_file, option.common_name)