caucase.{cli,client,utils},shell/caucase.sh: Add support for CA cert directories.

Not all programs support having multiple CA certificates per file, so add
support for creating and maintaining certificate directories containing
a single certificate each.

caucase/cli.py

@@ -379,13 +379,27 @@ def main(argv=None, stdout=sys.stdout, stderr=sys.stderr):
     '--ca-crt',
     default='cas.crt.pem',
     metavar='CRT_PATH',
-    help='Services CA certificate file location. default: %(default)s',
+    help='Services CA certificate file location. '
+    'May be an existing directory or file, or non-existing. '
+    'If non-existing and given path has an extension, a file will be created, '
+    'otherwise a directory will be. '
+    'When it is a file, it may contain multiple PEM-encoded concatenated '
+    'certificates. When it is a directory, it may contain multiple files, '
+    'each containing a single PEM-encoded certificate. '
+    'default: %(default)s',
   )
   parser.add_argument(
     '--user-ca-crt',
     default='cau.crt.pem',
     metavar='CRT_PATH',
-    help='Users CA certificate file location. default: %(default)s',
+    help='Users CA certificate file location. '
+    'May be an existing directory or file, or non-existing. '
+    'If non-existing and given path has an extension, a file will be created, '
+    'otherwise a directory will be. '
+    'When it is a file, it may contain multiple PEM-encoded concatenated '
+    'certificates. When it is a directory, it may contain multiple files, '
+    'each containing a single PEM-encoded certificate. '
+    'default: %(default)s',
   )
   parser.add_argument(
     '--crl',
@@ -707,8 +721,14 @@ def updater(argv=None, until=utils.until):
     required=True,
     metavar='CRT_PATH',
     help='Service CA certificate file location used to verify connection '
-    'to caucase. Will be maintained up-to-date. '
-    'default: %(default)s',
+    'to caucase. '
+    'May be an existing directory or file, or non-existing. '
+    'If non-existing and given path has an extension, a file will be created, '
+    'otherwise a directory will be. '
+    'When it is a file, it may contain multiple PEM-encoded concatenated '
+    'certificates. When it is a directory, it may contain multiple files, '
+    'each containing a single PEM-encoded certificate. '
+    'Will be maintained up-to-date.',
   )
   parser.add_argument(
     '--threshold',
@@ -783,7 +803,13 @@ def updater(argv=None, until=utils.until):
     required=True,
     metavar='CRT_PATH',
     help='Path of your CA certificate for MODE. '
-    'Will be maintained up-to-date.'
+    'May be an existing directory or file, or non-existing. '
+    'If non-existing and given path has an extension, a file will be created, '
+    'otherwise a directory will be. '
+    'When it is a file, it may contain multiple PEM-encoded concatenated '
+    'certificates. When it is a directory, it may contain multiple files, '
+    'each containing a single PEM-encoded certificate. '
+    'Will be maintained up-to-date.',
   )
   parser.add_argument(
     '--crl',

caucase/client.py

@@ -65,20 +65,24 @@ class CaucaseClient(object):
     url (str)
       URL to caucase, ending in eithr /cas or /cau.
     ca_crt_path (str)
-      Path to the CA certificate file, which may not exist.
+      Path to the CA certificate file or directory, which may not exist.
+      If it does not exist, it is created. If there is an extension, a file is
+      created, otherwise a directory is.
 
     Return whether an update happened (including whether an already-known
     certificate expired and was discarded).
     """
-    if not os.path.exists(ca_crt_path):
+    loaded_ca_pem_list = utils.getCertList(ca_crt_path)
+    if not loaded_ca_pem_list:
       ca_pem = cls(ca_url=url).getCACertificate()
-      with open(ca_crt_path, 'wb') as ca_crt_file:
-        ca_crt_file.write(ca_pem)
+      utils.saveCertList(ca_crt_path, [ca_pem])
       updated = True
+      # Note: reloading from file instead of using ca_pem, to exercise the
+      # same code path as future executions, to apply the same checks.
+      loaded_ca_pem_list = utils.getCertList(ca_crt_path)
     else:
       updated = False
     now = datetime.datetime.utcnow()
-    loaded_ca_pem_list = utils.getCertList(ca_crt_path)
     ca_pem_list = [
       x
       for x in loaded_ca_pem_list
@@ -88,9 +92,7 @@ class CaucaseClient(object):
       cls(ca_url=url, ca_crt_pem_list=ca_pem_list).getCACertificateChain(),
     )
     if ca_pem_list != loaded_ca_pem_list:
-      data = b''.join(ca_pem_list)
-      with open(ca_crt_path, 'wb') as ca_crt_file:
-        ca_crt_file.write(data)
+      utils.saveCertList(ca_crt_path, ca_pem_list)
       updated = True
     return updated
 

caucase/test.py

@@ -338,6 +338,7 @@ class CaucaseTest(unittest.TestCase):
     os.mkdir(client_dir)
     # pylint: disable=bad-whitespace
     self._client_ca_crt      = os.path.join(client_dir, 'cas.crt.pem')
+    self._client_ca_dir      = os.path.join(client_dir, 'cas_crt')
     self._client_user_ca_crt = os.path.join(client_dir, 'cau.crt.pem')
     self._client_crl         = os.path.join(client_dir, 'cas.crl.pem')
     self._client_user_crl    = os.path.join(client_dir, 'cau.crl.pem')
@@ -2809,7 +2810,7 @@ class CaucaseTest(unittest.TestCase):
           '--key', re_key_path,
           '--csr', re_csr_path,
           '--crt', re_crt_path,
-          '--ca', self._client_ca_crt,
+          '--ca', self._client_ca_dir,
           '--crl', self._client_crl,
         ),
         'until': until_updater,
@@ -2886,6 +2887,66 @@ class CaucaseTest(unittest.TestCase):
       until_updater.action = ON_EVENT_RAISE
       updater_event.set()
       updater_thread.join(2)
+    self.assertTrue(os.path.exists(self._client_ca_dir))
+    self.assertTrue(os.path.isdir(self._client_ca_dir))
+    # There was no CA renewal, so there should be a single file
+    ca_crt, = os.listdir(self._client_ca_dir)
+    with open(self._client_ca_crt, 'rb') as client_ca_file:
+      client_ca_crt = utils.load_ca_certificate(client_ca_file.read())
+    with open(os.path.join(self._client_ca_dir, ca_crt), 'rb') as ca_file:
+      ca_crt = utils.load_ca_certificate(ca_file.read())
+    self.assertEqual(client_ca_crt, ca_crt)
+
+  def testCAFilesystemStorage(self):
+    """
+    Test CA certificate storage in filesystem.
+    """
+    # Loading from non-existsent files
+    self.assertFalse(os.path.exists(self._client_ca_dir))
+    self.assertEqual(utils.getCertList(self._client_ca_dir), [])
+    self.assertFalse(os.path.exists(self._client_ca_crt))
+    self.assertEqual(utils.getCertList(self._client_ca_crt), [])
+    # Creation
+    _, crt0 = self._getCAKeyPair()
+    crt0_pem = utils.dump_certificate(crt0)
+    _, crt1 = self._getCAKeyPair()
+    crt1_pem = utils.dump_certificate(crt1)
+    utils.saveCertList(self._client_ca_dir, [crt0_pem])
+    self.assertTrue(os.path.exists(self._client_ca_dir))
+    self.assertTrue(os.path.isdir(self._client_ca_dir))
+    crt0_name, = os.listdir(self._client_ca_dir)
+    self.assertItemsEqual(utils.getCertList(self._client_ca_dir), [crt0_pem])
+    utils.saveCertList(self._client_ca_crt, [crt0_pem])
+    self.assertTrue(os.path.exists(self._client_ca_crt))
+    self.assertTrue(os.path.isfile(self._client_ca_crt))
+    self.assertItemsEqual(utils.getCertList(self._client_ca_crt), [crt0_pem])
+    # Invalid file gets deleted
+    dummy_file_path = os.path.join(self._client_ca_dir, 'not_a_pem')
+    with open(dummy_file_path, 'wb') as dummy:
+      pass
+    self.assertTrue(os.path.exists(dummy_file_path))
+    utils.saveCertList(self._client_ca_dir, [crt0_pem])
+    self.assertFalse(os.path.exists(dummy_file_path))
+    # Storing and loading multiple certificates
+    utils.saveCertList(self._client_ca_dir, [crt0_pem, crt1_pem])
+    crta, crtb = os.listdir(self._client_ca_dir)
+    crt1_name, = [x for x in (crta, crtb) if x != crt0_name]
+    self.assertItemsEqual(
+      utils.getCertList(self._client_ca_dir),
+      [crt0_pem, crt1_pem],
+    )
+    utils.saveCertList(self._client_ca_crt, [crt0_pem, crt1_pem])
+    self.assertItemsEqual(
+      utils.getCertList(self._client_ca_crt),
+      [crt0_pem, crt1_pem],
+    )
+    # Removing a previously-stored certificate
+    utils.saveCertList(self._client_ca_dir, [crt1_pem])
+    crta, = os.listdir(self._client_ca_dir)
+    self.assertEqual(crta, crt1_name)
+    self.assertItemsEqual(utils.getCertList(self._client_ca_dir), [crt1_pem])
+    utils.saveCertList(self._client_ca_crt, [crt1_pem])
+    self.assertItemsEqual(utils.getCertList(self._client_ca_crt), [crt1_pem])
 
   def testHttpSSLRenewal(self):
     """

caucase/utils.py

@@ -126,11 +126,89 @@ def getCertList(crt_path):
   Return a list of certificates.
   Raises if there is anything else than a certificate.
   """
-  type_dict = _getPEMTypeDict(crt_path)
-  crt_list = type_dict.pop(pem.Certificate)
-  if type_dict:
-    raise ValueError('%s contains more than just certificates' % (crt_path, ))
-  return [x.as_bytes() for x in crt_list]
+  if not os.path.exists(crt_path):
+    return []
+  if os.path.isdir(crt_path):
+    file_list = [os.path.join(crt_path, x) for x in os.listdir(crt_path)]
+  else:
+    file_list = [crt_path]
+  result = []
+  for file_name in file_list:
+    type_dict = _getPEMTypeDict(file_name)
+    crt_list = type_dict.pop(pem.Certificate)
+    if type_dict:
+      raise ValueError('%s contains more than just certificates' % (file_name, ))
+    result.extend(x.as_bytes() for x in crt_list)
+  return result
+
+def saveCertList(crt_path, cert_pem_list):
+  """
+  Store given list of PEm-encoded certificates in given path.
+
+  crt_path (str)
+  May point to a directory a file, or nothing.
+  If it does not exist, and this value contains an extension, a file is
+  created, otherwise a directory is.
+  If it is a file, all certificates are written in it.
+  If it is a folder, each certificate is stored in a separate file.
+
+  cert_pem_list (list of bytes)
+  """
+  if os.path.exists(crt_path):
+    if os.path.isfile(crt_path):
+      saveCertListTo = _saveCertListToFile
+    elif os.path.isdir(crt_path):
+      saveCertListTo = _saveCertListToDirectory
+    else:
+      raise TypeError('%s exist and is neither a directory nor a file' % (
+        crt_path,
+      ))
+  else:
+    saveCertListTo = (
+      _saveCertListToFile
+      if os.path.splitext(crt_path)[1] else
+      _saveCertListToDirectory
+    )
+  saveCertListTo(crt_path, cert_pem_list)
+
+def _saveCertListToFile(ca_crt_path, cert_pem_list):
+  with open(ca_crt_path, 'wb') as ca_crt_file:
+    ca_crt_file.write(b''.join(cert_pem_list))
+
+def _saveCertListToDirectory(crt_dir, cert_pem_list):
+  if not os.path.exists(crt_dir):
+    os.mkdir(crt_dir)
+  ca_cert_dict = {
+    '%x.pem' % (load_ca_certificate(x).serial_number, ): x
+    for x in cert_pem_list
+  }
+  for cert_filename in os.listdir(crt_dir):
+    ca_crt_path = os.path.join(crt_dir, cert_filename)
+    if not os.path.isfile(ca_crt_path):
+      # Not a file and not a symlink to a file, ignore
+      continue
+    if not os.path.islink(ca_crt_path) and cert_filename in ca_cert_dict:
+      try:
+        # pylint: disable=unbalanced-tuple-unpacking
+        cert, = getCertList(ca_crt_path)
+        # pylint: enable=unbalanced-tuple-unpacking
+      # pylint: disable=broad-except
+      except Exception:
+      # pylint: enable=broad-except
+        # Inconsistent content (multiple certificates, not CA certificates,
+        # ...): overwrite file
+        pass
+      else:
+        if cert == ca_cert_dict[cert_filename]:
+          # Already consistent, do not edit.
+          del ca_cert_dict[cert_filename]
+    else:
+      # Unknown file (ex: expired certificate), or a symlink to a file: delete
+      os.unlink(ca_crt_path)
+  for cert_filename, cert_pem in ca_cert_dict.items():
+    ca_crt_path = os.path.join(crt_dir, cert_filename)
+    with open(ca_crt_path, 'wb') as ca_crt_file:
+      ca_crt_file.write(cert_pem)
 
 def getCert(crt_path):
   """

shell/caucase.sh

@@ -291,6 +291,19 @@ printIfExpiresAfter () {
   printf '%s\n' "$crt" | expiresBefore "$1" || printf '%s\n' "$crt"
 }
 
+storeCertBySerial () {
+  # Store certificate in a file named after its serial, in given directory
+  # and using given printf format string.
+  # Usage: storeCertBySerial <dir> <patterm> < certificate
+  # shellcheck disable=SC2039
+  local crt
+  crt="$(cat)"
+  serial="$(printf "%s\n" "$crt" \
+    | openssl x509 -serial -noout | sed 's/^[^=]*=\(.*\)/\L\1/')"
+  test $? -ne 0 && return 1
+  printf "%s\n" "$crt" > "$(printf "%s/$2" "$1" "$serial")"
+}
+
 appendValidCA () {
   # TODO: test
   # Append CA to given file if it is signed by a CA we know of already.
@@ -448,15 +461,46 @@ updateCACertificate () {
     ca="$2" \
     future_ca \
     status \
-    orig_ca \
+    orig_ca="" \
+    ca_is_file \
+    ca_file \
     valid_ca
-  orig_ca="$(
-    if [ -e "$ca" ]; then
-      cat "$ca"
+  if [ -e "$ca" ]; then
+    if [ -f "$ca" ]; then
+      ca_is_file=1
+      orig_ca="$(cat "$ca")"
+    elif [ -d "$ca" ]; then
+      ca_is_file=0
     else
-      _curlInsecure "$url/crt/ca.crt.pem"
+      printf "%s exists and is neither a directory nor a file\n" "$ca"
+      return 1
     fi
-  )"
+  else
+    case "$ca" in
+      *.*)
+        ca_is_file=1
+        ;;
+      *)
+        mkdir "$ca"
+        ca_is_file=0
+        ;;
+    esac
+  fi
+  if [ $ca_is_file -eq 0 ]; then
+    for ca_file in "$ca"/*; do
+      # double use:
+      # - skips non-files
+      # - skips the one iteration when there is nothing in "$ca"/
+      if [ -f "$ca_file" ] && [ ! -h "$ca_file" ]; then
+        orig_ca="$( \
+          printf "%s\n%s" "$orig_ca" "$(cat "$ca_file")" \
+        )"
+      fi
+    done
+  fi
+  if [ -z "$orig_ca" ]; then
+    orig_ca="$(_curlInsecure "$url/crt/ca.crt.pem")"
+  fi
   status=$?
   test $status -ne 0 && return 1
   valid_ca="$(
@@ -465,7 +509,18 @@ updateCACertificate () {
   )"
   status=$?
   test $status -ne 0 && return 1
-  printf '%s\n' "$valid_ca" > "$ca"
+  if [ $ca_is_file -eq 1 ]; then
+    printf '%s\n' "$valid_ca" > "$ca"
+  else
+    for ca_file in "$ca"/*; do
+      test -f "$ca_file" && rm "$ca_file"
+    done
+    printf '%s\n' "$valid_ca" \
+    | forEachCertificate storeCertBySerial "$ca" "%s.pem"
+    # other commands (openssl crl, curl) may need openssl-style subject hash
+    # symlinks, so create them.
+    openssl rehash "$ca" > /dev/null
+  fi
   if [ ! -r "$cas_ca" ]; then
     # Should never be reached, as this function should be run once with
     # cas_ca == ca (to update CAS' CA), in which case cas_ca exists by this
@@ -482,7 +537,13 @@ updateCACertificate () {
 
 getCertificateRevocationList () {
   # Usage: <url> <ca>
-  _curlInsecure "$1/crl" | openssl crl -CAfile "$2" 2> /dev/null
+  _curlInsecure "$1/crl" | openssl crl "$(
+    if [ -d "$2" ]; then
+      printf -- '-CApath'
+    else
+      printf -- '-CAfile'
+    fi
+  )" "$2" 2> /dev/null
 }
 
 getCertificateSigningRequest () {
@@ -561,10 +622,13 @@ General options
   Ex: http://caucase.example.com:8000
 --ca-crt PATH
   Default: cas.crt.pem
-  Path of the service CA certificate file. Updated on each run.
+  Path of the service CA certificate file or directory. Updated on each run.
+  If nothing with that name exists, a directory is created if given name does
+  not contain a dot, and a file otherwise.
 --user-ca-crt PATH
   Default: cau.crt.pem
-  Path of the user CA certificate file. See --update-user .
+  Path of the user CA certificate file or directory. See --update-user and
+  --ca-crt.
 --crl PATH
   Default: cas.crl.pem
   Path of the service revocation list. Updated on each run.
@@ -1116,6 +1180,8 @@ EOF
   _test() {
     # shellcheck disable=SC2039
     local netloc="$1" \
+      cas_file \
+      cas_found=0 \
       csr_id \
       status \
       tmp_dir \
@@ -1205,14 +1271,30 @@ EOF
       -out user_csr.pem 2> /dev/null
     echo 'Bootstraping trust and submitting csr for a user certificate...'
     csr_id="$(_main \
+      --ca-crt "cas_crt" \
       --ca-url "http://$netloc" \
       --update-user \
       --mode user \
       --send-csr user_csr.pem \
       | sed 's/\s.*//' \
     )"
-    if [ ! -f cas.crt.pem ]; then
-      echo 'cas.crt.pem not created'
+    if [ ! -d cas_crt ]; then
+      echo 'cas_crt not created'
+      find . -ls
+      return 1
+    fi
+    for cas_file in cas_crt/*; do
+      if [ -r "$cas_file" ] && [ ! -h "$cas_file" ]; then
+        if [ "$cas_found" -eq 1 ]; then
+          echo 'Multiple CAS CA certificates found'
+          find . -ls
+          return 1
+        fi
+        cas_found=1
+      fi
+    done
+    if [ "$cas_found" -eq 0 ]; then
+      echo 'No CAS CA certificates found, but directory exists'
       find . -ls
       return 1
     fi
@@ -1223,6 +1305,7 @@ EOF
     fi
     echo 'Retrieving auto-issued user certificate...'
     if _main \
+      --ca-crt "cas_crt" \
       --ca-url "http://$netloc" \
       --mode user \
       --get-crt "$csr_id" user_crt.pem
@@ -1235,6 +1318,7 @@ EOF
     fi
     echo 'Using the user certificate...'
     if _main \
+      --ca-crt "cas_crt" \
       --ca-url "http://$netloc" \
       --user-key user_crt.pem \
       --list-csr \