caucase.test: Reuse caucased datadir when running multiple tests.

caucase/test.py

@@ -46,6 +46,7 @@ import sqlite3
 import ssl
 import subprocess
 import sys
+import tarfile
 import tempfile
 import threading
 import time
@@ -317,6 +318,7 @@ def print_buffer_on_error(func):
       raise
   return wrapper
 
+_clean_caucased_snapshot = None
 class CaucaseTest(unittest.TestCase):
   """
   Test a complete caucase setup: spawn a caucase-http server on CAUCASE_NETLOC
@@ -329,6 +331,7 @@ class CaucaseTest(unittest.TestCase):
     Prepare test data directory and file paths, and start caucased as most
     tests will need to interact with it.
     """
+    global _clean_caucased_snapshot
     self._data_dir = data_dir = tempfile.mkdtemp(prefix='caucase_test_')
 
     self._client_dir = client_dir = os.path.join(data_dir, 'client')
@@ -366,6 +369,30 @@ class CaucaseTest(unittest.TestCase):
       'Something else is already listening on %r, define CAUCASE_NETLOC '
       'environment variable with a different ip/port' % (netloc, ),
     )
+    # Starting caucased the first time is expensive: it needs to create at
+    # least 4 asymetric key pairs (CAU, CAS, HTTP CA, HTTP cert). This can
+    # take over 40 seconds on a Raspberry Pi 1 (which is the low-end platform
+    # which should reliably pass these tests).
+    # So backup the database after the first start, and restore it on further
+    # starts. Save it as an in-memory uncompressed tarball (should be under a
+    # megabyte).
+    # This is intentionally independent from caucased own backup mechanism.
+    if _clean_caucased_snapshot is None:
+      self._startServer(timeout=60)
+      self._stopServer()
+      server_raw = BytesIO()
+      with tarfile.TarFile(mode='w', fileobj=server_raw) as server_tarball:
+        server_tarball.add(
+          self._server_dir,
+          arcname=os.path.basename(self._server_dir),
+        )
+      _clean_caucased_snapshot = server_raw.getvalue()
+    else:
+      with tarfile.TarFile(
+        mode='r',
+        fileobj=BytesIO(_clean_caucased_snapshot),
+      ) as server_tarball:
+        server_tarball.extractall(path=self._data_dir)
     self._startServer()
 
   @print_buffer_on_error
@@ -556,7 +583,7 @@ class CaucaseTest(unittest.TestCase):
       return 1
     return 0
 
-  def _startServer(self, *argv):
+  def _startServer(self, argv=(), timeout=10):
     """
     Start caucased server
     """
@@ -585,10 +612,12 @@ class CaucaseTest(unittest.TestCase):
         self.assertTrue(server.is_alive(), 'caucased crashed on startup') or
         canConnect((parsed_url.hostname, parsed_url.port))
       ),
+      try_count=timeout * 10,
     ): # pragma: no cover
       self._stopServer()
-      raise AssertionError('Could not connect to %r after 1 second' % (
+      raise AssertionError('Could not connect to %r after %i seconds' % (
         self._caucase_url,
+        timeout,
       ))
 
   def _stopServer(self):
@@ -1189,9 +1218,9 @@ class CaucaseTest(unittest.TestCase):
 
     user_key_path = self._createFirstUser()
     self._stopServer()
-    self._startServer(
+    self._startServer((
       '--service-max-csr', '5',
-    )
+    ))
     assertCanSend(5)
     # But resubmitting one of the accepted ones is still fine
     _, csr_path = csr_list[0]
@@ -1219,22 +1248,22 @@ class CaucaseTest(unittest.TestCase):
     Verify that auto-approve limit freezing works.
     """
     self._stopServer()
-    self._startServer(
+    self._startServer((
       '--user-auto-approve-count', '2',
       '--lock-auto-approve-count',
-    )
+    ))
     self._stopServer()
-    self._startServer(
+    self._startServer((
       '--user-auto-approve-count', '3',
-    )
+    ))
     self._createFirstUser()
     self._createFirstUser()
     self.assertRaises(TypeError, self._createFirstUser)
     self._stopServer()
-    self._startServer(
+    self._startServer((
       '--user-auto-approve-count', '3',
       '--lock-auto-approve-count',
-    )
+    ))
     self.assertRaises(TypeError, self._createFirstUser)
 
   def testCSRFiltering(self):
@@ -2249,7 +2278,7 @@ class CaucaseTest(unittest.TestCase):
     backup_glob = os.path.join(self._server_backup_path, '*.sql.caucased')
 
     self._stopServer()
-    self._startServer('--backup-directory', self._server_backup_path)
+    self._startServer(('--backup-directory', self._server_backup_path))
     # XXX: waiting for nothing to happen
     time.sleep(1)
     # No backup must have been created, as no user exists.
@@ -2276,7 +2305,7 @@ class CaucaseTest(unittest.TestCase):
       self._server_db,
       table_prefix='cau',
     ).dumpIterator())
-    self._startServer('--backup-directory', self._server_backup_path)
+    self._startServer(('--backup-directory', self._server_backup_path))
     backup_path_list = retry(lambda: glob.glob(backup_glob))
     if not backup_path_list: # pragma: no cover
       raise AssertionError('Backup file not created after 1 second')
@@ -2391,7 +2420,7 @@ class CaucaseTest(unittest.TestCase):
     )
     try:
       caucase.ca.DEFAULT_BACKUP_SYMETRIC_CIPHER = 'TESTS_INTERNAL_USE_ONLY'
-      self._startServer('--backup-directory', self._server_backup_path)
+      self._startServer(('--backup-directory', self._server_backup_path))
     finally:
       caucase.ca.DEFAULT_BACKUP_SYMETRIC_CIPHER = (
         DEFAULT_BACKUP_SYMETRIC_CIPHER_orig
@@ -2410,7 +2439,7 @@ class CaucaseTest(unittest.TestCase):
     self._stopServer()
     for backup_path in glob.glob(backup_glob):
       os.unlink(backup_path)
-    self._startServer('--backup-directory', self._server_backup_path)
+    self._startServer(('--backup-directory', self._server_backup_path))
     backup_path_list = retry(lambda: glob.glob(backup_glob))
     if not backup_path_list: # pragma: no cover
       raise AssertionError('Backup file not created after 1 second')
@@ -2445,7 +2474,7 @@ class CaucaseTest(unittest.TestCase):
         c.execute(bloat_query, bloat_value)
     db.close()
     del db
-    self._startServer('--backup-directory', self._server_backup_path)
+    self._startServer(('--backup-directory', self._server_backup_path))
     backup_path_list = retry(lambda: glob.glob(backup_glob), try_count=300)
     if not backup_path_list: # pragma: no cover
       raise AssertionError('Backup file took too long to be created')