kedifa: Test stabilizing fixup, wait a bit longer on the cauase first start

kedifa/test.py

@@ -17,7 +17,7 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-from six import StringIO
+from six import StringIO, BytesIO
 import contextlib
 import datetime
 from six.moves import http_client as httplib
@@ -31,6 +31,7 @@ import requests.exceptions
 import shutil
 import signal
 import sys
+import tarfile
 import tempfile
 import time
 import unittest
@@ -60,6 +61,30 @@ def findFreeTCPPort(ip=''):
     s.bind((ip, 0))
     return str(s.getsockname()[1])
 
+def retry(callback, try_count=10, try_delay=0.1):
+    """
+    Poll <callback> every <try_delay> for <try_count> times or until it returns
+    a true value.
+    Always returns the value returned by latest callback invocation.
+    """
+    for _ in xrange(try_count):
+      result = callback()
+      if result:
+        break
+      time.sleep(try_delay)
+    return result
+
+def canConnect(caucase_url):
+  """
+  Returns True if a connection can be established to given address, False
+  otherwise.
+  """
+  try:
+    requests.get(caucase_url)
+  except BaseException:
+    return False
+  return True
+
 
 class KedifaMixin(object):
   def setUp(self):
@@ -70,7 +95,52 @@ class KedifaMixin(object):
     self.addCleanup(cleanTestDir)
 
 
+_clean_caucased_snapshot = None
 class KedifaMixinCaucase(KedifaMixin):
+  def _startCaucaseServer(self, argv=(), timeout=10):
+    """
+    Start caucased server
+    """
+    ip, port = os.environ['SLAPOS_TEST_IPV6'],\
+       findFreeTCPPort(os.environ['SLAPOS_TEST_IPV6'])
+    self.caucase_runtime = caucase_runtime = multiprocessing.Process(
+      target=caucase.http.main,
+      kwargs=dict(
+        argv=[
+          '--db', self.caucase_db,
+          '--server-key', os.path.join(self.caucased, 'server.key.pem'),
+          '--netloc', '[%s]:%s' % (ip, port),
+          '--service-auto-approve-count', '50'
+        ]
+      )
+    )
+    self.caucase_runtime.start()
+    self.caucase_url = 'http://[%s]:%s' % (ip, port)
+
+    if not retry(
+      lambda: (
+        self.assertTrue(caucase_runtime.is_alive()) or
+        canConnect(self.caucase_url)
+      ),
+      try_count=timeout * 10,
+    ):
+      self._stopCaucaseServer()
+      raise AssertionError('Could not connect to %r after %i seconds' % (
+        self.caucase_url,
+        timeout,
+      ))
+
+  def _stopCaucaseServer(self):
+    """
+    Stop a running caucased server
+    """
+    caucase_runtime = self.caucase_runtime
+    caucase_runtime.terminate()
+    caucase_runtime.join()
+    if caucase_runtime.is_alive():
+      raise ValueError('%r does not wish to die' % (caucase_runtime, ))
+
+
   def createKey(self):
     key = rsa.generate_private_key(
       public_exponent=65537, key_size=2048, backend=default_backend())
@@ -136,40 +206,31 @@ class KedifaMixinCaucase(KedifaMixin):
     self.pem = certificate_pem + key_pem
 
   def setUpCaucase(self):
-    ip, port = os.environ['SLAPOS_TEST_IPV6'],\
-       findFreeTCPPort(os.environ['SLAPOS_TEST_IPV6'])
+    global _clean_caucased_snapshot # pylint: disable=global-statement
     self.caucased = os.path.join(self.testdir, 'caucased')
-    caucase_db = os.path.join(self.caucased, 'caucase.sqlite')
+    self.caucase_db = os.path.join(self.caucased, 'caucase.sqlite')
     self.caucase_service = os.path.join(self.testdir, 'service')
     os.mkdir(self.caucased)
     os.mkdir(self.caucase_service)
-    self.caucase_runtime = multiprocessing.Process(
-      target=caucase.http.main,
-      kwargs=dict(
-        argv=[
-          '--db', caucase_db,
-          '--server-key', os.path.join(self.caucased, 'server.key.pem'),
-          '--netloc', '[%s]:%s' % (ip, port),
-          '--service-auto-approve-count', '50'
-        ]
-      )
-    )
-    self.addCleanup(self.caucase_runtime.terminate)
-    self.caucase_runtime.start()
-    self.assertTrue(self.caucase_runtime.is_alive())
-    self.caucase_url = 'http://[%s]:%s' % (ip, port)
-    # give 5s for caucase to be available
-    b = time.time()
-    for i in range(50):
-      try:
-        requests.get(self.caucase_url)
-      except BaseException:
-        time.sleep(0.1)
-      else:
-        break
+
+    if _clean_caucased_snapshot is None:
+      self._startCaucaseServer(timeout=60)
+      self._stopCaucaseServer()
+      server_raw = BytesIO()
+      with tarfile.TarFile(mode='w', fileobj=server_raw) as server_tarball:
+        server_tarball.add(
+          self.caucased,
+          arcname=os.path.basename(self.caucased),
+        )
+      _clean_caucased_snapshot = server_raw.getvalue()
     else:
-      self.fail(
-        'Caucase not available after %.2fs seconds' % (time.time() - b))
+      with tarfile.TarFile(
+        mode='r',
+        fileobj=BytesIO(_clean_caucased_snapshot),
+      ) as server_tarball:
+        server_tarball.extractall(path=self.testdir)
+    self._startCaucaseServer()
+    self.addCleanup(self.caucase_runtime.terminate)
 
   def setUpKey(self, common_name):
     # create key for the service and keep its downloaded ca_crt
@@ -266,7 +327,7 @@ ez+ONyvetfvjD8cxyQ==
 
     # give 5s for KeDiFa to be available
     b = time.time()
-    for i in range(50):
+    for i in range(100):
       try:
         self.requests_get(self.kedifa_url + 'ping')
       except BaseException: