Extend code to print more information and check if processes are started

slapos/resiliencytest/suites/erp5.py

@@ -64,10 +64,37 @@ class ERP5TestSuite(SlaprunnerTestSuite):
     data = self._connectToSlaprunner(
         resource='getConnectionParameter/slappart8'
     )
-    url = json.loads(json.loads(data)['_'])['default-v6']
+    data_json = json.loads(data)
+    url = json.loads(data_json['_'])['default-v6']
     self.logger.info('Retrieved erp5 url is:\n%s' % url)
     return url
 
+  def _checkApacheBackend(self):
+    data = self._connectToSlaprunner(
+        resource='getConnectionParameter/slappart8'
+    )
+    data_json = json.loads(data)
+    url = json.loads(data_json['_'])['default']
+    self.logger.info('Retrieved apache url is:\n%s' % url)
+    try:
+      response = self._connectToERP5(url)
+      self.logger.info(response)
+    except:
+      self.logger.info("Failed to connect to %s" % url)
+
+  def _checkERP5Backend(self):
+    data = self._connectToSlaprunner(
+        resource='getConnectionParameter/slappart7'
+    )
+    data_json = json.loads(data)
+    url = "http://%s" % json.loads(data_json['_'])["zope-address-list"][0][0]
+    self.logger.info('Retrieved erp5 url is:\n%s' % url)
+    try:
+      response = self._connectToERP5(url)
+      self.logger.info(response)
+    except:
+      self.logger.info("Failed to connect to %s" % url)
+
   def _getERP5Password(self):
     data = self._connectToSlaprunner(
         resource='getConnectionParameter/slappart0'
@@ -130,8 +157,9 @@ class ERP5TestSuite(SlaprunnerTestSuite):
     else:
       result = opener_director.open(url)
 
-    if result.getcode() is not 200:
-      raise NotHttpOkException(result.getcode())
+    code = result.getcode()
+    if code is not 200:
+      raise NotHttpOkException(code)
     return result.read()
 
   def _createRandomERP5Document(self, password=None):
@@ -144,7 +172,8 @@ class ERP5TestSuite(SlaprunnerTestSuite):
     erp5_site_title = self.slaprunner_user
     url = "%s/erp5?__ac_name=zope&__ac_password=%s" % (self._getERP5Url(), password)
     form = 'title%%3AUTF-8:string=%s&manage_editProperties%%3Amethod=Save+Changes' % erp5_site_title
-    self._connectToERP5(url, form)
+    response = self._connectToERP5(url, form)
+    self.logger.info("Connection response %s" % response)
     return erp5_site_title
 
   def generateData(self):
@@ -190,6 +219,9 @@ class ERP5TestSuite(SlaprunnerTestSuite):
     self.logger.info('Starting all partitions ...')
     self._connectToSlaprunner('/startAllPartition')
 
+    self.logger.info("Supervisord Status")
+    self.logger.info(json.loads(self._connectToSlaprunner('/supervisordStatus')))
+
     self.logger.info('Waiting 30 seconds so that erp5 can be bootstrapped...')
     for i in range(10):
       time.sleep(30)
@@ -230,9 +262,28 @@ class ERP5TestSuite(SlaprunnerTestSuite):
     self._login()
     self._waitForSoftwareBuild()
     self._deployInstance()
+    self._deployInstance()
+    self._deployInstance()
+    self._deployInstance()
     time.sleep(60)
 
     self._editHAProxyconfiguration()
+    time.sleep(10)
+    self.logger.info('Starting all partitions again ...')
+    self._connectToSlaprunner('/startAllPartition')
+    time.sleep(60 * 2)
+
+    self.logger.info("Inspect Instance")
+    self.logger.info(self._connectToSlaprunner('/inspectInstance'))
+
+    self.logger.info("Supervisord Status Again")
+    self.logger.info(json.loads(self._connectToSlaprunner('/supervisordStatus')))
+
+    self.logger.info("Check Apache backend")
+    self._checkApacheBackend()
+    self.logger.info("Check ERP5 backend")
+    self._checkERP5Backend()
+
     new_data = self._getCreatedERP5Document()
 
     if new_data == self.data:

slapos/resiliencytest/suites/slaprunner.py

@@ -200,11 +200,11 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
     self.logger.info('Instance has been deployed.')
 
   def _gitClone(self):
-    self.logger.debug('Doing git clone of https://lab.nexedi.com/nexedi/slapos.git..')
+    self.logger.debug('Doing git clone of https://lab.nexedi.com/gabriel/slapos.git..')
     try:
       data = self._connectToSlaprunner(
           resource='cloneRepository',
-          data='repo=https://lab.nexedi.com/nexedi/slapos.git&name=workspace/slapos&email=slapos@slapos.org&user=slapos'
+          data='repo=https://lab.nexedi.com/gabriel/slapos.git&name=workspace/slapos&email=slapos@slapos.org&user=slapos'
       )
       data = json.loads(data)
       if data['code'] == 0:

slapos/runner/utils.py

@@ -300,6 +300,7 @@ def runSlapgridWithLock(config, step, process_name, lock=False):
   if step == 'instance' and not requestInstance(config):
     return 1
   try:
+    logger.debug("running runProcess %s %s" % (config, process_name))
     sup_process.runProcess(config, process_name)
     if lock:
       sup_process.waitForProcessEnd(config, process_name)
@@ -443,10 +444,20 @@ def svcStopAll(config):
 def svcStartAll(config):
   """Start all Instance processes on this computer"""
   try:
-    return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
-                  'start', 'all']).communicate()[0]
-  except:
-    pass
+    stdoutdata, stderrdata = Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
+                  'start', 'all']).communicate()
+    logger.info(("Start all processes", stdoutdata, stderrdata))
+    if "no such file" in stdoutdata or "500" in stdoutdata:
+      stdoutdata, stderrdata = Popen([config['slapos'], 'node', 'supervisord', '--cfg',
+        config['configuration_file_path']]).communicate()
+      logger.info(("Calling supervisord", stdoutdata, stderrdata))
+      stdoutdata, stderrdata = Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
+                    'start', 'all']).communicate()
+      logger.info(("Start all processes again", stdoutdata, stderrdata))
+    return stdoutdata, stderrdata
+  except Exception, e:
+    logger.info("ERROR to start all")
+    raise e
 
 def removeInstanceRootDirectory(config):
   """Clean instance directory"""
@@ -489,6 +500,7 @@ def removeCurrentInstance(config):
 
 def getSvcStatus(config):
   """Return all Softwares Instances process Information"""
+  logger.info(("configuration_file_path", config['configuration_file_path']))
   result = Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
                   'status']).communicate()[0]
   regex = "(^unix:.+\.socket)|(^error:)|(^watchdog).*$"
@@ -889,6 +901,7 @@ def buildAndRun(config):
 def runSlapgridUntilSuccess(config, step):
   """Run slapos several times,
   in the maximum of the constant MAX_RUN_~~~~"""
+  logger.debug("Running runSlapgridUntilSuccess")
   params = getBuildAndRunParams(config)
   if step == "instance":
     max_tries = (params['max_run_instance'] if params['run_instance'] else 0)