scalability: reinstall software during tests

- allow slapos master communicator to supply software destruction
- runner reinstalls software on every commit (temporarily)
- refactor in software profile construction
- some try controls added

/reviewed-on nexedi/erp5!644

erp5/util/testnode/ScalabilityTestRunner.py

@@ -133,7 +133,7 @@ class ScalabilityTestRunner():
     self.last_slapos_answer_request = []
     self.frontend_instance_guid = None
     
-  def _prepareSlapOS(self, software_path, computer_guid, create_partition=0):
+  def _prepareSlapOS(self, software_path, computer_guid, create_partition=0, state="available"):
     # create_partition is kept for compatibility
     """
     A proxy to supply : Install a software on a specific node
@@ -141,7 +141,7 @@ class ScalabilityTestRunner():
     logger.info("testnode, supply : %s %s", software_path, computer_guid)
     if self.authorize_supply :
       self.remaining_software_installation_dict[computer_guid] = software_path
-      self.slapos_communicator.supply(software_path, computer_guid)
+      self.slapos_communicator.supply(software_path, computer_guid, state)
       return {'status_code' : 0}                                          
     else:
       raise ValueError("Too late to supply now. ('self.authorize_supply' is False)")
@@ -212,7 +212,7 @@ class ScalabilityTestRunner():
     else:
       raise ValueError("Softwares release not ready yet to launch instan\
 ces or already launched.")
-      return {'status_code' : 1}  
+      return {'status_code' : 1}
 
   def prepareSlapOSForTestNode(self, test_node_slapos=None):
     """
@@ -336,12 +336,9 @@ Require valid-user
     htaccess_file.write(file_content)
     htaccess_file.close()
     password_path = testsuite_directory + PASSWORD_FILE
-    password_file = open(password_path, "r+") if os.path.isfile(password_path) else open(password_path, "w+")
-    password = password_file.read()
-    if len(password) != PASSWORD_LENGTH:
+    with open(password_path, "w") as password_file:
       password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH))
       password_file.write(password)
-    password_file.close()
     user = TESTNODE_USER
     command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password]
     self.testnode.process_manager.spawn(*command)
@@ -374,22 +371,8 @@ Require valid-user
       os.path.join("["+self.testnode.config['httpd_ip']+"]"+":"+self.testnode.config['httpd_software_access_port'],
       "software",self.randomized_path))
     self.reachable_profile = os.path.join(self.reachable_address, "software.cfg")
-    # Write the reachable address in the software.cfg file,
-    # by replacing <obfuscated_url> occurences by the current reachable address.
-    software_file = open(node_test_suite.custom_profile_path, "r")
-    file_content = software_file.readlines()
-    new_file_content = []
-    for line in file_content:
-      new_file_content.append(line.replace('<obfuscated_url>', self.reachable_address))
-    software_file.close()
-    os.remove(node_test_suite.custom_profile_path)
-    software_file = open(node_test_suite.custom_profile_path, "w")
-    for line in new_file_content:
-      software_file.write(line)
-    software_file.close()
 
   def newFrontendMasterSoftware(self, frontend_master_reference, frontend_software):
-    frontend_master_reference = frontend_master_reference.replace(";",",")
     slappart_directory = self.testnode.config['srv_directory'].rsplit("srv", 1)[0]
     software_dict_file = slappart_directory + "var/" + SR_DICT
     software_dict = self.getDictionaryFromFile(software_dict_file)
@@ -488,6 +471,7 @@ Require valid-user
 					supply, 
 					self.reachable_profile, 
 					computer_guid=self.launcher_nodes_computer_guid[0])
+
       # Ask for SR installation
       for computer_guid in self.involved_nodes_computer_guid:
         self._prepareSlapOS(self.reachable_profile, computer_guid) 
@@ -507,15 +491,20 @@ Require valid-user
       # Ask for frontend SR installation and instance request
       # XXX TODO: frontend software url will come from test_configuration soon
       self.frontend_software = FRONTEND_SOFTWARE
-      self.frontend_instance_guid = self.prepareFrontendMasterInstance(self.launcher_nodes_computer_guid[0],
-                                                                       self.frontend_software,
-                                                                       node_test_suite.test_suite_title)
+      try:
+        self.frontend_instance_guid = self.prepareFrontendMasterInstance(self.launcher_nodes_computer_guid[0],
+                                                                         self.frontend_software,
+                                                                         node_test_suite.test_suite_title)
+      except Exception as e:
+        logger.error("Error preparing frontend master instance: " + str(e))
+        return {'status_code' : 1}
       if self.remainSoftwareToInstall() :
         # All softwares are not installed, however maxtime is elapsed, that's a failure.
         logger.error("All softwares are not installed.")
         return {'status_code' : 1}
       self.authorize_request = True
       logger.debug("Softwares installed.")
+
       # Launch instance
       try:
         self._createInstance(self.reachable_profile, 
@@ -525,11 +514,10 @@ Require valid-user
                              self.frontend_software,
                              self.frontend_instance_guid)
         logger.debug("Scalability instance requested.")
-      except Exception:
-        msg = "Unable to launch instance"
-        logger.exception(msg)
-        raise ValueError(msg)
-      self._waitInstanceCreation(self.instance_title)
+        self._waitInstanceCreation(self.instance_title)
+      except Exception as e:
+        logger.error("Error creating instance: " + str(e))
+        return {'status_code' : 1}
 
       return {'status_code' : 0}
     return {'status_code' : 1}
@@ -577,9 +565,9 @@ Require valid-user
           else:
             response_dict = json.loads(result['stdout'])
             if response_dict["status_code"] != 0:
-              error_message = "Error while bootstrapping: " + response_dict["error_message"]
+              error_message = "Error while bootstrapping: " + str(response_dict["error_message"])
             bootstrap_password = response_dict["password"]
-          logger.info("Bootstrap password: " + bootstrap_password)
+          logger.info("Bootstrap password: " + str(bootstrap_password))
       except Exception as e:
         error_message = "Error while bootstrapping: " + str(e)
     return bootstrap_password, error_message
@@ -642,6 +630,7 @@ Require valid-user
     suite, repo_location = self.makeSuite(node_test_suite.test_suite, self.testnode.config['repository_path_list'])
     if suite == None:
       error_message = "Could not find test suite %s in the repositories" % node_test_suite.test_suite
+      test_result_proxy.reportFailure(stdout=error_message)
       return {'status_code' : 1, 'error_message': error_message }
 
     # Each cluster configuration is tested
@@ -649,10 +638,18 @@ Require valid-user
       # First configuration doesn't need XML configuration update.
       if count > 0:
         logger.info("Requesting instance with configuration %d" % count)
-        self._updateInstanceXML(configuration, self.instance_title,
-                                node_test_suite.test_result, node_test_suite.test_suite, self.frontend_software, self.frontend_instance_guid)
-
-      self.slapos_communicator.waitInstanceStarted(self.instance_title)
+        try:
+          self._updateInstanceXML(configuration, self.instance_title,
+                                  node_test_suite.test_result, node_test_suite.test_suite, self.frontend_software, self.frontend_instance_guid)
+        except Exception as e:
+          error_message = "Error updating instance configuration: " + str(e)
+      try:
+        self.slapos_communicator.waitInstanceStarted(self.instance_title)
+      except Exception as e:
+          error_message = "Error starting instance: " + str(e)
+      if error_message:
+        test_result_proxy.reportFailure(stdout=error_message)
+        return {'status_code' : 1, 'error_message': error_message }
       logger.debug("INSTANCE CORRECTLY STARTED")
 
       # Start only the current test
@@ -740,6 +737,12 @@ Require valid-user
         error_message = "Error in communication with master: " + str(e)
         break
 
+    # Remove installed software
+    self.authorize_supply = True
+    logger.info("Removing installed software")
+    for computer_guid in self.involved_nodes_computer_guid:
+      self._prepareSlapOS(self.reachable_profile, computer_guid, state="destroyed")
+
     # If error appears then that's a test failure.    
     if error_message:
       logger.info("There was an error during the testsuite run: " + str(error_message))

erp5/util/testnode/SlapOSMasterCommunicator.py

@@ -81,12 +81,12 @@ class SlapOSMasterCommunicator(object):
     self.url = url  
 
   @retryOnNetworkFailure
-  def _supply(self):
+  def _supply(self, state="available"):
     if self.computer_guid is None:
       logger.info('Nothing to supply for %s.', self.name)
       return None
     logger.info('Supply %s@%s', self.url, self.computer_guid)
-    return self.slap_supply.supply(self.url, self.computer_guid)
+    return self.slap_supply.supply(self.url, self.computer_guid, state)
 
   @retryOnNetworkFailure
   def _request(self, state, instance_title=None, request_kw=None, shared=False, software_type="RootSoftwareInstance"):
@@ -401,12 +401,12 @@ class SlapOSTester(SlapOSMasterCommunicator):
     info += "Instance Requested (Parameters): %s\n" % self.request_kw
     return info
 
-  def supply(self, software_path=None, computer_guid=None):
+  def supply(self, software_path=None, computer_guid=None, state="available"):
     if software_path is not None:
       self.url = software_path
     if computer_guid is not None:
       self.computer_guid = computer_guid
-    self._supply()
+    self._supply(state)
 
   def requestInstanceStart(self, instance_title=None, request_kw=None, shared=False, software_type="RootSoftwareInstance"):
     self.instance = self._request(INSTANCE_STATE_STARTED, instance_title, request_kw, shared, software_type)
@@ -494,6 +494,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
           self._request(INSTANCE_STATE_DESTROYED, instance["title"])
         else:
           root_instance = instance
+      logger.info("Going to destroy root partition: " + str(instance_title))
       self._request(INSTANCE_STATE_DESTROYED, root_instance["title"])
     else:
       logger.info("Instance not found")