Skip to content

erp5
erp5
Commit 6d563d4f authored by Roque's avatar Roque Committed by Klaus Wölfel
Browse files
Options

testnode: refactoring in SlapOS communicator and Scalability runner 

- method waitInstance moved from runner to communicator
- new simple methods in communicator for request and wait instance
- new methods to modularize runner code
- communicator initialization code moved in runner
parent 11a3bac1
Hide whitespace changes
Inline Side-by-side
Showing with 267 additions and 223 deletions
erp5/util/testnode/ScalabilityTestRunner.py
View file @ 6d563d4f
...@@ -68,7 +68,6 @@ class ScalabilityTestRunner(): ...@@ -68,7 +68,6 @@ class ScalabilityTestRunner():
# Create the slapos account configuration file and dir # Create the slapos account configuration file and dir
key = self.testnode.test_suite_portal.getSlaposAccountKey() key = self.testnode.test_suite_portal.getSlaposAccountKey()
certificate = self.testnode.test_suite_portal.getSlaposAccountCertificate() certificate = self.testnode.test_suite_portal.getSlaposAccountCertificate()
# Get Slapos Master Url # Get Slapos Master Url
self.slapos_url = '' self.slapos_url = ''
try: try:
...@@ -106,7 +105,7 @@ class ScalabilityTestRunner(): ...@@ -106,7 +105,7 @@ class ScalabilityTestRunner():
self.log("testnode, supply : %s %s", software_path, computer_guid) self.log("testnode, supply : %s %s", software_path, computer_guid)
if self.authorize_supply : if self.authorize_supply :
self.remaining_software_installation_dict[computer_guid] = software_path self.remaining_software_installation_dict[computer_guid] = software_path
self.slapos_communicator._supply() self.slapos_communicator.supply(software_path, computer_guid)
return {'status_code' : 0} return {'status_code' : 0}
else: else:
raise ValueError("Too late to supply now. ('self.authorize_supply' is False)") raise ValueError("Too late to supply now. ('self.authorize_supply' is False)")
...@@ -150,7 +149,7 @@ class ScalabilityTestRunner(): ...@@ -150,7 +149,7 @@ class ScalabilityTestRunner():
config = self._generateInstanceXML(software_configuration, config = self._generateInstanceXML(software_configuration,
test_result, test_suite) test_result, test_suite)
request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} } request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} }
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED, instance_title, request_kw) self.slapos_communicator.requestStart(instance_title, request_kw)
self.authorize_request = False self.authorize_request = False
return {'status_code' : 0} return {'status_code' : 0}
else: else:
...@@ -193,7 +192,7 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -193,7 +192,7 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
""" """
self.log("Current software state: " + str(self.slapos_communicator._getSoftwareState())) self.log("Current software state: " + str(self.slapos_communicator._getSoftwareState()))
return self.slapos_communicator._getSoftwareState() == SlapOSMasterCommunicator.SOFTWARE_STATE_INSTALLED return self.slapos_communicator._getSoftwareState() == SlapOSMasterCommunicator.SOFTWARE_STATE_INSTALLED
def remainSoftwareToInstall(self): def remainSoftwareToInstall(self):
""" """
Return True if it remains softwares to install, otherwise return False Return True if it remains softwares to install, otherwise return False
...@@ -211,29 +210,9 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -211,29 +210,9 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
config = self._generateInstanceXML(software_configuration, config = self._generateInstanceXML(software_configuration,
test_result, test_suite) test_result, test_suite)
request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} } request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} }
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED, instance_title, request_kw) self.slapos_communicator.requestStart(instance_title, request_kw)
return {'status_code' : 0} return {'status_code' : 0}
def _waitInstance(self, instance_title, state, max_time=MAX_INSTANCE_TIME):
"""
Wait for 'max_time' an instance specific state
"""
self.log("Waiting for instance state: %s" %state)
start_time = time.time()
while (not self.slapos_communicator._getInstanceState() == state
and (max_time > (time.time()-start_time))):
self.log("Instance(s) not in %s state yet." % state)
time.sleep(15)
if (time.time()-start_time) > max_time:
error_message = "Instance '%s' not '%s' after %s seconds" %(instance_title, state, str(time.time()-start_time))
self.log(error_message)
self.log("Do you use instance state propagation in your project?")
self.log("Instance '%s' will be stopped and test aborted." %instance_title)
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
time.sleep(60)
raise ValueError(error_message)
self.log("Instance correctly '%s' after %s seconds." %(state, str(time.time()-start_time)))
def _waitInstanceCreation(self, instance_title, max_time=MAX_CREATION_INSTANCE_TIME): def _waitInstanceCreation(self, instance_title, max_time=MAX_CREATION_INSTANCE_TIME):
""" """
Wait for 'max_time' the instance creation Wait for 'max_time' the instance creation
...@@ -248,16 +227,10 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -248,16 +227,10 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
raise ValueError("Instance '%s' not found after %s seconds" %(instance_title, max_time)) raise ValueError("Instance '%s' not found after %s seconds" %(instance_title, max_time))
self.log("Instance found on slapOSMaster") self.log("Instance found on slapOSMaster")
def prepareSlapOSForTestSuite(self, node_test_suite): def _initializeSlapOSConnection(self):
""" """
Install testsuite softwares Initialize communication with slapos
""" """
self.log('Preparing SlapOS for Test Suite...')
max_time = MAX_PREPARE_TEST_SUITE
interval_time = 60
start_time = time.time()
# Initialize communication with slapos
slap = slapos.slap.slap() slap = slapos.slap.slap()
retry = 0 retry = 0
while True: while True:
...@@ -279,13 +252,65 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -279,13 +252,65 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
if getattr(slap, '_hateoas_navigator', None) is None: if getattr(slap, '_hateoas_navigator', None) is None:
raise ValueError("Fail to load _hateoas_navigator") raise ValueError("Fail to load _hateoas_navigator")
hateoas = getattr(slap, '_hateoas_navigator', None)
supply = slap.registerSupply() supply = slap.registerSupply()
order = slap.registerOpenOrder() order = slap.registerOpenOrder()
return slap, supply, order
def createSoftwareReachableProfilePath(self, node_test_suite):
# Create an obfuscated link to the testsuite directory
path_to_suite = os.path.join(
self.testnode.config['working_directory'],
node_test_suite.reference)
self.obfuscated_link_path = os.path.join(
self.testnode.config['software_directory'],
self.randomized_path)
if ( not os.path.lexists(self.obfuscated_link_path) and
not os.path.exists(self.obfuscated_link_path) ) :
try :
os.symlink(path_to_suite, self.obfuscated_link_path)
self.log("testnode, Symbolic link (%s->%s) created."
%(self.obfuscated_link_path, path_to_suite))
except :
self.log("testnode, Unable to create symbolic link to the testsuite.")
raise ValueError("testnode, Unable to create symbolic link to the testsuite.")
self.log("Sym link : %s %s" %(path_to_suite, self.obfuscated_link_path))
# Construct the ipv6 obfuscated url of the software profile reachable from outside
self.reachable_address = os.path.join(
"https://","["+self.testnode.config['httpd_ip']+"]"+":"+self.testnode.config['httpd_software_access_port'],
self.randomized_path)
self.reachable_profile = os.path.join(self.reachable_address, "software.cfg")
# ROQUE XXX: below urls are hardcoded until we find ways to make buildout extending through unsafe https
self.reachable_address = "https://lab.nexedi.com/rporchetto/telecom/tree/master"
self.reachable_profile = "https://lab.nexedi.com/rporchetto/telecom/raw/master/software_release/software.cfg"
# Write the reachable address in the software.cfg file,
# by replacing <obfuscated_url> occurences by the current reachable address.
software_file = open(node_test_suite.custom_profile_path, "r")
file_content = software_file.readlines()
new_file_content = []
for line in file_content:
new_file_content.append(line.replace('<obfuscated_url>', self.reachable_address))
software_file.close()
os.remove(node_test_suite.custom_profile_path)
software_file = open(node_test_suite.custom_profile_path, "w")
for line in new_file_content:
software_file.write(line)
software_file.close()
def prepareSlapOSForTestSuite(self, node_test_suite):
"""
Install testsuite softwares
"""
self.log('Preparing SlapOS for Test Suite...')
max_time = MAX_PREPARE_TEST_SUITE
interval_time = 60
start_time = time.time()
# Initialize communication with slapos
slap, supply, order = self._initializeSlapOSConnection()
# Only master testnode must order software installation # Only master testnode must order software installation
if self.testnode.test_suite_portal.isMasterTestnode( if self.testnode.test_suite_portal.isMasterTestnode(self.testnode.config['test_node_title']):
self.testnode.config['test_node_title']):
# Get from ERP5 Master the configuration of the cluster for the test # Get from ERP5 Master the configuration of the cluster for the test
test_configuration = Utils.deunicodeData( test_configuration = Utils.deunicodeData(
json.loads(self.testnode.test_suite_portal.generateConfiguration( json.loads(self.testnode.test_suite_portal.generateConfiguration(
...@@ -300,64 +325,26 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -300,64 +325,26 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
self.log("ERP5 Master indicates : %s" %(self.error_message,)) self.log("ERP5 Master indicates : %s" %(self.error_message,))
return {'status_code' : 1} return {'status_code' : 1}
involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid']
configuration_list = test_configuration['configuration_list'] configuration_list = test_configuration['configuration_list']
node_test_suite.edit(configuration_list=configuration_list) node_test_suite.edit(configuration_list=configuration_list)
self.launcher_nodes_computer_guid = test_configuration['launcher_nodes_computer_guid'] self.launcher_nodes_computer_guid = test_configuration['launcher_nodes_computer_guid']
self.instance_title = self._generateInstanceTitle(node_test_suite.test_suite_title)
# Create an obfuscated link to the testsuite directory self.createSoftwareReachableProfilePath(node_test_suite)
path_to_suite = os.path.join( self.log("Software reachable profile path is : %s " %(self.reachable_profile))
self.testnode.config['working_directory'],
node_test_suite.reference)
self.obfuscated_link_path = os.path.join(
self.testnode.config['software_directory'],
self.randomized_path)
if ( not os.path.lexists(self.obfuscated_link_path) and
not os.path.exists(self.obfuscated_link_path) ) :
try :
os.symlink(path_to_suite, self.obfuscated_link_path)
self.log("testnode, Symbolic link (%s->%s) created."
%(self.obfuscated_link_path, path_to_suite))
except :
self.log("testnode, Unable to create symbolic link to the testsuite.")
raise ValueError("testnode, Unable to create symbolic link to the testsuite.")
self.log("Sym link : %s %s" %(path_to_suite, self.obfuscated_link_path))
# Construct the ipv6 obfuscated url of the software profile reachable from outside
self.reachable_address = os.path.join(
"https://","["+self.testnode.config['httpd_ip']+"]"+":"+self.testnode.config['httpd_software_access_port'],
self.randomized_path)
self.reachable_profile = os.path.join(self.reachable_address, "software.cfg")
# ROQUE XXX: below urls are hardcoded until we find ways to make buildout extending through unsafe https
self.reachable_address = "https://lab.nexedi.com/rporchetto/telecom/tree/master"
self.reachable_profile = "https://lab.nexedi.com/rporchetto/telecom/raw/master/software_release/software.cfg"
# Write the reachable address in the software.cfg file,
# by replacing <obfuscated_url> occurences by the current reachable address.
software_file = open(node_test_suite.custom_profile_path, "r")
file_content = software_file.readlines()
new_file_content = []
for line in file_content:
new_file_content.append(line.replace('<obfuscated_url>', self.reachable_address))
software_file.close()
os.remove(node_test_suite.custom_profile_path)
software_file = open(node_test_suite.custom_profile_path, "w")
for line in new_file_content:
software_file.write(line)
software_file.close()
self.log("Software reachable profile path is : %s "
%(self.reachable_profile,))
# Ask for SR installation # Initialize SlapOS Master Communicator
for computer_guid in self.involved_nodes_computer_guid: self.slapos_communicator = SlapOSMasterCommunicator.SoftwareReleaseTester(
self.slapos_communicator = SlapOSMasterCommunicator.SoftwareReleaseTester( self.instance_title,
"SlaposMasterCommunicator",
self.log, self.log,
slap, slap,
order, order,
supply, supply,
self.reachable_profile, self.reachable_profile,
computer_guid=computer_guid) computer_guid=self.launcher_nodes_computer_guid[0])
# Ask for SR installation
for computer_guid in self.involved_nodes_computer_guid:
self._prepareSlapOS(self.reachable_profile, computer_guid) self._prepareSlapOS(self.reachable_profile, computer_guid)
# From the line below we would not supply any more softwares # From the line below we would not supply any more softwares
self.authorize_supply = False self.authorize_supply = False
...@@ -378,13 +365,12 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -378,13 +365,12 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
self.authorize_request = True self.authorize_request = True
self.log("Softwares installed.") self.log("Softwares installed.")
# Launch instance # Launch instance
self.instance_title = self._generateInstanceTitle(node_test_suite.test_suite_title)
try: try:
self._createInstance(self.reachable_profile, self._createInstance(self.reachable_profile,
configuration_list[0], configuration_list[0],
self.instance_title, self.instance_title,
node_test_suite.test_result, node_test_suite.test_result,
node_test_suite.test_suite) node_test_suite.test_suite)
self.log("Scalability instance requested.") self.log("Scalability instance requested.")
except: except:
self.log("Unable to launch instance") self.log("Unable to launch instance")
...@@ -410,20 +396,20 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -410,20 +396,20 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
count = 0 count = 0
error_message = None error_message = None
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED) self.slapos_communicator.waitInstanceStarted(self.instance_title)
# Each cluster configuration are tested # Each cluster configuration are tested
for configuration in configuration_list: for configuration in configuration_list:
# First configuration doesn't need XML configuration update. # First configuration doesn't need XML configuration update.
if count > 0: if count > 0:
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED) self.slapos_communicator.requestStop()
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED) self.slapos_communicator.waitInstanceStopped(self.instance_title)
self._updateInstanceXML(configuration, self.instance_title, self._updateInstanceXML(configuration, self.instance_title,
node_test_suite.test_result, node_test_suite.test_suite) node_test_suite.test_result, node_test_suite.test_suite)
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED) self.slapos_communicator.waitInstanceStarted(self.instance_title)
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED) self.slapos_communicator.requestStart()
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED) self.slapos_communicator.waitInstanceStarted(self.instance_title)
self.log("[DEBUG] INSTANCE CORRECTLY STARTED") self.log("[DEBUG] INSTANCE CORRECTLY STARTED")
# ROQUE XXX : for debug # ROQUE XXX : for debug
...@@ -465,11 +451,12 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -465,11 +451,12 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
break break
# Stop current instance # Stop current instance
self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED) self.slapos_communicator.requestStop()
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED) self.slapos_communicator.waitInstanceStopped(self.instance_title)
# Delete old instances # Delete old instances
self._cleanUpOldInstance() # ROQUE: for now, the instance is not removed.
#self._cleanUpOldInstance()
# If error appears then that's a test failure. # If error appears then that's a test failure.
if error_message: if error_message:
......
erp5/util/testnode/SlapOSMasterCommunicator.py
View file @ 6d563d4f
...@@ -13,10 +13,11 @@ from slapos.slap.slap import ConnectionError ...@@ -13,10 +13,11 @@ from slapos.slap.slap import ConnectionError
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST
# max time to instance changing state: 2 hour
MAX_INSTANCE_TIME = 60*60*2
SOFTWARE_PRODUCT_NAMESPACE = "product." SOFTWARE_PRODUCT_NAMESPACE = "product."
SOFTWARE_STATE_UNKNOWN = "SOFTWARE_STATE_UNKNOWN" SOFTWARE_STATE_UNKNOWN = "SOFTWARE_STATE_UNKNOWN"
SOFTWARE_STATE_INSTALLING = "SOFTWARE_STATE_INSTALLING" SOFTWARE_STATE_INSTALLING = "SOFTWARE_STATE_INSTALLING"
SOFTWARE_STATE_INSTALLED = "SOFTWARE_STATE_INSTALLED" SOFTWARE_STATE_INSTALLED = "SOFTWARE_STATE_INSTALLED"
...@@ -28,7 +29,7 @@ INSTANCE_STATE_STARTED = "started" ...@@ -28,7 +29,7 @@ INSTANCE_STATE_STARTED = "started"
INSTANCE_STATE_STARTED_WITH_ERROR = "INSTANCE_STATE_STARTED_WITH_ERROR" INSTANCE_STATE_STARTED_WITH_ERROR = "INSTANCE_STATE_STARTED_WITH_ERROR"
INSTANCE_STATE_STOPPING = "INSTANCE_STATE_STOPPING" INSTANCE_STATE_STOPPING = "INSTANCE_STATE_STOPPING"
INSTANCE_STATE_STOPPED = "stopped" INSTANCE_STATE_STOPPED = "stopped"
INSTANCE_STATE_DESTROYING = "INSTANCE_STATE_DESTROYING" INSTANCE_STATE_DESTROYED = "destroyed"
TESTER_STATE_INITIAL = "TESTER_STATE_INITIAL" TESTER_STATE_INITIAL = "TESTER_STATE_INITIAL"
TESTER_STATE_NOTHING = "TESTER_STATE_NOTHING" TESTER_STATE_NOTHING = "TESTER_STATE_NOTHING"
...@@ -66,7 +67,7 @@ def retryOnNetworkFailure(func): ...@@ -66,7 +67,7 @@ def retryOnNetworkFailure(func):
class SlapOSMasterCommunicator(object): class SlapOSMasterCommunicator(object):
latest_state = None
def __init__(self, slap, slap_supply, slap_order, url, logger ): def __init__(self, slap, slap_supply, slap_order, url, logger ):
...@@ -91,7 +92,7 @@ class SlapOSMasterCommunicator(object): ...@@ -91,7 +92,7 @@ class SlapOSMasterCommunicator(object):
@retryOnNetworkFailure @retryOnNetworkFailure
def _supply(self): def _supply(self):
if self.computer_guid is None: if self.computer_guid is None:
self._logger ('Nothing to supply for %s.' % (self.name)) self._logger('Nothing to supply for %s.' % (self.name))
return None return None
self._logger('Supply %s@%s', self.url, self.computer_guid) self._logger('Supply %s@%s', self.url, self.computer_guid)
return self.slap_supply.supply(self.url, self.computer_guid) return self.slap_supply.supply(self.url, self.computer_guid)
...@@ -231,12 +232,139 @@ class SlapOSMasterCommunicator(object): ...@@ -231,12 +232,139 @@ class SlapOSMasterCommunicator(object):
result['_links']['action_object_slap'], 'getHateoasInformation') result['_links']['action_object_slap'], 'getHateoasInformation')
result = self.hateoas_navigator.GET(object_link) result = self.hateoas_navigator.GET(object_link)
return json.loads(result) return json.loads(result)
def _getSoftwareState(self):
if self.computer_guid is None:
return SOFTWARE_STATE_INSTALLED
message = self.getSoftwareInstallationNews()
if message.startswith("#error no data found"):
return SOFTWARE_STATE_UNKNOWN
if message.startswith('#access software release'):
return SOFTWARE_STATE_INSTALLED
if message.startswith('#error'):
return SOFTWARE_STATE_INSTALLING
return SOFTWARE_STATE_UNKNOWN
@retryOnNetworkFailure
def _getInstanceState(self):
latest_state = self.latest_state
self._logger('latest_state = %r', latest_state)
if latest_state is None:
return INSTANCE_STATE_UNKNOWN
message_list = []
try:
for instance in self.getInstanceUrlList():
news = self.getNewsFromInstance(instance["href"])
information = self.getInformationFromInstance(instance["href"])
state = INSTANCE_STATE_UNKNOWN
monitor_information_dict = {}
info_created_at = "-1"
is_slave = information['slave']
if is_slave:
if (information["connection_dict"]) > 0:
state = INSTANCE_STATE_STARTED
else:
# not slave
instance_state = news[0]
if instance_state.get('created_at', '-1') != "-1":
# the following does NOT take TZ into account
created_at = datetime.datetime.strptime(instance_state['created_at'],
'%a, %d %b %Y %H:%M:%S %Z')
gmt_now = datetime.datetime(*time.gmtime()[:6])
info_created_at = '%s (%d)' % (
instance_state['created_at'], (gmt_now - created_at).seconds)
if instance_state['text'].startswith('#access'):
state = INSTANCE_STATE_STARTED
if instance_state['text'].startswith('#access Instance correctly stopped'):
state = INSTANCE_STATE_STOPPED
if instance_state['text'].startswith('#error'):
state = INSTANCE_STATE_STARTED_WITH_ERROR
if state == INSTANCE_STATE_STARTED_WITH_ERROR:
# search for monitor url
monitor_v6_url = information["connection_dict"].get("monitor_v6_url")
try:
monitor_information_dict = self.getRSSEntryFromMonitoring(monitor_v6_url)
except Exception:
self._logger('Unable to download promises for: %s' % (instance["title"]))
self._logger(traceback.format_exc())
monitor_information_dict = {"message": "Unable to download"}
message_list.append({
'title': instance["title"],
'slave': is_slave,
'news': news[0],
'information': information,
'monitor': monitor_information_dict,
'state': state
})
except slapos.slap.ServerError:
self._logger('Got an error requesting partition for '
'its state')
return INSTANCE_STATE_UNKNOWN
except:
self._logger("ERROR getting instance state")
return INSTANCE_STATE_UNKNOWN
started = 0
stopped = 0
self.message_history.append(message_list)
for instance in message_list:
if not instance['slave'] and \
instance['state'] in (INSTANCE_STATE_UNKNOWN, INSTANCE_STATE_STARTED_WITH_ERROR):
return instance['state']
elif not instance['slave'] and instance['state'] == INSTANCE_STATE_STARTED:
started = 1
elif not instance['slave'] and instance['state'] == INSTANCE_STATE_STOPPED:
stopped = 1
if instance['slave'] and instance['state'] == INSTANCE_STATE_UNKNOWN:
return instance['state']
if started and stopped:
return INSTANCE_STATE_STOPPED
return INSTANCE_STATE_UNKNOWN
if started:
return INSTANCE_STATE_STARTED
if stopped:
return INSTANCE_STATE_STOPPED
@retryOnNetworkFailure
def _waitInstance(self, instance_title, state, max_time=MAX_INSTANCE_TIME):
"""
Wait for 'max_time' an instance specific state
"""
self._logger("Waiting for instance state: %s" %state)
start_time = time.time()
while (not self._getInstanceState() == state
and (max_time > (time.time()-start_time))):
self._logger("Instance(s) not in %s state yet." % state)
self._logger("Current state: %s" % self._getInstanceState())
time.sleep(15)
if (time.time()-start_time) > max_time:
error_message = "Instance '%s' not '%s' after %s seconds" %(instance_title, state, str(time.time()-start_time))
return {'error_message' : error_message}
self._logger("Instance correctly '%s' after %s seconds." %(state, str(time.time()-start_time)))
return {'error_message' : None}
class SoftwareReleaseTester(SlapOSMasterCommunicator): class SoftwareReleaseTester(SlapOSMasterCommunicator):
deadline = None deadline = None
latest_state = None
def __init__(self, def __init__(self,
name, name,
...@@ -288,21 +416,21 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -288,21 +416,21 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
None, None,
), ),
TESTER_STATE_SOFTWARE_INSTALLED: ( TESTER_STATE_SOFTWARE_INSTALLED: (
lambda t: t._request("started"), lambda t: t._request(INSTANCE_STATE_STARTED),
int(instance_timeout), int(instance_timeout),
TESTER_STATE_INSTANCE_STARTED, TESTER_STATE_INSTANCE_STARTED,
None, None,
INSTANCE_STATE_STARTED, INSTANCE_STATE_STARTED,
), ),
TESTER_STATE_INSTANCE_STARTED: ( TESTER_STATE_INSTANCE_STARTED: (
lambda t: t._request("destroyed"), lambda t: t._request(INSTANCE_STATE_DESTROYED),
int(1200), int(1200),
TESTER_STATE_INSTANCE_UNINSTALLED, TESTER_STATE_INSTANCE_UNINSTALLED,
None, None,
INSTANCE_STATE_STOPPED, INSTANCE_STATE_STOPPED,
), ),
TESTER_STATE_INSTANCE_UNINSTALLED: ( TESTER_STATE_INSTANCE_UNINSTALLED: (
lambda t: t._supply("destroyed"), lambda t: t._supply(INSTANCE_STATE_DESTROYED),
int(1200), int(1200),
None, None,
None, None,
...@@ -310,6 +438,46 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -310,6 +438,46 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
), ),
} }
def supply(self, software_path=None, computer_guid=None):
if software_path is not None:
self.url = software_path
if computer_guid is not None:
self.computer_guid = computer_guid
self._supply()
def requestStart(self, instance_title=None, request_kw=None):
self._request(INSTANCE_STATE_STARTED, instance_title, request_kw)
def requestStop(self, instance_title=None, request_kw=None):
self._request(INSTANCE_STATE_STOPPED, instance_title, request_kw)
def requestDestroy(self, instance_title=None, request_kw=None):
self._request(INSTANCE_STATE_DESTROYED, instance_title, request_kw)
def waitInstanceStarted(self, instance_title):
error_message = self._waitInstance(instance_title, INSTANCE_STATE_STARTED)["error_message"]
if error_message is not None:
self._logger(error_message)
self._logger("Do you use instance state propagation in your project?")
self._logger("Instance '%s' will be stopped and test aborted." %instance_title)
self.requestStop()
time.sleep(60)
raise ValueError(error_message)
def waitInstanceStopped(self, instance_title):
error_message = self._waitInstance(instance_title, INSTANCE_STATE_STOPPED)["error_message"]
if error_message is not None:
self._logger(error_message)
self._logger("Do you use instance state propagation in your project?")
raise ValueError(error_message)
def waitInstanceDestroyed(self, instance_title):
error_message = self._waitInstance(instance_title, INSTANCE_STATE_DESTROYED)["error_message"]
if error_message is not None:
self._logger(error_message)
self._logger("Do you use instance state propagation in your project?")
raise ValueError(error_message)
def __repr__(self): def __repr__(self):
deadline = self.deadline deadline = self.deadline
if deadline is not None: if deadline is not None:
...@@ -352,22 +520,6 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -352,22 +520,6 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
return summary + message return summary + message
def _getSoftwareState(self):
if self.computer_guid is None:
return SOFTWARE_STATE_INSTALLED
message = self.getSoftwareInstallationNews()
if message.startswith("#error no data found"):
return SOFTWARE_STATE_UNKNOWN
if message.startswith('#access software release'):
return SOFTWARE_STATE_INSTALLED
if message.startswith('#error'):
return SOFTWARE_STATE_INSTALLING
return SOFTWARE_STATE_UNKNOWN
@retryOnNetworkFailure @retryOnNetworkFailure
def getRSSEntryFromMonitoring(self, base_url): def getRSSEntryFromMonitoring(self, base_url):
if base_url is None: if base_url is None:
...@@ -383,110 +535,16 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -383,110 +535,16 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
return {} return {}
@retryOnNetworkFailure
def _getInstanceState(self):
latest_state = self.latest_state
self._logger('latest_state = %r', latest_state)
if latest_state is None:
return INSTANCE_STATE_UNKNOWN
message_list = []
try:
for instance in self.getInstanceUrlList():
news = self.getNewsFromInstance(instance["href"])
information = self.getInformationFromInstance(instance["href"])
state = INSTANCE_STATE_UNKNOWN
monitor_information_dict = {}
info_created_at = "-1"
is_slave = information['slave']
if is_slave:
if (information["connection_dict"]) > 0:
state = INSTANCE_STATE_STARTED
else:
# not slave
instance_state = news[0]
if instance_state.get('created_at', '-1') != "-1":
# the following does NOT take TZ into account
created_at = datetime.datetime.strptime(instance_state['created_at'],
'%a, %d %b %Y %H:%M:%S %Z')
gmt_now = datetime.datetime(*time.gmtime()[:6])
info_created_at = '%s (%d)' % (
instance_state['created_at'], (gmt_now - created_at).seconds)
if instance_state['text'].startswith('#access'):
state = INSTANCE_STATE_STARTED
if instance_state['text'].startswith('#access Instance correctly stopped'):
state = INSTANCE_STATE_STOPPED
if instance_state['text'].startswith('#error'):
state = INSTANCE_STATE_STARTED_WITH_ERROR
if state == INSTANCE_STATE_STARTED_WITH_ERROR:
# search for monitor url
monitor_v6_url = information["connection_dict"].get("monitor_v6_url")
try:
monitor_information_dict = self.getRSSEntryFromMonitoring(monitor_v6_url)
except Exception:
self._logger ('Unable to download promises for: %s' % (instance["title"]))
self._logger (traceback.format_exc())
monitor_information_dict = {"message": "Unable to download"}
message_list.append({
'title': instance["title"],
'slave': is_slave,
'news': news[0],
'information': information,
'monitor': monitor_information_dict,
'state': state
})
except slapos.slap.ServerError:
self._logger ('Got an error requesting partition for '
'its state')
return INSTANCE_STATE_UNKNOWN
except:
self._logger("ERROR getting instance state")
return INSTANCE_STATE_UNKNOWN
started = 0
stopped = 0
self.message_history.append(message_list)
for instance in message_list:
if not instance['slave'] and \
instance['state'] in (INSTANCE_STATE_UNKNOWN, INSTANCE_STATE_STARTED_WITH_ERROR):
return instance['state']
elif not instance['slave'] and instance['state'] == INSTANCE_STATE_STARTED:
started = 1
elif not instance['slave'] and instance['state'] == INSTANCE_STATE_STOPPED:
stopped = 1
if instance['slave'] and instance['state'] == INSTANCE_STATE_UNKNOWN:
return instance['state']
if started and stopped:
return INSTANCE_STATE_STOPPED
return INSTANCE_STATE_UNKNOWN
if started:
return INSTANCE_STATE_STARTED
if stopped:
return INSTANCE_STATE_STOPPED
@retryOnNetworkFailure @retryOnNetworkFailure
def teardown(self): def teardown(self):
""" """
Interrupt a running test sequence, putting it in idle state. Interrupt a running test sequence, putting it in idle state.
""" """
self._logger ('Invoking TearDown for %s@%s' % (self.url, self.name)) self._logger('Invoking TearDown for %s@%s' % (self.url, self.name))
if self.request_kw is not None: if self.request_kw is not None:
self._request('destroyed') self._request(INSTANCE_STATE_DESTROYED)
if self.computer_guid is not None: if self.computer_guid is not None:
self._supply('destroyed') self._supply(INSTANCE_STATE_DESTROYED)
self.state = TESTER_STATE_INSTANCE_UNINSTALLED self.state = TESTER_STATE_INSTANCE_UNINSTALLED
def tic(self, now): def tic(self, now):
...@@ -494,7 +552,7 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -494,7 +552,7 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
Check for missed deadlines (-> test failure), conditions for moving to Check for missed deadlines (-> test failure), conditions for moving to
next state, and actually moving to next state (executing its payload). next state, and actually moving to next state (executing its payload).
""" """
self._logger ('[DEBUG] TIC') self._logger('[DEBUG] TIC')
deadline = self.deadline deadline = self.deadline
if deadline < now and deadline is not None: if deadline < now and deadline is not None:
...@@ -508,7 +566,7 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -508,7 +566,7 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
instance_state is None or instance_state is None or
instance_state == self._getInstanceState()): instance_state == self._getInstanceState()):
self._logger ('[DEBUG] Going to state %s (%r)', next_state, instance_state) self._logger('[DEBUG] Going to state %s (%r)', next_state, instance_state)
if next_state is None: if next_state is None:
return None return None
...@@ -517,4 +575,3 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator): ...@@ -517,4 +575,3 @@ class SoftwareReleaseTester(SlapOSMasterCommunicator):
self.deadline = now + delay self.deadline = now + delay
stepfunc(self) stepfunc(self)
return self.deadline return self.deadline
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7