Commit 83f1427c authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

scalability: support test repeat count instead of test duration.

also repeat count and duration can be different by test suite.
parent c1deaaed
...@@ -171,17 +171,17 @@ class BenchmarkProcess(multiprocessing.Process): ...@@ -171,17 +171,17 @@ class BenchmarkProcess(multiprocessing.Process):
try: try:
with result_instance as result: with result_instance as result:
self._browser = self.getBrowser(result_instance.log_file) self._browser = self.getBrowser(result_instance.log_file)
if self._argument_namespace.duration > 0: if self._argument_namespace.repeat > -1:
self._logger.info("Iterate until repeat %d" % self._argument_namespace.repeat)
while self._current_repeat != (self._argument_namespace.repeat + 1):
runIteration(result)
self._current_repeat += 1
elif:
self._logger.info("Iterate until duration %d" % self._argument_namespace.duration) self._logger.info("Iterate until duration %d" % self._argument_namespace.duration)
start_time = time.time() start_time = time.time()
while self._argument_namespace.duration > (time.time()-start_time): while self._argument_namespace.duration > (time.time()-start_time):
runIteration(result) runIteration(result)
self._current_repeat += 1 self._current_repeat += 1
else:
self._logger.info("Iterate until repeat %d" % self._argument_namespace.repeat)
while self._current_repeat != (self._argument_namespace.repeat + 1):
runIteration(result)
self._current_repeat += 1
except StopIteration as e: except StopIteration as e:
self._logger.error(e) self._logger.error(e)
......
...@@ -17,6 +17,7 @@ from erp5.util import taskdistribution ...@@ -17,6 +17,7 @@ from erp5.util import taskdistribution
from erp5.util.testnode import Utils from erp5.util.testnode import Utils
from erp5.util.testnode.ProcessManager import ProcessManager from erp5.util.testnode.ProcessManager import ProcessManager
import datetime import datetime
from inspect import getargspec
MAX_INSTALLATION_TIME = 60*50 MAX_INSTALLATION_TIME = 60*50
MAX_TESTING_TIME = 60 MAX_TESTING_TIME = 60
...@@ -222,15 +223,11 @@ class ScalabilityLauncher(object): ...@@ -222,15 +223,11 @@ class ScalabilityLauncher(object):
self.log("Test Case %s going to be run." %(current_test.title)) self.log("Test Case %s going to be run." %(current_test.title))
# Prepare configuration # Prepare configuration
current_test_number = int(current_test.title) current_test_number = int(current_test.title)
test_duration = suite.getTestDuration(current_test_number)
if not test_duration or test_duration == 0:
test_duration = SCALABILITY_TEST_DURATION
benchmarks_path = os.path.join(self.__argumentNamespace.repo_location, suite.getTestPath()) benchmarks_path = os.path.join(self.__argumentNamespace.repo_location, suite.getTestPath())
user_file_full_path = os.path.join(self.__argumentNamespace.repo_location, suite.getUsersFilePath()) user_file_full_path = os.path.join(self.__argumentNamespace.repo_location, suite.getUsersFilePath())
user_file_path = os.path.split(user_file_full_path)[0] user_file_path = os.path.split(user_file_full_path)[0]
user_file = os.path.split(user_file_full_path)[1] user_file = os.path.split(user_file_full_path)[1]
tester_path = self.__argumentNamespace.runner_path tester_path = self.__argumentNamespace.runner_path
user_quantity = suite.getUserQuantity(current_test_number)
instance_url = self.__argumentNamespace.instance_url instance_url = self.__argumentNamespace.instance_url
metric_url = self.__argumentNamespace.metric_url metric_url = self.__argumentNamespace.metric_url
...@@ -239,9 +236,19 @@ class ScalabilityLauncher(object): ...@@ -239,9 +236,19 @@ class ScalabilityLauncher(object):
metric_thread = TestMetricThread(metric_url, self.log, metric_thread_stop_event, interval=TEST_METRIC_TIME_INTERVAL) metric_thread = TestMetricThread(metric_url, self.log, metric_thread_stop_event, interval=TEST_METRIC_TIME_INTERVAL)
metric_thread.start() metric_thread.start()
try:
bootstrap_password = self.__argumentNamespace.bootstrap_password bootstrap_password = self.__argumentNamespace.bootstrap_password
self.updateUsersFile(user_quantity, bootstrap_password, user_file_full_path + ".py") # Prepare users
user_quantity_dict = {}
total_user_quantity = 0
for test_suite in test_suite_list:
if 'test_name' in getargspec(suite.getUserQuantity).args:
user_quantity = suite.getUserQuantity(current_test_number, test_suite)
else:
user_quantity = suite.getUserQuantity(current_test_number)
user_quantity_dict[test_suite] = user_quantity
total_user_quantity += user_quantity
try:
self.updateUsersFile(total_user_quantity, bootstrap_password, user_file_full_path + ".py")
except Exception as e: except Exception as e:
self.log("ERROR while updating file: " + str(e)) self.log("ERROR while updating file: " + str(e))
...@@ -250,12 +257,33 @@ class ScalabilityLauncher(object): ...@@ -250,12 +257,33 @@ class ScalabilityLauncher(object):
command_list = [] command_list = []
user_index = 0 user_index = 0
max_test_duration = 0.0
# Prepare commands # Prepare commands
for test_suite in test_suite_list: for test_suite in test_suite_list:
getTestDuration = getattr(suite, 'getTestDuration', None)
if getTestDuration is not None:
if len(getargspec(getTestDuration).args) == 3:
test_duration = getTestDuration(current_test_number, test_suite)
else:
test_duration = getTestDuration(current_test_number)
else:
test_duration = 0
getTestRepeat = getattr(suite, 'getTestRepeat', None)
if getTestRepeat is not None:
if len(getargspec(getTestRepeat).args) == 3:
test_repeat = getTestRepeat(current_test_number, test_suite)
else:
test_repeat = getTestRepeat(current_test_number)
else:
test_repeat = -1
if not test_duration:
test_duration = SCALABILITY_TEST_DURATION
if max_test_duration < test_duration:
max_test_duration = test_duration
log_file_name_prefix = "%s_%s_suite_%s" %(LOG_FILE_PREFIX, current_test.title, test_suite) log_file_name_prefix = "%s_%s_suite_%s" %(LOG_FILE_PREFIX, current_test.title, test_suite)
command_list.append([tester_path, command_list.append([tester_path,
instance_url, instance_url,
str(user_quantity//len(test_suite_list)), str(user_quantity_dict[test_suite]),
test_suite, test_suite,
'--benchmark-path-list', benchmarks_path, '--benchmark-path-list', benchmarks_path,
'--users-file-path', user_file_path, '--users-file-path', user_file_path,
...@@ -264,9 +292,10 @@ class ScalabilityLauncher(object): ...@@ -264,9 +292,10 @@ class ScalabilityLauncher(object):
'--report-directory', self.__argumentNamespace.log_path, '--report-directory', self.__argumentNamespace.log_path,
'--max-errors', str(MAX_ERRORS), '--max-errors', str(MAX_ERRORS),
'--user-index', str(user_index), '--user-index', str(user_index),
"--repeat", "%d"%test_repeat,
"--duration", "%d"%test_duration, "--duration", "%d"%test_duration,
]) ])
user_index += user_quantity//len(test_suite_list) user_index += user_quantity_dict[test_suite]
# Launch commands # Launch commands
exec_env = os.environ.copy() exec_env = os.environ.copy()
exec_env.update({'raise_error_if_fail': False}) exec_env.update({'raise_error_if_fail': False})
...@@ -274,8 +303,8 @@ class ScalabilityLauncher(object): ...@@ -274,8 +303,8 @@ class ScalabilityLauncher(object):
test_thread = TestThread(process_manager, command, self.log, env=exec_env) test_thread = TestThread(process_manager, command, self.log, env=exec_env)
test_thread.start() test_thread.start()
# Sleep # Sleep
self.log("Going to sleep for %s seconds." % str(test_duration)) self.log("Going to sleep for %s seconds." % str(max_test_duration))
time.sleep(test_duration) time.sleep(max_test_duration)
process_manager.killPreviousRun() process_manager.killPreviousRun()
self.moveLogs(log_dir, current_test) self.moveLogs(log_dir, current_test)
...@@ -298,12 +327,12 @@ class ScalabilityLauncher(object): ...@@ -298,12 +327,12 @@ class ScalabilityLauncher(object):
test_details = "number of users=%d\n"\ test_details = "number of users=%d\n"\
"number of tests=%d\n"\ "number of tests=%d\n"\
"tests=%s\n"\ "tests=%s\n"\
"duration=%d\n"\ "maximum duration=%d\n"\
%( %(
(user_quantity//len(test_suite_list))*len(test_suite_list), total_user_quantity,
len(test_suite_list), len(test_suite_list),
'_'.join(test_suite_list), '_'.join(test_suite_list),
test_duration max_test_duration
) )
self.log("Test details: %s" % test_details) self.log("Test details: %s" % test_details)
...@@ -331,7 +360,7 @@ class ScalabilityLauncher(object): ...@@ -331,7 +360,7 @@ class ScalabilityLauncher(object):
test_result_line_test.stop(stdout=test_output, test_result_line_test.stop(stdout=test_output,
command=test_details, command=test_details,
test_count=len(test_suite_list), test_count=len(test_suite_list),
duration=test_duration) duration=max_test_duration)
except Exception as e: except Exception as e:
self.log("Error during communication to master: " + str(e)) self.log("Error during communication to master: " + str(e))
raise e raise e
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment