Commit 4964820a authored by Bryton Lacquement's avatar Bryton Lacquement 🚪 Committed by Matevz Golob

fixup! erp5.util: add support for Python 3

31804f68 was merged too soon.

/reviewed-on nexedi/erp5!913
parent 7b0ad5f4
...@@ -57,7 +57,7 @@ class ArgumentType(object): ...@@ -57,7 +57,7 @@ class ArgumentType(object):
try: try:
module = __import__(module_name, globals(), locals(), [object_name], -1) module = __import__(module_name, globals(), locals(), [object_name], -1)
except Exception, e: except Exception as e:
raise argparse.ArgumentTypeError("Cannot import '%s.%s': %s" % \ raise argparse.ArgumentTypeError("Cannot import '%s.%s': %s" % \
(module_name, object_name, str(e))) (module_name, object_name, str(e)))
......
...@@ -264,7 +264,7 @@ class PerformanceTester(object): ...@@ -264,7 +264,7 @@ class PerformanceTester(object):
try: try:
error_message = exit_msg_queue.get() error_message = exit_msg_queue.get()
except KeyboardInterrupt, e: except KeyboardInterrupt as e:
print("\nInterrupted by user, stopping gracefully...", file=sys.stderr) print("\nInterrupted by user, stopping gracefully...", file=sys.stderr)
exit_status = 2 exit_status = 2
...@@ -272,7 +272,7 @@ class PerformanceTester(object): ...@@ -272,7 +272,7 @@ class PerformanceTester(object):
# blocking system call above and the system call should not be restarted # blocking system call above and the system call should not be restarted
# (using siginterrupt), otherwise the process will stall forever as its # (using siginterrupt), otherwise the process will stall forever as its
# child has already exited # child has already exited
except IOError, e: except IOError as e:
if e.errno == errno.EINTR: if e.errno == errno.EINTR:
continue continue
......
...@@ -88,13 +88,13 @@ class BenchmarkProcess(multiprocessing.Process): ...@@ -88,13 +88,13 @@ class BenchmarkProcess(multiprocessing.Process):
target(result, self._browser) target(result, self._browser)
except StopIteration: except StopIteration:
raise raise
except Exception, e: except Exception as e:
self._logger.info("Exception while running target suite for user %s: %s" % (self._browser._username, str(e))) self._logger.info("Exception while running target suite for user %s: %s" % (self._browser._username, str(e)))
msg = "%s: %s" % (target, traceback.format_exc()) msg = "%s: %s" % (target, traceback.format_exc())
try: try:
msg += "Last response headers:\n%s\nLast response contents:\n%s" % \ msg += "Last response headers:\n%s\nLast response contents:\n%s" % \
(self._browser.headers, self._browser.contents) (self._browser.headers, self._browser.contents)
except: except Exception:
pass pass
self._error_counter += 1 self._error_counter += 1
...@@ -123,7 +123,7 @@ class BenchmarkProcess(multiprocessing.Process): ...@@ -123,7 +123,7 @@ class BenchmarkProcess(multiprocessing.Process):
try: try:
self._logger.info(str(result.getCurrentSuiteUseCaseStat())) self._logger.info(str(result.getCurrentSuiteUseCaseStat()))
except: except Exception:
pass pass
result.iterationFinished() result.iterationFinished()
...@@ -183,14 +183,14 @@ class BenchmarkProcess(multiprocessing.Process): ...@@ -183,14 +183,14 @@ class BenchmarkProcess(multiprocessing.Process):
runIteration(result) runIteration(result)
self._current_repeat += 1 self._current_repeat += 1
except StopIteration, e: except StopIteration as e:
self._logger.error(e) self._logger.error(e)
except RuntimeError, e: except RuntimeError as e:
exit_msg = str(e) exit_msg = str(e)
exit_status = 1 exit_status = 1
except BaseException, e: except BaseException as e:
exit_msg = traceback.format_exc() exit_msg = traceback.format_exc()
self._logger.error(exit_msg) self._logger.error(exit_msg)
exit_status = 2 exit_status = 2
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
from __future__ import print_function from __future__ import print_function
import argparse import argparse
import re import re
import six
def parseArguments(): def parseArguments():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
...@@ -98,13 +99,15 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list, ...@@ -98,13 +99,15 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list,
merged_label_dict = {} merged_label_dict = {}
for filename in filename_list: for filename in filename_list:
reader = csv.reader(open(filename, 'rb'), delimiter=',', reader = csv.reader(open(filename, 'r'), delimiter=',',
quoting=csv.QUOTE_MINIMAL) quoting=csv.QUOTE_MINIMAL)
reader_list.append(reader) reader_list.append(reader)
# Get headers # Get headers
row_list = [ unicode(row, 'utf-8') for row in reader.next() ] if str is bytes:
row_list = [row.decode('utf-8') for row in next(reader)]
else:
row_list = [list(next(reader))]
if not label_list: if not label_list:
label_list = row_list label_list = row_list
label_merged_index = 0 label_merged_index = 0
...@@ -156,8 +159,8 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list, ...@@ -156,8 +159,8 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list,
report_dict['results'].setdefault(stat.full_label, []).append(stat) report_dict['results'].setdefault(stat.full_label, []).append(stat)
if row_list != label_list: if row_list != label_list:
raise AssertionError, "ERROR: Result labels: %s != %s" % \ raise AssertionError("ERROR: Result labels: %s != %s" %
(label_list, row_list) (label_list, row_list))
iteration_index = 0 iteration_index = 0
for row_list in reader: for row_list in reader:
...@@ -169,7 +172,7 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list, ...@@ -169,7 +172,7 @@ def computeStatisticFromFilenameList(argument_namespace, filename_list,
use_case_suite = row_use_case_mapping_dict.get(idx, None) use_case_suite = row_use_case_mapping_dict.get(idx, None)
if use_case_suite: if use_case_suite:
current_count = int(row) current_count = int(row)
current_duration = float(row_iter.next()[1]) / 3600.0 current_duration = float(next(row_iter)[1]) / 3600
if not current_count: if not current_count:
continue continue
...@@ -587,8 +590,8 @@ def generateReport(): ...@@ -587,8 +590,8 @@ def generateReport():
(nb_users_list[0], (nb_users_list[0],
nb_users_list[-1]) nb_users_list[-1])
for suite_name, report_dict in range_user_report_dict.iteritems(): for suite_name, report_dict in six.iteritems(range_user_report_dict):
for label, stat_list in report_dict['results'].iteritems(): for label, stat_list in six.iteritems(report_dict['results']):
drawConcurrentUsersPlot( drawConcurrentUsersPlot(
pdf, pdf,
title_fmt % label, title_fmt % label,
......
...@@ -334,7 +334,7 @@ class CSVBenchmarkResult(BenchmarkResult): ...@@ -334,7 +334,7 @@ class CSVBenchmarkResult(BenchmarkResult):
from cStringIO import StringIO from cStringIO import StringIO
import xmlrpclib from six.moves import xmlrpc_client as xmlrpclib
import datetime import datetime
class ERP5BenchmarkResult(BenchmarkResult): class ERP5BenchmarkResult(BenchmarkResult):
......
...@@ -51,16 +51,16 @@ class ScalabilityTester(PerformanceTester): ...@@ -51,16 +51,16 @@ class ScalabilityTester(PerformanceTester):
def postRun(self, error_message_set): def postRun(self, error_message_set):
from logging import Formatter from logging import Formatter
import sys import sys
import urllib from six.moves.urllib.request import urlencode
import urllib2 from six.moves.urllib.parse import urlopen
try: try:
urllib2.urlopen("http://[%s]:%d/report" % \ urlopen("http://[%s]:%d/report" % \
(self._argument_namespace.manager_address, (self._argument_namespace.manager_address,
self._argument_namespace.manager_port), self._argument_namespace.manager_port),
urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close() urlencode({'error_message_set': '|'.join(error_message_set)})).close()
except: except Exception:
print("ERROR: %s" % Formatter().formatException(sys.exc_info()), file=sys.stderr) print("ERROR: %s" % Formatter().formatException(sys.exc_info()), file=sys.stderr)
def getResultClass(self): def getResultClass(self):
......
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import division
import argparse import argparse
import os import os
import shutil import shutil
...@@ -253,7 +255,7 @@ class ScalabilityLauncher(object): ...@@ -253,7 +255,7 @@ class ScalabilityLauncher(object):
log_file_name_prefix = "%s_%s_suite_%s" %(LOG_FILE_PREFIX, current_test.title, test_suite) log_file_name_prefix = "%s_%s_suite_%s" %(LOG_FILE_PREFIX, current_test.title, test_suite)
command_list.append([tester_path, command_list.append([tester_path,
instance_url, instance_url,
str(user_quantity/len(test_suite_list)), str(user_quantity//len(test_suite_list)),
test_suite, test_suite,
'--benchmark-path-list', benchmarks_path, '--benchmark-path-list', benchmarks_path,
'--users-file-path', user_file_path, '--users-file-path', user_file_path,
...@@ -264,7 +266,7 @@ class ScalabilityLauncher(object): ...@@ -264,7 +266,7 @@ class ScalabilityLauncher(object):
'--user-index', str(user_index), '--user-index', str(user_index),
"--duration", "%d"%test_duration, "--duration", "%d"%test_duration,
]) ])
user_index += user_quantity/len(test_suite_list) user_index += user_quantity//len(test_suite_list)
# Launch commands # Launch commands
exec_env = os.environ.copy() exec_env = os.environ.copy()
exec_env.update({'raise_error_if_fail': False}) exec_env.update({'raise_error_if_fail': False})
...@@ -298,7 +300,7 @@ class ScalabilityLauncher(object): ...@@ -298,7 +300,7 @@ class ScalabilityLauncher(object):
"tests=%s\n"\ "tests=%s\n"\
"duration=%d\n"\ "duration=%d\n"\
%( %(
(user_quantity/len(test_suite_list))*len(test_suite_list), (user_quantity//len(test_suite_list))*len(test_suite_list),
len(test_suite_list), len(test_suite_list),
'_'.join(test_suite_list), '_'.join(test_suite_list),
test_duration test_duration
......
...@@ -170,7 +170,7 @@ class TestResultLineProxy(RPCRetry): ...@@ -170,7 +170,7 @@ class TestResultLineProxy(RPCRetry):
try: try:
return bool(self._retryRPC('isTestCaseAlive', return bool(self._retryRPC('isTestCaseAlive',
(self._test_result_line_path,))) (self._test_result_line_path,)))
except: except Exception:
raise ValueError('isTestCaseAlive Failed.') raise ValueError('isTestCaseAlive Failed.')
def stop(self, test_count=None, error_count=None, failure_count=None, def stop(self, test_count=None, error_count=None, failure_count=None,
......
...@@ -32,8 +32,8 @@ ...@@ -32,8 +32,8 @@
import logging import logging
import sys import sys
import urllib from six.moves.urllib.parse import urlencode
import Cookie from six.moves import http_cookies as Cookie
import re import re
from zope.testbrowser._compat import urlparse from zope.testbrowser._compat import urlparse
...@@ -205,7 +205,7 @@ class Browser(ExtendedTestBrowser): ...@@ -205,7 +205,7 @@ class Browser(ExtendedTestBrowser):
url_or_path = urlparse.urljoin(self._erp5_base_url, url_or_path) url_or_path = urlparse.urljoin(self._erp5_base_url, url_or_path)
if isinstance(data, dict): if isinstance(data, dict):
data = urllib.urlencode(data) data = urlencode(data)
self._logger.debug("Opening: " + url_or_path) self._logger.debug("Opening: " + url_or_path)
super(Browser, self).open(url_or_path, data) super(Browser, self).open(url_or_path, data)
...@@ -274,7 +274,7 @@ class Browser(ExtendedTestBrowser): ...@@ -274,7 +274,7 @@ class Browser(ExtendedTestBrowser):
location_without_query_string, query_string = location.split('?') location_without_query_string, query_string = location.split('?')
location = ( location = (
location_without_query_string + location_without_query_string +
'?' + urllib.urlencode(urlparse.parse_qs(query_string, '?' + urlencode(urlparse.parse_qs(query_string,
strict_parsing=True), strict_parsing=True),
doseq=True)) doseq=True))
# END: Bugfix # END: Bugfix
...@@ -318,7 +318,7 @@ class Browser(ExtendedTestBrowser): ...@@ -318,7 +318,7 @@ class Browser(ExtendedTestBrowser):
url_or_path = urlparse.urljoin(self._erp5_base_url, url_or_path) url_or_path = urlparse.urljoin(self._erp5_base_url, url_or_path)
if isinstance(data, dict): if isinstance(data, dict):
data = urllib.urlencode(data) data = urlencode(data)
url = self._absoluteUrl(url_or_path) url = self._absoluteUrl(url_or_path)
self._logger.debug("Opening: " + url) self._logger.debug("Opening: " + url)
...@@ -365,11 +365,7 @@ class Browser(ExtendedTestBrowser): ...@@ -365,11 +365,7 @@ class Browser(ExtendedTestBrowser):
@return: Cookie value @return: Cookie value
@rtype: str @rtype: str
""" """
for cookie_name, cookie_value in self.cookies.iteritems(): return self.cookies.get(name, default)
if name == cookie_name:
return cookie_value
return default
@property @property
def mainForm(self): def mainForm(self):
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import division, print_function from __future__ import division, print_function
import six
from erp5.util.testbrowser.browser import Browser from erp5.util.testbrowser.browser import Browser
ITERATION = 20 ITERATION = 20
...@@ -89,5 +90,5 @@ if __name__ == '__main__': ...@@ -89,5 +90,5 @@ if __name__ == '__main__':
benchmarkAddPerson(counter, result_dict) benchmarkAddPerson(counter, result_dict)
counter += 1 counter += 1
for title, time_list in result_dict.iteritems(): for title, time_list in six.iteritems(result_dict):
print("%s: %.4fs" % (title, sum(time_list) / ITERATION)) print("%s: %.4fs" % (title, sum(time_list) / ITERATION))
...@@ -104,7 +104,7 @@ class ScalabilityTestRunner(): ...@@ -104,7 +104,7 @@ class ScalabilityTestRunner():
self.slapos_url = self.testnode.taskdistribution.getSlaposUrl() self.slapos_url = self.testnode.taskdistribution.getSlaposUrl()
if not self.slapos_url: if not self.slapos_url:
self.slapos_url = self.testnode.config['server_url'] self.slapos_url = self.testnode.config['server_url']
except: except Exception:
self.slapos_url = self.testnode.config['server_url'] self.slapos_url = self.testnode.config['server_url']
# Get Slapos Master url used for api rest (using hateoas) # Get Slapos Master url used for api rest (using hateoas)
...@@ -458,7 +458,7 @@ Require valid-user ...@@ -458,7 +458,7 @@ Require valid-user
suite_class = getattr(module, test_suite) suite_class = getattr(module, test_suite)
suite = suite_class(**kwargs) suite = suite_class(**kwargs)
repo_location = "%s/%s/" % (location, SCALABILITY_TEST) repo_location = "%s/%s/" % (location, SCALABILITY_TEST)
except: except Exception:
pass pass
return suite, repo_location return suite, repo_location
......
...@@ -363,7 +363,7 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -363,7 +363,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
def getInstanceGuid(): def getInstanceGuid():
try: try:
return self.instance.getInstanceGuid() return self.instance.getInstanceGuid()
except: except Exception:
return None return None
frontend_master_ipv6 = None frontend_master_ipv6 = None
instance_guid = None instance_guid = None
......
...@@ -35,6 +35,7 @@ import imp ...@@ -35,6 +35,7 @@ import imp
import gzip import gzip
import getopt import getopt
from time import time from time import time
import six
PROFILING_ENABLED = False PROFILING_ENABLED = False
if PROFILING_ENABLED: if PROFILING_ENABLED:
...@@ -150,7 +151,7 @@ def parseFile(filename, measure_dict): ...@@ -150,7 +151,7 @@ def parseFile(filename, measure_dict):
if line_number > 0: if line_number > 0:
duration = time() - begin duration = time() - begin
print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \ print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
(match_count, (match_count / line_number) * 100, skip_count, (skip_count / line_number) * 100, (line_number - match_count - skip_count), (1 - (match_count + skip_count) / line_number)) * 100, duration, line_number // duration), (match_count, (match_count / line_number) * 100, skip_count, (skip_count / line_number) * 100, (line_number - match_count - skip_count), (1 - (match_count + skip_count) / line_number) * 100, duration, line_number // duration),
file=sys.stderr) file=sys.stderr)
debug = False debug = False
...@@ -209,9 +210,9 @@ if len(load_file_name_list): ...@@ -209,9 +210,9 @@ if len(load_file_name_list):
with open(load_file_name) as load_file: with open(load_file_name) as load_file:
temp_measure_dict = eval(load_file.read(), {}) temp_measure_dict = eval(load_file.read(), {})
assert isinstance(measure_dict, dict) assert isinstance(measure_dict, dict)
for filter_id, result_dict in temp_measure_dict.iteritems(): for filter_id, result_dict in six.iteritems(temp_measure_dict):
for result, date_dict in result_dict.iteritems(): for result, date_dict in six.iteritems(result_dict):
for date, duration_list in date_dict.iteritems(): for date, duration_list in six.iteritems(date_dict):
measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list) measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list)
print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr) print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr)
...@@ -231,18 +232,17 @@ if outfile_prefix is not None: ...@@ -231,18 +232,17 @@ if outfile_prefix is not None:
append = measure_id_list.append append = measure_id_list.append
sheet_dict = {} sheet_dict = {}
line_dict = {} line_dict = {}
for match_id, match_dict in measure_dict.iteritems(): for match_id, match_dict in six.iteritems(measure_dict):
for result_id, result_dict in match_dict.iteritems(): for result_id, result_dict in six.iteritems(match_dict):
measure_id = (match_id, result_id) measure_id = (match_id, result_id)
sheet_dict.setdefault(match_id, []).append((result_id, measure_id)) sheet_dict.setdefault(match_id, []).append((result_id, measure_id))
append(measure_id) append(measure_id)
for date, measure_list in result_dict.iteritems(): for date, measure_list in six.iteritems(result_dict):
first_level_dict = line_dict.setdefault(date, {}) first_level_dict = line_dict.setdefault(date, {})
assert measure_id not in first_level_dict assert measure_id not in first_level_dict
first_level_dict[measure_id] = measure_list first_level_dict[measure_id] = measure_list
date_list = line_dict.keys() date_list = sorted(line_dict, key=date_key)
date_list.sort(key=date_key)
def render_cell(value_list, format): def render_cell(value_list, format):
if isinstance(value_list, (list, tuple)): if isinstance(value_list, (list, tuple)):
...@@ -251,7 +251,7 @@ if outfile_prefix is not None: ...@@ -251,7 +251,7 @@ if outfile_prefix is not None:
return value_list return value_list
def renderOutput(data_format, filename_suffix): def renderOutput(data_format, filename_suffix):
for sheet_id, sheet_column_list in sheet_dict.iteritems(): for sheet_id, sheet_column_list in six.iteritems(sheet_dict):
outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix) outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix)
print('Writing to %r...' % (outfile_name, ), file=sys.stderr) print('Writing to %r...' % (outfile_name, ), file=sys.stderr)
with open(outfile_name, 'w') as outfile: with open(outfile_name, 'w') as outfile:
...@@ -259,7 +259,7 @@ if outfile_prefix is not None: ...@@ -259,7 +259,7 @@ if outfile_prefix is not None:
decimate_dict = {} decimate_dict = {}
decimate = 0 decimate = 0
for date in date_list: for date in date_list:
for key, value in line_dict[date].iteritems(): for key, value in six.iteritems(line_dict[date]):
decimate_dict.setdefault(key, []).extend(value) decimate_dict.setdefault(key, []).extend(value)
decimate += 1 decimate += 1
if decimate == decimate_count: if decimate == decimate_count:
......
...@@ -84,9 +84,9 @@ class CSVFile(object): ...@@ -84,9 +84,9 @@ class CSVFile(object):
if cell > value_max.get(key, 0): if cell > value_max.get(key, 0):
value_max[key] = cell value_max[key] = cell
column_dict[key].append(cell) column_dict[key].append(cell)
line_num = line_num / 100 line_num /= 100
for key in ratio_dict: for key in ratio_dict:
ratio_dict[key] //= line_num ratio_dict[key] /= line_num
def getColumn(self, column_id): def getColumn(self, column_id):
return self.column_dict[self.column_list[column_id]] return self.column_dict[self.column_list[column_id]]
...@@ -136,7 +136,7 @@ def main(): ...@@ -136,7 +136,7 @@ def main():
# date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...] # date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...]
factor = 1 factor = 1
if len(date_string_list) > 20: if len(date_string_list) > 20:
factor = int(len(date_string_list) // 20) factor = len(date_string_list) // 20
i = 0 i = 0
for date_string in date_string_list: for date_string in date_string_list:
if i % factor == 0: if i % factor == 0:
......
...@@ -39,6 +39,7 @@ import tempfile ...@@ -39,6 +39,7 @@ import tempfile
from datetime import datetime from datetime import datetime
import threading import threading
import signal import signal
import six
_MARKER = [] _MARKER = []
...@@ -517,7 +518,7 @@ class HTTPCacheCheckerTestSuite(object): ...@@ -517,7 +518,7 @@ class HTTPCacheCheckerTestSuite(object):
logging.info('End of second pass\n') logging.info('End of second pass\n')
if self.report_dict: if self.report_dict:
report_message_list = ['*Errors*:'] report_message_list = ['*Errors*:']
for url, message_list in self.report_dict.iteritems(): for url, message_list in six.iteritems(self.report_dict):
unique_message_list = [] unique_message_list = []
for message in message_list: for message in message_list:
if message not in unique_message_list: if message not in unique_message_list:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment