Commit 31804f68 authored by Bryton Lacquement's avatar Bryton Lacquement 🚪 Committed by Julien Muchembled

erp5.util: add support for Python 3

/reviewed-on nexedi/erp5!830
parent 5abb074d
This diff is collapsed.
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import argparse import argparse
import os import os
import sys import sys
...@@ -264,7 +265,7 @@ class PerformanceTester(object): ...@@ -264,7 +265,7 @@ class PerformanceTester(object):
error_message = exit_msg_queue.get() error_message = exit_msg_queue.get()
except KeyboardInterrupt, e: except KeyboardInterrupt, e:
print >>sys.stderr, "\nInterrupted by user, stopping gracefully..." print("\nInterrupted by user, stopping gracefully...", file=sys.stderr)
exit_status = 2 exit_status = 2
# An IOError may be raised when receiving a SIGINT which interrupts the # An IOError may be raised when receiving a SIGINT which interrupts the
...@@ -337,7 +338,7 @@ class PerformanceTester(object): ...@@ -337,7 +338,7 @@ class PerformanceTester(object):
def main(): def main():
error_message_set, exit_status = PerformanceTester().run() error_message_set, exit_status = PerformanceTester().run()
for error_message in error_message_set: for error_message in error_message_set:
print >>sys.stderr, "ERROR: %s" % error_message print("ERROR: %s" % error_message, file=sys.stderr)
sys.exit(exit_status) sys.exit(exit_status)
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import argparse import argparse
import re import re
...@@ -537,7 +538,7 @@ def generateReport(): ...@@ -537,7 +538,7 @@ def generateReport():
for filename in filename_iter: for filename in filename_iter:
# There may be no results at all in case of errors # There may be no results at all in case of errors
if not os.stat(filename).st_size: if not os.stat(filename).st_size:
print >>sys.stderr, "Ignoring empty file %s" % filename print("Ignoring empty file %s" % filename, file=sys.stderr)
continue continue
report_dict = per_nb_users_report_dict.setdefault( report_dict = per_nb_users_report_dict.setdefault(
...@@ -546,10 +547,8 @@ def generateReport(): ...@@ -546,10 +547,8 @@ def generateReport():
report_dict['filename'].append(filename) report_dict['filename'].append(filename)
if not per_nb_users_report_dict: if not per_nb_users_report_dict:
print >>sys.stderr, "ERROR: No result file found, perhaps "\ sys.exit("ERROR: No result file found, perhaps ``--filename-prefix'' should"
"``--filename-prefix'' should be specified?" "be specified?")
sys.exit(1)
pdf = PdfPages(argument_namespace.output_filename) pdf = PdfPages(argument_namespace.output_filename)
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
from .result import CSVBenchmarkResult, NothingFlushedException from .result import CSVBenchmarkResult, NothingFlushedException
class CSVScalabilityBenchmarkResult(CSVBenchmarkResult): class CSVScalabilityBenchmarkResult(CSVBenchmarkResult):
...@@ -60,7 +61,7 @@ class ScalabilityTester(PerformanceTester): ...@@ -60,7 +61,7 @@ class ScalabilityTester(PerformanceTester):
urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close() urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close()
except: except:
print >>sys.stderr, "ERROR: %s" % Formatter().formatException(sys.exc_info()) print("ERROR: %s" % Formatter().formatException(sys.exc_info()), file=sys.stderr)
def getResultClass(self): def getResultClass(self):
if not self._argument_namespace.erp5_publish_url: if not self._argument_namespace.erp5_publish_url:
......
...@@ -27,7 +27,7 @@ def main(): ...@@ -27,7 +27,7 @@ def main():
if error_message_set: if error_message_set:
exit_status = 1 exit_status = 1
for error in error_message_set: for error in error_message_set:
print error print(error)
elif result: elif result:
print result print(result)
sys.exit(exit_status) sys.exit(exit_status)
...@@ -5,23 +5,15 @@ import os ...@@ -5,23 +5,15 @@ import os
import shutil import shutil
import time import time
import sys import sys
import multiprocessing
import signal
import errno
import json import json
import logging import logging
import logging.handlers import logging.handlers
import glob import glob
import urlparse
import httplib
import base64
import threading import threading
from erp5.util.benchmark.argument import ArgumentType
from erp5.util.benchmark.performance_tester import PerformanceTester
from erp5.util.benchmark.thread import TestThread, TestMetricThread from erp5.util.benchmark.thread import TestThread, TestMetricThread
from erp5.util import taskdistribution from erp5.util import taskdistribution
from erp5.util.testnode import Utils from erp5.util.testnode import Utils
from erp5.util.testnode.ProcessManager import SubprocessError, ProcessManager, CancellationError from erp5.util.testnode.ProcessManager import ProcessManager
import datetime import datetime
MAX_INSTALLATION_TIME = 60*50 MAX_INSTALLATION_TIME = 60*50
...@@ -179,31 +171,28 @@ class ScalabilityLauncher(object): ...@@ -179,31 +171,28 @@ class ScalabilityLauncher(object):
""" """
data_array = self.__argumentNamespace.current_test_data.split(',') data_array = self.__argumentNamespace.current_test_data.split(',')
data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]}) data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]})
decoded_data = Utils.deunicodeData(json.loads(data)) encoded_data = Utils.deunicodeData(json.loads(data))
return ScalabilityTest(decoded_data, self.test_result) return ScalabilityTest(encoded_data, self.test_result)
def clearUsersFile(self, user_file_path): def clearUsersFile(self, user_file_path):
self.log("Clearing users file: %s" % user_file_path) self.log("Clearing users file: %s" % user_file_path)
os.remove(user_file_path) os.remove(user_file_path)
users_file = open(user_file_path, "w") with open(user_file_path, "w") as users_file:
for line in self.users_file_original_content: for line in self.users_file_original_content:
users_file.write(line) users_file.write(line)
users_file.close()
def updateUsersFile(self, user_quantity, password, user_file_path): def updateUsersFile(self, user_quantity, password, user_file_path):
self.log("Updating users file: %s" % user_file_path) self.log("Updating users file: %s" % user_file_path)
users_file = open(user_file_path, "r") with open(user_file_path, "r") as users_file:
file_content = users_file.readlines() file_content = users_file.readlines()
self.users_file_original_content = file_content self.users_file_original_content = file_content
new_file_content = [] new_file_content = []
for line in file_content: for line in file_content:
new_file_content.append(line.replace('<password>', password).replace('<user_quantity>', str(user_quantity))) new_file_content.append(line.replace('<password>', password).replace('<user_quantity>', str(user_quantity)))
users_file.close()
os.remove(user_file_path) os.remove(user_file_path)
users_file = open(user_file_path, "w") with open(user_file_path, "w") as users_file:
for line in new_file_content: for line in new_file_content:
users_file.write(line) users_file.write(line)
users_file.close()
def run(self): def run(self):
self.log("Scalability Launcher started, with:") self.log("Scalability Launcher started, with:")
......
...@@ -40,11 +40,15 @@ Example use: ...@@ -40,11 +40,15 @@ Example use:
test_line.stop() test_line.stop()
""" """
from __future__ import print_function from __future__ import print_function
import httplib import six
from six.moves import (
map,
http_client as httplib,
xmlrpc_client as xmlrpclib,
)
import socket import socket
import threading import threading
import time import time
import xmlrpclib
__all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser'] __all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser']
...@@ -89,11 +93,17 @@ def patchRPCParser(error_handler): ...@@ -89,11 +93,17 @@ def patchRPCParser(error_handler):
def verbose_feed(self, data): def verbose_feed(self, data):
try: try:
return original_feed(self, data) return original_feed(self, data)
except Exception, exc: except Exception as exc:
if not error_handler(data, exc): if not error_handler(data, exc):
raise raise
parser_klass.feed = verbose_feed parser_klass.feed = verbose_feed
try: # PY3
basestring
except NameError:
basestring = bytes, str
unicode = str
def binarize_args(arg): def binarize_args(arg):
# Converts recursively basestring arg into xmlrpclib.Binary, as they can # Converts recursively basestring arg into xmlrpclib.Binary, as they can
# contain non-XML allowed characters # contain non-XML allowed characters
...@@ -102,9 +112,9 @@ def binarize_args(arg): ...@@ -102,9 +112,9 @@ def binarize_args(arg):
arg = arg.encode('utf-8') arg = arg.encode('utf-8')
return xmlrpclib.Binary(arg) return xmlrpclib.Binary(arg)
if isinstance(arg, (list, tuple, set)): if isinstance(arg, (list, tuple, set)):
return map(binarize_args, arg) return list(map(binarize_args, arg))
if isinstance(arg, dict): if isinstance(arg, dict):
return {k: binarize_args(v) for k, v in arg.iteritems()} return {k: binarize_args(v) for k, v in six.iteritems(arg)}
return arg return arg
class RPCRetry(object): class RPCRetry(object):
...@@ -350,7 +360,7 @@ class TestResultProxy(RPCRetry): ...@@ -350,7 +360,7 @@ class TestResultProxy(RPCRetry):
caption_list = [] caption_list = []
append = caption_list.append append = caption_list.append
for name, (stream, max_history_bytes) in \ for name, (stream, max_history_bytes) in \
self._watcher_dict.iteritems(): six.iteritems(self._watcher_dict):
append('==> %s <==' % (name, )) append('==> %s <==' % (name, ))
start = stream.tell() start = stream.tell()
stream.seek(0, 2) stream.seek(0, 2)
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
# #
# TODO: There must be a better way than the code below to do that though... # TODO: There must be a better way than the code below to do that though...
from __future__ import print_function
import sys import sys
from erp5.util.testbrowser.browser import Browser from erp5.util.testbrowser.browser import Browser
...@@ -19,11 +20,8 @@ try: ...@@ -19,11 +20,8 @@ try:
user_nbr = int(user_nbr) user_nbr = int(user_nbr)
except ValueError: except ValueError:
print >>sys.stderr, "ERROR: Missing arguments: %s URL USERNAME " \ sys.exit("ERROR: Missing arguments: %s URL USERNAME PASSWORD NUMBER_OF_USERS "
"PASSWORD NUMBER_OF_USERS NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % \ "NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % sys.argv[0])
sys.argv[0]
sys.exit(1)
# Create a browser instance # Create a browser instance
browser = Browser(url, username, password) browser = Browser(url, username, password)
......
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import division, print_function
from erp5.util.testbrowser.browser import Browser from erp5.util.testbrowser.browser import Browser
ITERATION = 20 ITERATION = 20
...@@ -89,4 +90,4 @@ if __name__ == '__main__': ...@@ -89,4 +90,4 @@ if __name__ == '__main__':
counter += 1 counter += 1
for title, time_list in result_dict.iteritems(): for title, time_list in result_dict.iteritems():
print "%s: %.4fs" % (title, float(sum(time_list)) / ITERATION) print("%s: %.4fs" % (title, sum(time_list) / ITERATION))
...@@ -32,6 +32,8 @@ import string ...@@ -32,6 +32,8 @@ import string
import random import random
from .Utils import createFolder from .Utils import createFolder
from six.moves import range
class SlapOSInstance(object): class SlapOSInstance(object):
""" """
Base of an software instance, Base of an software instance,
...@@ -69,14 +71,14 @@ class NodeTestSuite(SlapOSInstance): ...@@ -69,14 +71,14 @@ class NodeTestSuite(SlapOSInstance):
def createSuiteLog(self): def createSuiteLog(self):
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log # /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log
alphabets = string.digits + string.letters alphabets = string.digits + string.ascii_letters
while 1: while 1:
log_folder_name = '%s-%s' % (self.reference, log_folder_name = '%s-%s' % (self.reference,
''.join(random.choice(alphabets) for i in xrange(10))) ''.join(random.choice(alphabets) for i in range(10)))
log_folder_path = os.path.join(self.log_directory, log_folder_name) log_folder_path = os.path.join(self.log_directory, log_folder_name)
try: try:
os.makedirs(log_folder_path) os.makedirs(log_folder_path)
except OSError, e: except OSError as e:
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
else: else:
......
...@@ -79,7 +79,8 @@ def subprocess_capture(p, log_prefix, get_output=True): ...@@ -79,7 +79,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
break break
if get_output: if get_output:
buffer.append(data) buffer.append(data)
log(log_prefix + data.rstrip('\n')) log(log_prefix + (data if str is bytes else
data.decode('utf-8', errors='replace')).rstrip('\n'))
if p.stdout: if p.stdout:
stdout = [] stdout = []
stdout_thread = threading.Thread(target=readerthread, stdout_thread = threading.Thread(target=readerthread,
...@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True): ...@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
stdout_thread.join() stdout_thread.join()
if p.stderr: if p.stderr:
stderr_thread.join() stderr_thread.join()
return (p.stdout and ''.join(stdout), return (p.stdout and b''.join(stdout),
p.stderr and ''.join(stderr)) p.stderr and b''.join(stderr))
def killCommand(pid): def killCommand(pid):
""" """
...@@ -109,7 +110,7 @@ def killCommand(pid): ...@@ -109,7 +110,7 @@ def killCommand(pid):
try: try:
process = psutil.Process(pid) process = psutil.Process(pid)
process.suspend() process.suspend()
except psutil.Error, e: except psutil.Error as e:
return return
process_list = [process] process_list = [process]
new_list = process.children(recursive=True) new_list = process.children(recursive=True)
...@@ -118,19 +119,19 @@ def killCommand(pid): ...@@ -118,19 +119,19 @@ def killCommand(pid):
for child in new_list: for child in new_list:
try: try:
child.suspend() child.suspend()
except psutil.Error, e: except psutil.Error as e:
logger.debug("killCommand/suspend: %s", e) logger.debug("killCommand/suspend: %s", e)
time.sleep(1) time.sleep(1)
new_list = set(process.children(recursive=True)).difference(process_list) new_list = set(process.children(recursive=True)).difference(process_list)
for process in process_list: for process in process_list:
try: try:
process.kill() process.kill()
except psutil.Error, e: except psutil.Error as e:
logger.debug("killCommand/kill: %s", e) logger.debug("killCommand/kill: %s", e)
class ProcessManager(object): class ProcessManager(object):
stdin = file(os.devnull) stdin = open(os.devnull)
def __init__(self, max_timeout=MAX_TIMEOUT): def __init__(self, max_timeout=MAX_TIMEOUT):
self.process_pid_set = set() self.process_pid_set = set()
......
...@@ -30,30 +30,31 @@ import subprocess ...@@ -30,30 +30,31 @@ import subprocess
import sys import sys
import time import time
import glob import glob
import SlapOSControler from . import SlapOSControler, SlapOSMasterCommunicator
import SlapOSMasterCommunicator
import json import json
import time import time
import shutil import shutil
import logging import logging
import string import string
import random import random
import urlparse from six.moves.urllib.parse import urlparse
import base64 import base64
import httplib from six.moves import http_client as httplib
import Utils from . import Utils
import requests import requests
import slapos.slap import slapos.slap
import cPickle as pickle from six.moves import cPickle as pickle
from ProcessManager import SubprocessError, ProcessManager, CancellationError from .ProcessManager import SubprocessError, ProcessManager, CancellationError
from subprocess import CalledProcessError from subprocess import CalledProcessError
from Updater import Updater from .Updater import Updater
from erp5.util import taskdistribution from erp5.util import taskdistribution
from erp5.util.benchmark.thread import TestThread from erp5.util.benchmark.thread import TestThread
# for dummy slapos answer # for dummy slapos answer
import signal import signal
from . import logger from . import logger
from six.moves import range
# max time to generate frontend instance: 1.5 hour # max time to generate frontend instance: 1.5 hour
MAX_FRONTEND_TIME = 60*90 MAX_FRONTEND_TIME = 60*90
# max time to register instance to slapOSMaster: 5 minutes # max time to register instance to slapOSMaster: 5 minutes
...@@ -322,18 +323,16 @@ ces or already launched.") ...@@ -322,18 +323,16 @@ ces or already launched.")
software_hash_directory = self.testnode.config['slapos_binary'].rsplit("bin/slapos", 1)[0] software_hash_directory = self.testnode.config['slapos_binary'].rsplit("bin/slapos", 1)[0]
apache_htpasswd = software_hash_directory + "parts/apache/bin/htpasswd" apache_htpasswd = software_hash_directory + "parts/apache/bin/htpasswd"
testsuite_directory = self.testnode.config['repository_path_list'][0].rsplit('/', 1)[0] testsuite_directory = self.testnode.config['repository_path_list'][0].rsplit('/', 1)[0]
htaccess_file = open(testsuite_directory + HTACCESS, "w") with open(testsuite_directory + HTACCESS, "w") as htaccess_file:
file_content = """ htaccess_file.write("""
AuthType Basic AuthType Basic
AuthName "Password Protected Area" AuthName "Password Protected Area"
AuthUserFile "%s%s" AuthUserFile "%s%s"
Require valid-user Require valid-user
""" % (testsuite_directory, HTPASSWD) """ % (testsuite_directory, HTPASSWD))
htaccess_file.write(file_content)
htaccess_file.close()
password_path = testsuite_directory + PASSWORD_FILE password_path = testsuite_directory + PASSWORD_FILE
with open(password_path, "w") as password_file: with open(password_path, "w") as password_file:
password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH)) password = ''.join(random.choice(string.digits + string.ascii_letters) for i in range(PASSWORD_LENGTH))
password_file.write(password) password_file.write(password)
user = TESTNODE_USER user = TESTNODE_USER
command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password] command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password]
...@@ -363,7 +362,7 @@ Require valid-user ...@@ -363,7 +362,7 @@ Require valid-user
user, password = self.generateProfilePasswordAccess() user, password = self.generateProfilePasswordAccess()
logger.info("Software Profile password: %s" % password) logger.info("Software Profile password: %s" % password)
self.reachable_profile = "https://%s:%s@%s" % (user, password, self.reachable_profile = "https://%s:%s@%s" % (user, password,
os.path.join(urlparse.urlparse(self.testnode.config['frontend_url']).netloc, os.path.join(urlparse(self.testnode.config['frontend_url']).netloc,
"software", self.randomized_path, "software.cfg")) "software", self.randomized_path, "software.cfg"))
def prepareSlapOSForTestSuite(self, node_test_suite): def prepareSlapOSForTestSuite(self, node_test_suite):
...@@ -526,7 +525,7 @@ Require valid-user ...@@ -526,7 +525,7 @@ Require valid-user
if not self.launchable: if not self.launchable:
return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." } return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." }
configuration_list = node_test_suite.configuration_list configuration_list = node_test_suite.configuration_list
test_list = range(0, len(configuration_list)) test_list = list(range(len(configuration_list)))
try: try:
test_result_proxy = self.testnode.taskdistribution.createTestResult( test_result_proxy = self.testnode.taskdistribution.createTestResult(
node_test_suite.revision, test_list, node_test_suite.revision, test_list,
......
...@@ -35,6 +35,8 @@ from slapos import client ...@@ -35,6 +35,8 @@ from slapos import client
from . import logger from . import logger
from .Utils import createFolder from .Utils import createFolder
from six.moves import range
MAX_PARTITIONS = 10 MAX_PARTITIONS = 10
MAX_SR_RETRIES = 3 MAX_SR_RETRIES = 3
...@@ -243,7 +245,7 @@ class SlapOSControler(object): ...@@ -243,7 +245,7 @@ class SlapOSControler(object):
computer = slap.registerComputer(config['computer_id']) computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established # Call a method to ensure connection to master can be established
computer.getComputerPartitionList() computer.getComputerPartitionList()
except slapos.slap.ConnectionError, e: except slapos.slap.ConnectionError as e:
retries += 1 retries += 1
if retries >= 60: if retries >= 60:
raise raise
...@@ -270,7 +272,7 @@ class SlapOSControler(object): ...@@ -270,7 +272,7 @@ class SlapOSControler(object):
# MySQL DB content) from previous runs. To support changes of partition # MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level. # naming scheme (which already happened), do this at instance_root level.
createFolder(instance_root, True) createFolder(instance_root, True)
for i in xrange(MAX_PARTITIONS): for i in range(MAX_PARTITIONS):
# create partition and configure computer # create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address # XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports # this is not a problem as usually all services are on different ports
...@@ -278,7 +280,7 @@ class SlapOSControler(object): ...@@ -278,7 +280,7 @@ class SlapOSControler(object):
partition_path = os.path.join(instance_root, partition_reference) partition_path = os.path.join(instance_root, partition_reference)
if not(os.path.exists(partition_path)): if not(os.path.exists(partition_path)):
os.mkdir(partition_path) os.mkdir(partition_path)
os.chmod(partition_path, 0750) os.chmod(partition_path, 0o750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'], 'address': config['ipv4_address'],
'instance_root': instance_root, 'instance_root': instance_root,
...@@ -318,7 +320,7 @@ class SlapOSControler(object): ...@@ -318,7 +320,7 @@ class SlapOSControler(object):
os.environ['PATH'] = environment['PATH'] os.environ['PATH'] = environment['PATH']
# a SR may fail for number of reasons (incl. network failures) # a SR may fail for number of reasons (incl. network failures)
# so be tolerant and run it a few times before giving up # so be tolerant and run it a few times before giving up
for _ in xrange(MAX_SR_RETRIES): for _ in range(MAX_SR_RETRIES):
status_dict = self.spawn(config['slapos_binary'], status_dict = self.spawn(config['slapos_binary'],
'node', 'software', '--all', 'node', 'software', '--all',
'--pidfile', os.path.join(self.software_root, 'slapos-node.pid'), '--pidfile', os.path.join(self.software_root, 'slapos-node.pid'),
...@@ -346,7 +348,7 @@ class SlapOSControler(object): ...@@ -346,7 +348,7 @@ class SlapOSControler(object):
# try to run for all partitions as one partition may in theory request another one # try to run for all partitions as one partition may in theory request another one
# this not always is required but curently no way to know how "tree" of partitions # this not always is required but curently no way to know how "tree" of partitions
# may "expand" # may "expand"
for _ in xrange(max_quantity): for _ in range(max_quantity):
status_dict = self.spawn(config['slapos_binary'], 'node', 'instance', status_dict = self.spawn(config['slapos_binary'], 'node', 'instance',
'--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'), '--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'),
'--cfg', self.slapos_config, raise_error_if_fail=False, '--cfg', self.slapos_config, raise_error_if_fail=False,
......
from __future__ import print_function
import datetime import datetime
import json import json
import traceback import traceback
...@@ -12,6 +14,8 @@ from requests.exceptions import HTTPError ...@@ -12,6 +14,8 @@ from requests.exceptions import HTTPError
from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST
from . import logger from . import logger
import six
# max time to instance changing state: 3 hour # max time to instance changing state: 3 hour
MAX_INSTANCE_TIME = 60*60*3 MAX_INSTANCE_TIME = 60*60*3
...@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func, ...@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func,
except _except_list: except _except_list:
traceback.print_exc() traceback.print_exc()
print 'Network failure. Retry method %s in %i seconds' % (func, retry_time) print('Network failure. Retry method %s in %i seconds' % (func, retry_time))
time.sleep(retry_time) time.sleep(retry_time)
retry_time = min(retry_time*1.5, 640) retry_time = min(retry_time*1.5, 640)
...@@ -92,8 +96,9 @@ class SlapOSMasterCommunicator(object): ...@@ -92,8 +96,9 @@ class SlapOSMasterCommunicator(object):
if instance_title is not None: if instance_title is not None:
self.name = instance_title self.name = instance_title
if request_kw is not None: if request_kw is not None:
if isinstance(request_kw, basestring) or \ if isinstance(request_kw, bytes):
isinstance(request_kw, unicode): self.request_kw = json.loads(request_kw.decode('utf-8'))
elif isinstance(request_kw, six.text_type):
self.request_kw = json.loads(request_kw) self.request_kw = json.loads(request_kw)
else: else:
self.request_kw = request_kw self.request_kw = request_kw
...@@ -214,7 +219,7 @@ class SlapOSMasterCommunicator(object): ...@@ -214,7 +219,7 @@ class SlapOSMasterCommunicator(object):
result = self.hateoas_navigator.GET(url) result = self.hateoas_navigator.GET(url)
result = json.loads(result) result = json.loads(result)
if result['_links'].get('action_object_slap', None) is None: if result['_links'].get('action_object_slap', None) is None:
print result['links'] print(result['links'])
return None return None
object_link = self.hateoas_navigator.hateoasGetLinkFromLinks( object_link = self.hateoas_navigator.hateoasGetLinkFromLinks(
...@@ -385,8 +390,9 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -385,8 +390,9 @@ class SlapOSTester(SlapOSMasterCommunicator):
self.name = name self.name = name
self.computer_guid = computer_guid self.computer_guid = computer_guid
if isinstance(request_kw, str) or \ if isinstance(request_kw, bytes):
isinstance(request_kw, unicode): self.request_kw = json.loads(request_kw.decode('utf-8'))
elif isinstance(request_kw, six.text_type):
self.request_kw = json.loads(request_kw) self.request_kw = json.loads(request_kw)
else: else:
self.request_kw = request_kw self.request_kw = request_kw
......
...@@ -30,6 +30,7 @@ import re ...@@ -30,6 +30,7 @@ import re
from . import logger from . import logger
from .ProcessManager import SubprocessError from .ProcessManager import SubprocessError
from .Utils import rmtree from .Utils import rmtree
from slapos.util import bytes2str, str2bytes
SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$') SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE) SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
...@@ -82,7 +83,7 @@ class Updater(object): ...@@ -82,7 +83,7 @@ class Updater(object):
# allow several processes clean the same folder at the same time # allow several processes clean the same folder at the same time
try: try:
os.remove(os.path.join(path, file)) os.remove(os.path.join(path, file))
except OSError, e: except OSError as e:
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
raise raise
...@@ -96,7 +97,7 @@ class Updater(object): ...@@ -96,7 +97,7 @@ class Updater(object):
**kw) **kw)
def _git(self, *args, **kw): def _git(self, *args, **kw):
return self.spawn(self.git_binary, *args, **kw)['stdout'].strip() return bytes2str(self.spawn(self.git_binary, *args, **kw)['stdout'].strip())
def git_update_server_info(self): def git_update_server_info(self):
return self._git('update-server-info', '-f') return self._git('update-server-info', '-f')
...@@ -219,7 +220,7 @@ class Updater(object): ...@@ -219,7 +220,7 @@ class Updater(object):
self.deletePycFiles(path) self.deletePycFiles(path)
try: try:
status_dict = self.spawn(*args) status_dict = self.spawn(*args)
except SubprocessError, e: except SubprocessError as e:
if 'cleanup' not in e.stderr: if 'cleanup' not in e.stderr:
raise raise
self.spawn('svn', 'cleanup', path) self.spawn('svn', 'cleanup', path)
......
...@@ -3,6 +3,9 @@ import stat ...@@ -3,6 +3,9 @@ import stat
import shutil import shutil
import errno import errno
import six
from six.moves import map
def rmtree(path): def rmtree(path):
"""Delete a path recursively. """Delete a path recursively.
...@@ -11,14 +14,22 @@ def rmtree(path): ...@@ -11,14 +14,22 @@ def rmtree(path):
def chmod_retry(func, failed_path, exc_info): def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable. """Make sure the directories are executable and writable.
""" """
# Depending on the Python version, the following items differ.
if six.PY3:
expected_error_type = PermissionError
expected_func = os.lstat
else:
expected_error_type = OSError
expected_func = os.listdir
e = exc_info[1] e = exc_info[1]
if isinstance(e, OSError): if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT: if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might # because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree. # have been already deleted by the recursive call to rmtree.
return return
if e.errno == errno.EACCES: if e.errno == errno.EACCES:
if func is os.listdir: if func is expected_func:
os.chmod(failed_path, 0o700) os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories. # corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523 # https://bugs.python.org/issue8523
...@@ -39,12 +50,16 @@ def createFolder(folder, clean=False): ...@@ -39,12 +50,16 @@ def createFolder(folder, clean=False):
rmtree(folder) rmtree(folder)
os.mkdir(folder) os.mkdir(folder)
def deunicodeData(data): if six.PY3:
def deunicodeData(data):
return data
else:
def deunicodeData(data):
if isinstance(data, list): if isinstance(data, list):
return map(deunicodeData, data) return list(map(deunicodeData, data))
if isinstance(data, unicode): if isinstance(data, unicode):
return data.encode('utf8') return data.encode('utf8')
if isinstance(data, dict): if isinstance(data, dict):
return {deunicodeData(key): deunicodeData(value) return {deunicodeData(key): deunicodeData(value)
for key, value in data.iteritems()} for key, value in six.iteritems(data)}
return data return data
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import ConfigParser from six.moves import configparser
import argparse import argparse
import logging import logging
import logging.handlers import logging.handlers
...@@ -64,7 +64,7 @@ def main(*args): ...@@ -64,7 +64,7 @@ def main(*args):
CONFIG = { CONFIG = {
'partition_reference': 'test0', 'partition_reference': 'test0',
} }
config = ConfigParser.SafeConfigParser() config = configparser.SafeConfigParser()
# do not change case of option keys # do not change case of option keys
config.optionxform = str config.optionxform = str
config.readfp(parsed_argument.configuration_file[0]) config.readfp(parsed_argument.configuration_file[0])
......
...@@ -171,7 +171,7 @@ shared = true ...@@ -171,7 +171,7 @@ shared = true
# only limit to particular error, if we run that code for all errors, # only limit to particular error, if we run that code for all errors,
# then if server having most repositories is down for some time, we would # then if server having most repositories is down for some time, we would
# erase all repositories and facing later hours of downloads # erase all repositories and facing later hours of downloads
if getattr(error, 'stderr', '').find('index') >= 0: if b'index' in getattr(error, 'stderr', b''):
rmtree(repository_path) rmtree(repository_path)
logger.warning("Error while getting repository, ignoring this test suite", logger.warning("Error while getting repository, ignoring this test suite",
exc_info=1) exc_info=1)
......
from __future__ import print_function
import argparse import argparse
import re, os, shlex, glob import re, os, shlex, glob
import sys, threading, subprocess import sys, threading, subprocess
import traceback import traceback
import errno import errno
import pprint
import six
from six.moves import range
from erp5.util import taskdistribution from erp5.util import taskdistribution
from pprint import pprint
if six.PY3:
stdbin = lambda x: x.buffer
else:
stdbin = lambda x: x
# PY3: use shlex.quote
_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search _format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'")) _format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
def format_command(*args, **kw): def format_command(*args, **kw):
...@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False): ...@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False):
buffer.append(data) buffer.append(data)
if p.stdout: if p.stdout:
stdout = [] stdout = []
output = quiet and (lambda data: None) or sys.stdout.write output = (lambda data: None) if quiet else stdbin(sys.stdout).write
stdout_thread = threading.Thread(target=readerthread, stdout_thread = threading.Thread(target=readerthread,
args=(p.stdout, output, stdout)) args=(p.stdout, output, stdout))
stdout_thread.setDaemon(True) stdout_thread.setDaemon(True)
...@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False): ...@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False):
if p.stderr: if p.stderr:
stderr = [] stderr = []
stderr_thread = threading.Thread(target=readerthread, stderr_thread = threading.Thread(target=readerthread,
args=(p.stderr, sys.stderr.write, stderr)) args=(p.stderr, stdbin(sys.stderr).write, stderr))
stderr_thread.setDaemon(True) stderr_thread.setDaemon(True)
stderr_thread.start() stderr_thread.start()
if p.stdout: if p.stdout:
...@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False): ...@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False):
if p.stderr: if p.stderr:
stderr_thread.join() stderr_thread.join()
p.wait() p.wait()
return (p.stdout and ''.join(stdout), return (p.stdout and b''.join(stdout),
p.stderr and ''.join(stderr)) p.stderr and b''.join(stderr))
class SubprocessError(EnvironmentError): class SubprocessError(EnvironmentError):
def __init__(self, status_dict): def __init__(self, status_dict):
...@@ -72,15 +82,15 @@ class Persistent(object): ...@@ -72,15 +82,15 @@ class Persistent(object):
def __getattr__(self, attr): def __getattr__(self, attr):
if attr == '_db': if attr == '_db':
try: try:
db = file(self._filename, 'r+') db = open(self._filename, 'r+')
except IOError, e: except IOError as e:
if e.errno != errno.ENOENT: if e.errno != errno.ENOENT:
raise raise
db = file(self._filename, 'w+') db = open(self._filename, 'w+')
else: else:
try: try:
self.__dict__.update(eval(db.read())) self.__dict__.update(eval(db.read()))
except StandardError: except Exception:
pass pass
self._db = db self._db = db
return db return db
...@@ -89,7 +99,7 @@ class Persistent(object): ...@@ -89,7 +99,7 @@ class Persistent(object):
def sync(self): def sync(self):
self._db.seek(0) self._db.seek(0)
db = dict(x for x in self.__dict__.iteritems() if x[0][:1] != '_') db = dict(x for x in six.iteritems(self.__dict__) if x[0][:1] != '_')
pprint.pprint(db, self._db) pprint.pprint(db, self._db)
self._db.truncate() self._db.truncate()
...@@ -103,10 +113,10 @@ class TestSuite(object): ...@@ -103,10 +113,10 @@ class TestSuite(object):
""" """
RUN_RE = re.compile( RUN_RE = re.compile(
r'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s', br'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
re.DOTALL) re.DOTALL)
STATUS_RE = re.compile(r""" STATUS_RE = re.compile(br"""
(OK|FAILED)\s+\( (OK|FAILED)\s+\(
(failures=(?P<failures>\d+),?\s*)? (failures=(?P<failures>\d+),?\s*)?
(errors=(?P<errors>\d+),?\s*)? (errors=(?P<errors>\d+),?\s*)?
...@@ -117,7 +127,7 @@ class TestSuite(object): ...@@ -117,7 +127,7 @@ class TestSuite(object):
""", re.DOTALL | re.VERBOSE) """, re.DOTALL | re.VERBOSE)
SUB_STATUS_RE = re.compile( SUB_STATUS_RE = re.compile(
r"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+ br"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
(?P<failures>\d+)\s+Failures\s* (?P<failures>\d+)\s+Failures\s*
\(? \(?
(skipped=(?P<skips>\d+),?\s*)? (skipped=(?P<skips>\d+),?\s*)?
...@@ -130,7 +140,10 @@ class TestSuite(object): ...@@ -130,7 +140,10 @@ class TestSuite(object):
mysql_db_count = 1 mysql_db_count = 1
allow_restart = False allow_restart = False
realtime_output = True realtime_output = True
stdin = file(os.devnull) try: # PY3
stdin = subprocess.DEVNULL
except AttributeError:
stdin = open(os.devnull, 'rb')
def __init__(self, max_instance_count, **kw): def __init__(self, max_instance_count, **kw):
self.__dict__.update(kw) self.__dict__.update(kw)
...@@ -139,8 +152,8 @@ class TestSuite(object): ...@@ -139,8 +152,8 @@ class TestSuite(object):
self.acquire = pool.acquire self.acquire = pool.acquire
self.release = pool.release self.release = pool.release
self._instance = threading.local() self._instance = threading.local()
self._pool = max_instance_count == 1 and [None] or \ self._pool = [None] if max_instance_count == 1 else \
range(1, max_instance_count + 1) list(range(1, max_instance_count + 1))
self._ready = set() self._ready = set()
self.running = {} self.running = {}
if max_instance_count != 1: if max_instance_count != 1:
...@@ -185,13 +198,14 @@ class TestSuite(object): ...@@ -185,13 +198,14 @@ class TestSuite(object):
def spawn(self, *args, **kw): def spawn(self, *args, **kw):
quiet = kw.pop('quiet', False) quiet = kw.pop('quiet', False)
cwd = kw.pop('cwd', None)
env = kw and dict(os.environ, **kw) or None env = kw and dict(os.environ, **kw) or None
command = format_command(*args, **kw) command = format_command(*args, **kw)
print '\n$ ' + command print('\n$ ' + command)
sys.stdout.flush() sys.stdout.flush()
try: try:
p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE, p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env) stderr=subprocess.PIPE, env=env, cwd=cwd)
except Exception: except Exception:
# Catch any exception here, to warn user instead of beeing silent, # Catch any exception here, to warn user instead of beeing silent,
# by generating fake error result # by generating fake error result
...@@ -229,20 +243,11 @@ class EggTestSuite(TestSuite): ...@@ -229,20 +243,11 @@ class EggTestSuite(TestSuite):
The python interpreter is ``python_interpreter`` The python interpreter is ``python_interpreter``
""" """
def run(self, test): def run(self, test):
print test print(test)
original_dir = os.getcwd()
try:
os.chdir(self.egg_test_path_dict[test])
return self.runUnitTest(test)
finally:
os.chdir(original_dir)
def runUnitTest(self, *args, **kw):
try: try:
runUnitTest = "{python} setup.py test".format(python=self.python_interpreter) status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test',
args = tuple(shlex.split(runUnitTest)) cwd=self.egg_test_path_dict[test])
status_dict = self.spawn(*args, **kw) except SubprocessError as e:
except SubprocessError, e:
status_dict = e.status_dict status_dict = e.status_dict
test_log = status_dict['stderr'] test_log = status_dict['stderr']
search = self.RUN_RE.search(test_log) search = self.RUN_RE.search(test_log)
...@@ -270,7 +275,7 @@ class EggTestSuite(TestSuite): ...@@ -270,7 +275,7 @@ class EggTestSuite(TestSuite):
return status_dict return status_dict
def getTestList(self): def getTestList(self):
return self.egg_test_path_dict.keys() return list(self.egg_test_path_dict)
def runTestSuite(): def runTestSuite():
parser = argparse.ArgumentParser(description='Run a test suite.') parser = argparse.ArgumentParser(description='Run a test suite.')
...@@ -327,7 +332,7 @@ def runTestSuite(): ...@@ -327,7 +332,7 @@ def runTestSuite():
if test_result is not None: if test_result is not None:
assert revision == test_result.revision, (revision, test_result.revision) assert revision == test_result.revision, (revision, test_result.revision)
while suite.acquire(): while suite.acquire():
test = test_result.start(suite.running.keys()) test = test_result.start(list(suite.running))
if test is not None: if test is not None:
suite.start(test.name, lambda status_dict, __test=test: suite.start(test.name, lambda status_dict, __test=test:
__test.stop(**status_dict)) __test.stop(**status_dict))
......
...@@ -27,6 +27,8 @@ ...@@ -27,6 +27,8 @@
# #
############################################################################## ##############################################################################
from __future__ import division, print_function
import os import os
import sys import sys
import imp import imp
...@@ -126,7 +128,7 @@ def parseFile(filename, measure_dict): ...@@ -126,7 +128,7 @@ def parseFile(filename, measure_dict):
sys.stderr.flush() sys.stderr.flush()
match_list = LINE_PATTERN.findall(line) match_list = LINE_PATTERN.findall(line)
if len(match_list) != 1: if len(match_list) != 1:
print >>sys.stderr, 'Unparseable line: %s:%i %r' % (filename, line_number, line) print('Unparseable line: %s:%i %r' % (filename, line_number, line), file=sys.stderr)
else: else:
result, filter_id, date, duration = processLine(match_list[0], filename, line_number) result, filter_id, date, duration = processLine(match_list[0], filename, line_number)
# Possible result values & meaning: # Possible result values & meaning:
...@@ -135,20 +137,21 @@ def parseFile(filename, measure_dict): ...@@ -135,20 +137,21 @@ def parseFile(filename, measure_dict):
# (string): use & skip to next line # (string): use & skip to next line
if result is False: if result is False:
if debug: if debug:
print >>sys.stderr, '? %s:%i %r' % (filename, line_number, match_list[0]) print('? %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
elif result is True: elif result is True:
if debug: if debug:
print >>sys.stderr, '- %s:%i %r' % (filename, line_number, match_list[0]) print('- %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
skip_count += 1 skip_count += 1
else: else:
measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).append(int(duration)) measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).append(int(duration))
match_count += 1 match_count += 1
line = logfile.readline() line = logfile.readline()
print >>sys.stderr, '%i' % (line_number, ) print('%i' % (line_number, ), file=sys.stderr)
if line_number > 0: if line_number > 0:
duration = time() - begin duration = time() - begin
print >>sys.stderr, "Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \ print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
(match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration) (match_count, (match_count / line_number) * 100, skip_count, (skip_count / line_number) * 100, (line_number - match_count - skip_count), (1 - (match_count + skip_count) / line_number)) * 100, duration, line_number // duration),
file=sys.stderr)
debug = False debug = False
outfile_prefix = None outfile_prefix = None
...@@ -161,9 +164,9 @@ decimate_count = 1 ...@@ -161,9 +164,9 @@ decimate_count = 1
try: try:
opts, file_list = getopt.getopt(sys.argv[1:], '', ['debug', 'config=', 'prefix=', 'no-average', 'sum', 'load=', 'save=', 'decimate=']) opts, file_list = getopt.getopt(sys.argv[1:], '', ['debug', 'config=', 'prefix=', 'no-average', 'sum', 'load=', 'save=', 'decimate='])
except Exception, reason: except Exception as reason:
print >>sys.stderr, reason print(reason, file=sys.stderr)
print >>sys.stderr, usage print(usage, file=sys.stderr)
sys.exit(1) sys.exit(1)
for name, value in opts: for name, value in opts:
...@@ -185,7 +188,7 @@ for name, value in opts: ...@@ -185,7 +188,7 @@ for name, value in opts:
decimate_count = int(value) decimate_count = int(value)
if configuration is None: if configuration is None:
raise ValueError, '--config is mandatory' raise ValueError('--config is mandatory')
config_file = os.path.splitext(os.path.basename(configuration))[0] config_file = os.path.splitext(os.path.basename(configuration))[0]
config_path = [os.path.dirname(os.path.abspath(configuration))] + sys.path config_path = [os.path.dirname(os.path.abspath(configuration))] + sys.path
...@@ -203,26 +206,24 @@ file_number = 0 ...@@ -203,26 +206,24 @@ file_number = 0
measure_dict = {} measure_dict = {}
if len(load_file_name_list): if len(load_file_name_list):
for load_file_name in load_file_name_list: for load_file_name in load_file_name_list:
load_file = open(load_file_name) with open(load_file_name) as load_file:
temp_measure_dict = eval(load_file.read(), {}) temp_measure_dict = eval(load_file.read(), {})
load_file.close()
assert isinstance(measure_dict, dict) assert isinstance(measure_dict, dict)
for filter_id, result_dict in temp_measure_dict.iteritems(): for filter_id, result_dict in temp_measure_dict.iteritems():
for result, date_dict in result_dict.iteritems(): for result, date_dict in result_dict.iteritems():
for date, duration_list in date_dict.iteritems(): for date, duration_list in date_dict.iteritems():
measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list) measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list)
print >>sys.stderr, 'Previous processing result restored from %r' % (load_file_name, ) print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr)
for filename in file_list: for filename in file_list:
file_number += 1 file_number += 1
print >>sys.stderr, 'Processing %s [%i/%i]...' % (filename, file_number, file_count) print('Processing %s [%i/%i]...' % (filename, file_number, file_count), file=sys.stderr)
parseFile(filename, measure_dict) parseFile(filename, measure_dict)
if save_file_name is not None: if save_file_name is not None:
save_file = open(save_file_name, 'w') with open(save_file_name, 'w') as save_file:
save_file.write(repr(measure_dict)) save_file.write(repr(measure_dict))
save_file.close() print('Processing result saved to %r' % (save_file_name, ), file=sys.stderr)
print >>sys.stderr, 'Processing result saved to %r' % (save_file_name, )
if outfile_prefix is not None: if outfile_prefix is not None:
## Generate a list of all measures and a 2-levels dictionnary with date as key and measure dictionnary as value ## Generate a list of all measures and a 2-levels dictionnary with date as key and measure dictionnary as value
...@@ -252,9 +253,9 @@ if outfile_prefix is not None: ...@@ -252,9 +253,9 @@ if outfile_prefix is not None:
def renderOutput(data_format, filename_suffix): def renderOutput(data_format, filename_suffix):
for sheet_id, sheet_column_list in sheet_dict.iteritems(): for sheet_id, sheet_column_list in sheet_dict.iteritems():
outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix) outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix)
print >>sys.stderr, 'Writing to %r...' % (outfile_name, ) print('Writing to %r...' % (outfile_name, ), file=sys.stderr)
outfile = open(outfile_name, 'w') with open(outfile_name, 'w') as outfile:
print >>outfile, '"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ) print('"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ), file=outfile)
decimate_dict = {} decimate_dict = {}
decimate = 0 decimate = 0
for date in date_list: for date in date_list:
...@@ -262,11 +263,11 @@ if outfile_prefix is not None: ...@@ -262,11 +263,11 @@ if outfile_prefix is not None:
decimate_dict.setdefault(key, []).extend(value) decimate_dict.setdefault(key, []).extend(value)
decimate += 1 decimate += 1
if decimate == decimate_count: if decimate == decimate_count:
print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])) print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
decimate_dict = {} decimate_dict = {}
decimate = 0 decimate = 0
if len(decimate_dict): if len(decimate_dict):
print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])) print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
if do_average: if do_average:
renderOutput('=%(sum)i/%(count)i', 'avg') renderOutput('=%(sum)i/%(count)i', 'avg')
......
...@@ -27,6 +27,8 @@ ...@@ -27,6 +27,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import division, print_function
from datetime import date from datetime import date
from os import path from os import path
import rpy2.robjects as robjects import rpy2.robjects as robjects
...@@ -82,9 +84,9 @@ class CSVFile(object): ...@@ -82,9 +84,9 @@ class CSVFile(object):
if cell > value_max.get(key, 0): if cell > value_max.get(key, 0):
value_max[key] = cell value_max[key] = cell
column_dict[key].append(cell) column_dict[key].append(cell)
line_num = float(line_num) / 100 line_num = line_num / 100
for key in ratio_dict: for key in ratio_dict:
ratio_dict[key] /= line_num ratio_dict[key] //= line_num
def getColumn(self, column_id): def getColumn(self, column_id):
return self.column_dict[self.column_list[column_id]] return self.column_dict[self.column_list[column_id]]
...@@ -101,7 +103,7 @@ def computeExpr(expr): ...@@ -101,7 +103,7 @@ def computeExpr(expr):
if expr: if expr:
assert expr[0] == '=' assert expr[0] == '='
num, denom = expr[1:].split('/') num, denom = expr[1:].split('/')
result = float(int(num)) / int(denom) result = int(num) / int(denom)
else: else:
result = None result = None
return result return result
...@@ -121,7 +123,7 @@ def main(): ...@@ -121,7 +123,7 @@ def main():
current_dir = os.getcwd() current_dir = os.getcwd()
for file_name in file_name_list: for file_name in file_name_list:
print 'Loading %s...' % (file_name, ) print('Loading %s...' % (file_name, ))
file = CSVFile(file_name) file = CSVFile(file_name)
date_string_list = file.getColumn(0) date_string_list = file.getColumn(0)
...@@ -134,7 +136,7 @@ def main(): ...@@ -134,7 +136,7 @@ def main():
# date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...] # date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...]
factor = 1 factor = 1
if len(date_string_list) > 20: if len(date_string_list) > 20:
factor = int(len(date_string_list) / 20) factor = int(len(date_string_list) // 20)
i = 0 i = 0
for date_string in date_string_list: for date_string in date_string_list:
if i % factor == 0: if i % factor == 0:
...@@ -183,13 +185,13 @@ def main(): ...@@ -183,13 +185,13 @@ def main():
y_data.append(value) y_data.append(value)
i += 1 i += 1
if len(x_data) == 0: if len(x_data) == 0:
print 'Nothing to plot for %s...' % (out_file_name, ) print('Nothing to plot for %s...' % (out_file_name, ))
continue continue
if options.minimal_non_empty_ratio is not None: if options.minimal_non_empty_ratio is not None:
column_len = len(column) column_len = len(column)
if column_len: if column_len:
if float(len(x_data))/column_len < options.minimal_non_empty_ratio: if len(x_data) / column_len < options.minimal_non_empty_ratio:
print 'Not enough values to plot for %s...' % (out_file_name, ) print('Not enough values to plot for %s...' % (out_file_name, ))
continue continue
r_y_data = robjects.FloatVector(y_data) r_y_data = robjects.FloatVector(y_data)
r_x_data = robjects.FloatVector(x_data) r_x_data = robjects.FloatVector(x_data)
...@@ -220,7 +222,7 @@ def main(): ...@@ -220,7 +222,7 @@ def main():
# stop changing the out-type file # stop changing the out-type file
r("""dev.off()""") r("""dev.off()""")
print 'Saving %s...' % (out_file_name, ) print('Saving %s...' % (out_file_name, ))
if __name__ == '__main__': if __name__ == '__main__':
main() main()
......
...@@ -26,6 +26,8 @@ ...@@ -26,6 +26,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import print_function
import os import os
import shutil import shutil
import sys import sys
...@@ -543,7 +545,7 @@ class HTTPCacheCheckerTestSuite(object): ...@@ -543,7 +545,7 @@ class HTTPCacheCheckerTestSuite(object):
from optparse import OptionParser from optparse import OptionParser
import ConfigParser from six.moves import configparser
def _formatConfiguration(configuration): def _formatConfiguration(configuration):
""" format the configuration""" """ format the configuration"""
...@@ -559,11 +561,11 @@ def web_checker_utility(): ...@@ -559,11 +561,11 @@ def web_checker_utility():
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if len(args) != 1 : if len(args) != 1 :
print parser.print_help() print(parser.print_help())
parser.error('incorrect number of arguments') parser.error('incorrect number of arguments')
config_path = args[0] config_path = args[0]
config = ConfigParser.RawConfigParser() config = configparser.RawConfigParser()
config.read(config_path) config.read(config_path)
working_directory = config.get('web_checker', 'working_directory') working_directory = config.get('web_checker', 'working_directory')
url = config.get('web_checker', 'url') url = config.get('web_checker', 'url')
...@@ -615,10 +617,9 @@ def web_checker_utility(): ...@@ -615,10 +617,9 @@ def web_checker_utility():
result = instance.start(prohibited_file_name_list=prohibited_file_name_list, result = instance.start(prohibited_file_name_list=prohibited_file_name_list,
prohibited_folder_name_list=prohibited_folder_name_list) prohibited_folder_name_list=prohibited_folder_name_list)
if options.output_file: if options.output_file:
file_object = open(options.output_file, 'w') with open(options.output_file, 'w') as file_object:
file_object.write(result) file_object.write(result)
file_object.close()
else: else:
print result print(result)
...@@ -35,12 +35,24 @@ import posixpath ...@@ -35,12 +35,24 @@ import posixpath
import tarfile import tarfile
import os import os
import sys import sys
import cgi try:
from html import escape
except ImportError:
from cgi import escape # Deprecated since version 3.2
from base64 import b64encode from base64 import b64encode
from cStringIO import StringIO from io import BytesIO
from hashlib import sha1 from hashlib import sha1
from urllib import unquote try:
from urllib.parse import unquote
except ImportError:
from urllib import unquote
if sys.version_info[0] == 3:
def iteritems(d):
return iter(d.items())
else:
def iteritems(d):
return d.iteritems()
# Order is important for installation # Order is important for installation
# We want to have: # We want to have:
...@@ -109,11 +121,11 @@ item_name_list = tuple('_%s_item' % x for x in item_name_list) ...@@ -109,11 +121,11 @@ item_name_list = tuple('_%s_item' % x for x in item_name_list)
class BusinessTemplateRevision(list): class BusinessTemplateRevision(list):
def hash(self, path, text): def hash(self, path, text):
self.append((path, sha1(text).digest())) self.append((path.encode('utf-8'), sha1(text).digest()))
def digest(self): def digest(self):
self.sort() self.sort()
return b64encode(sha1('\0'.join(h + p for (h, p) in self)).digest()) return b64encode(sha1(b'\0'.join(h + p for (h, p) in self)).digest())
class BusinessTemplate(dict): class BusinessTemplate(dict):
...@@ -151,7 +163,7 @@ force_install ...@@ -151,7 +163,7 @@ force_install
def __iter__(self): def __iter__(self):
self['revision'] = self.revision.digest() self['revision'] = self.revision.digest()
return iter(sorted(self.iteritems())) return iter(sorted(iteritems(self)))
@classmethod @classmethod
def fromTar(cls, tar): def fromTar(cls, tar):
...@@ -179,8 +191,8 @@ force_install ...@@ -179,8 +191,8 @@ force_install
return iter(self) return iter(self)
def generateInformation(dir, info=id, err=None): def generateInformation(dir, info=id, err=None):
xml = StringIO() xml = BytesIO()
xml.write('<?xml version="1.0"?>\n<repository>\n') xml.write(b'<?xml version="1.0"?>\n<repository>\n')
for name in sorted(os.listdir(dir)): for name in sorted(os.listdir(dir)):
path = os.path.join(dir, name) path = os.path.join(dir, name)
if name.endswith('.bt5'): if name.endswith('.bt5'):
...@@ -201,13 +213,16 @@ def generateInformation(dir, info=id, err=None): ...@@ -201,13 +213,16 @@ def generateInformation(dir, info=id, err=None):
property_list = BusinessTemplate.fromDir(path) property_list = BusinessTemplate.fromDir(path)
else: else:
continue continue
xml.write(' <template id="%s">\n' % name) xml.write(b' <template id="%s">\n' % name.encode())
for k, v in property_list: for k, v in property_list:
for v in (v,) if type(v) is str else v: if str is not bytes:
xml.write(' <%s>%s</%s>\n' % (k, cgi.escape(v), k)) k = k.encode()
xml.write(' </template>\n') for v in (v,) if type(v) is bytes else v:
xml.write(b' <%s>%s</%s>\n' % (k, escape(v) if str is bytes else
escape(v.decode()).encode(), k))
xml.write(b' </template>\n')
info('done\n') info('done\n')
xml.write('</repository>\n') xml.write(b'</repository>\n')
return xml return xml
def main(dir_list=None, **kw): def main(dir_list=None, **kw):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment