Commit 7467ef4c authored by Bryton Lacquement's avatar Bryton Lacquement 🚪 Committed by Julien Muchembled

Add Python 3 support

parent ae15060e
...@@ -45,6 +45,6 @@ def do_bang(configp, message): ...@@ -45,6 +45,6 @@ def do_bang(configp, message):
slap = slapos.slap.slap() slap = slapos.slap.slap()
slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file) slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
computer = slap.registerComputer(computer_id) computer = slap.registerComputer(computer_id)
print 'Banging to %r' % master_url print('Banging to %r' % master_url)
computer.bang(message) computer.bang(message)
print 'Bang with message %r' % message print('Bang with message %r' % message)
...@@ -27,8 +27,10 @@ ...@@ -27,8 +27,10 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import subprocess import subprocess
import urlparse from six.moves.urllib.parse import urlparse
from time import sleep from time import sleep
import glob import glob
import os import os
...@@ -46,14 +48,14 @@ def _removeTimestamp(instancehome): ...@@ -46,14 +48,14 @@ def _removeTimestamp(instancehome):
""" """
timestamp_glob_path = "%s/slappart*/.timestamp" % instancehome timestamp_glob_path = "%s/slappart*/.timestamp" % instancehome
for timestamp_path in glob.glob(timestamp_glob_path): for timestamp_path in glob.glob(timestamp_glob_path):
print "Removing %s" % timestamp_path print("Removing %s" % timestamp_path)
os.remove(timestamp_path) os.remove(timestamp_path)
def _runBang(app): def _runBang(app):
""" """
Launch slapos node format. Launch slapos node format.
""" """
print "[BOOT] Invoking slapos node bang..." print("[BOOT] Invoking slapos node bang...")
result = app.run(['node', 'bang', '-m', 'Reboot']) result = app.run(['node', 'bang', '-m', 'Reboot'])
if result == 1: if result == 1:
return 0 return 0
...@@ -63,7 +65,7 @@ def _runFormat(app): ...@@ -63,7 +65,7 @@ def _runFormat(app):
""" """
Launch slapos node format. Launch slapos node format.
""" """
print "[BOOT] Invoking slapos node format..." print("[BOOT] Invoking slapos node format...")
result = app.run(['node', 'format', '--now', '--verbose']) result = app.run(['node', 'format', '--now', '--verbose'])
if result == 1: if result == 1:
return 0 return 0
...@@ -73,30 +75,30 @@ def _ping(hostname): ...@@ -73,30 +75,30 @@ def _ping(hostname):
""" """
Ping a hostname Ping a hostname
""" """
print "[BOOT] Invoking ipv4 ping to %s..." % hostname print("[BOOT] Invoking ipv4 ping to %s..." % hostname)
p = subprocess.Popen( p = subprocess.Popen(
["ping", "-c", "2", hostname], ["ping", "-c", "2", hostname],
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
if p.returncode == 0: if p.returncode == 0:
print "[BOOT] IPv4 network reachable..." print("[BOOT] IPv4 network reachable...")
return 1 return 1
print "[BOOT] [ERROR] IPv4 network unreachable..." print("[BOOT] [ERROR] IPv4 network unreachable...")
return 0 return 0
def _ping6(hostname): def _ping6(hostname):
""" """
Ping an ipv6 address Ping an ipv6 address
""" """
print "[BOOT] Invoking ipv6 ping to %s..." % hostname print("[BOOT] Invoking ipv6 ping to %s..." % hostname)
p = subprocess.Popen( p = subprocess.Popen(
["ping6", "-c", "2", hostname], ["ping6", "-c", "2", hostname],
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
if p.returncode == 0: if p.returncode == 0:
print "[BOOT] IPv6 network reachable..." print("[BOOT] IPv6 network reachable...")
return 1 return 1
print "[BOOT] [ERROR] IPv6 network unreachable..." print("[BOOT] [ERROR] IPv6 network unreachable...")
return 0 return 0
def _test_ping(hostname): def _test_ping(hostname):
...@@ -126,7 +128,7 @@ def _waitIpv6Ready(ipv6_interface): ...@@ -126,7 +128,7 @@ def _waitIpv6Ready(ipv6_interface):
test if ipv6 is ready on ipv6_interface test if ipv6 is ready on ipv6_interface
""" """
ipv6_address = "" ipv6_address = ""
print "[BOOT] Checking if %r has IPv6..." % ipv6_interface print("[BOOT] Checking if %r has IPv6..." % ipv6_interface)
while ipv6_address == "": while ipv6_address == "":
for inet_dict in netifaces.ifaddresses(ipv6_interface)[socket.AF_INET6]: for inet_dict in netifaces.ifaddresses(ipv6_interface)[socket.AF_INET6]:
ipv6_address = inet_dict['addr'].split('%')[0] ipv6_address = inet_dict['addr'].split('%')[0]
...@@ -134,8 +136,8 @@ def _waitIpv6Ready(ipv6_interface): ...@@ -134,8 +136,8 @@ def _waitIpv6Ready(ipv6_interface):
break break
else: else:
ipv6_address = "" ipv6_address = ""
print "[BOOT] [ERROR] No IPv6 found on interface %r, " \ print("[BOOT] [ERROR] No IPv6 found on interface %r, "
"try again in 5 seconds..." % ipv6_interface "try again in 5 seconds..." % ipv6_interface))
sleep(5) sleep(5)
class BootCommand(ConfigCommand): class BootCommand(ConfigCommand):
...@@ -155,7 +157,7 @@ class BootCommand(ConfigCommand): ...@@ -155,7 +157,7 @@ class BootCommand(ConfigCommand):
def take_action(self, args): def take_action(self, args):
configp = self.fetch_config(args) configp = self.fetch_config(args)
instance_root = configp.get('slapos','instance_root') instance_root = configp.get('slapos','instance_root')
master_url = urlparse.urlparse(configp.get('slapos','master_url')) master_url = urlparse(configp.get('slapos','master_url'))
master_hostname = master_url.hostname master_hostname = master_url.hostname
# Check that we have IPv6 ready # Check that we have IPv6 ready
...@@ -177,12 +179,12 @@ class BootCommand(ConfigCommand): ...@@ -177,12 +179,12 @@ class BootCommand(ConfigCommand):
app = SlapOSApp() app = SlapOSApp()
# Make sure slapos node format returns ok # Make sure slapos node format returns ok
while not _runFormat(app): while not _runFormat(app):
print "[BOOT] [ERROR] Fail to format, try again in 15 seconds..." print("[BOOT] [ERROR] Fail to format, try again in 15 seconds...")
sleep(15) sleep(15)
# Make sure slapos node bang returns ok # Make sure slapos node bang returns ok
while not _runBang(app): while not _runBang(app):
print "[BOOT] [ERROR] Fail to bang, try again in 15 seconds..." print("[BOOT] [ERROR] Fail to bang, try again in 15 seconds...")
sleep(15) sleep(15)
_removeTimestamp(instance_root) _removeTimestamp(instance_root)
...@@ -27,7 +27,7 @@ ...@@ -27,7 +27,7 @@
# #
############################################################################## ##############################################################################
import ConfigParser from six.moves import configparser
import os import os
from slapos.cli.command import Command from slapos.cli.command import Command
...@@ -77,7 +77,7 @@ class ConfigCommand(Command): ...@@ -77,7 +77,7 @@ class ConfigCommand(Command):
if not os.path.exists(cfg_path): if not os.path.exists(cfg_path):
raise ConfigError('Configuration file does not exist: %s' % cfg_path) raise ConfigError('Configuration file does not exist: %s' % cfg_path)
configp = ConfigParser.SafeConfigParser() configp = configparser.SafeConfigParser()
if configp.read(cfg_path) != [cfg_path]: if configp.read(cfg_path) != [cfg_path]:
# bad permission, etc. # bad permission, etc.
raise ConfigError('Cannot parse configuration file: %s' % cfg_path) raise ConfigError('Cannot parse configuration file: %s' % cfg_path)
......
...@@ -31,6 +31,7 @@ import re ...@@ -31,6 +31,7 @@ import re
import os import os
import sys import sys
import json import json
from six.moves import input
import requests import requests
...@@ -94,7 +95,7 @@ def fetch_configuration_template(): ...@@ -94,7 +95,7 @@ def fetch_configuration_template():
def do_configure_client(logger, master_url_web, token, config_path, master_url): def do_configure_client(logger, master_url_web, token, config_path, master_url):
while not token: while not token:
token = raw_input('Credential security token: ').strip() token = input('Credential security token: ').strip()
# Check for existence of previous configuration, certificate or key files # Check for existence of previous configuration, certificate or key files
# where we expect to create them. If so, ask the use to manually remove them. # where we expect to create them. If so, ask the use to manually remove them.
......
...@@ -39,6 +39,7 @@ from slapos.cli.config import ConfigCommand ...@@ -39,6 +39,7 @@ from slapos.cli.config import ConfigCommand
from slapos.grid.slapgrid import create_slapgrid_object from slapos.grid.slapgrid import create_slapgrid_object
from slapos.grid.utils import updateFile, createPrivateDirectory from slapos.grid.utils import updateFile, createPrivateDirectory
from slapos.grid.svcbackend import launchSupervisord from slapos.grid.svcbackend import launchSupervisord
from slapos.util import bytes2str
DEFAULT_COMPUTER_ID = 'local_computer' DEFAULT_COMPUTER_ID = 'local_computer'
...@@ -126,8 +127,8 @@ def _replaceParameterValue(original_content, to_replace): ...@@ -126,8 +127,8 @@ def _replaceParameterValue(original_content, to_replace):
def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args): def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
template_arg_list = (__name__, '../../slapos.cfg.example') template_arg_list = (__name__, '../../slapos.cfg.example')
with pkg_resources.resource_stream(*template_arg_list) as fout: slapos_node_configuration_template = \
slapos_node_configuration_template = fout.read() bytes2str(pkg_resources.resource_string(*template_arg_list))
master_url = 'http://%s:%s' % (args.daemon_listen_ip, args.daemon_listen_port) master_url = 'http://%s:%s' % (args.daemon_listen_ip, args.daemon_listen_port)
slapos_home = args.slapos_buildout_directory slapos_home = args.slapos_buildout_directory
to_replace = [ to_replace = [
...@@ -149,12 +150,12 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args): ...@@ -149,12 +150,12 @@ def _generateSlaposNodeConfigurationFile(slapos_node_config_path, args):
'(key_file|cert_file|certificate_repository_path).*=.*\n', '(key_file|cert_file|certificate_repository_path).*=.*\n',
'', slapos_node_configuration_content) '', slapos_node_configuration_content)
with open(slapos_node_config_path, 'w') as fout: with open(slapos_node_config_path, 'w') as fout:
fout.write(slapos_node_configuration_content.encode('utf8')) fout.write(slapos_node_configuration_content)
def _generateSlaposProxyConfigurationFile(conf): def _generateSlaposProxyConfigurationFile(conf):
template_arg_list = (__name__, '../../slapos-proxy.cfg.example') template_arg_list = (__name__, '../../slapos-proxy.cfg.example')
with pkg_resources.resource_stream(*template_arg_list) as fout: slapos_proxy_configuration_template = \
slapos_proxy_configuration_template = fout.read() bytes2str(pkg_resources.resource_string(*template_arg_list))
slapos_proxy_configuration_path = os.path.join( slapos_proxy_configuration_path = os.path.join(
conf.slapos_configuration_directory, 'slapos-proxy.cfg') conf.slapos_configuration_directory, 'slapos-proxy.cfg')
listening_ip, listening_port = \ listening_ip, listening_port = \
...@@ -172,7 +173,7 @@ def _generateSlaposProxyConfigurationFile(conf): ...@@ -172,7 +173,7 @@ def _generateSlaposProxyConfigurationFile(conf):
slapos_proxy_configuration_template, to_replace) slapos_proxy_configuration_template, to_replace)
with open(slapos_proxy_configuration_path, 'w') as fout: with open(slapos_proxy_configuration_path, 'w') as fout:
fout.write(slapos_proxy_configuration_content.encode('utf8')) fout.write(slapos_proxy_configuration_content)
return slapos_proxy_configuration_path return slapos_proxy_configuration_path
......
...@@ -32,6 +32,8 @@ import textwrap ...@@ -32,6 +32,8 @@ import textwrap
from slapos.cli.config import ClientConfigCommand from slapos.cli.config import ClientConfigCommand
from slapos.client import init, do_console, ClientConfig from slapos.client import init, do_console, ClientConfig
from six import exec_
class ShellNotFound(Exception): class ShellNotFound(Exception):
pass pass
...@@ -90,7 +92,9 @@ class ConsoleCommand(ClientConfigCommand): ...@@ -90,7 +92,9 @@ class ConsoleCommand(ClientConfigCommand):
local = init(conf, self.app.log) local = init(conf, self.app.log)
if args.script_file: if args.script_file:
return execfile(args.script_file, globals(), local) with open(args.script_file) as f:
code = compile(f.read(), args.script_file, 'exec')
return exec_(code, globals(), local)
if not any([args.python, args.ipython, args.bpython]): if not any([args.python, args.ipython, args.bpython]):
args.ipython = True args.ipython = True
......
...@@ -247,7 +247,7 @@ class SlapOSApp(App): ...@@ -247,7 +247,7 @@ class SlapOSApp(App):
return return
if self.options.log_color: if self.options.log_color:
import coloredlogs from slapos.cli import coloredlogs
console = coloredlogs.ColoredStreamHandler(show_name=True, # logger name (slapos) and PID console = coloredlogs.ColoredStreamHandler(show_name=True, # logger name (slapos) and PID
show_severity=True, show_severity=True,
show_timestamps=self.options.log_time, show_timestamps=self.options.log_time,
......
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
import logging import logging
import sys import sys
import six
from slapos.cli.config import ClientConfigCommand from slapos.cli.config import ClientConfigCommand
from slapos.client import init, ClientConfig from slapos.client import init, ClientConfig
...@@ -64,5 +65,5 @@ def do_list(logger, conf, local): ...@@ -64,5 +65,5 @@ def do_list(logger, conf, local):
logger.info('No existing service.') logger.info('No existing service.')
return return
logger.info('List of services:') logger.info('List of services:')
for title, instance in instance_dict.iteritems(): for title, instance in six.iteritems(instance_dict):
logger.info('%s %s', title, instance._software_release_url) logger.info('%s %s', title, instance._software_release_url)
...@@ -34,7 +34,6 @@ import logging ...@@ -34,7 +34,6 @@ import logging
import sys import sys
import os import os
import subprocess import subprocess
import StringIO
import lxml.etree import lxml.etree
import prettytable import prettytable
...@@ -43,8 +42,17 @@ import sqlite3 ...@@ -43,8 +42,17 @@ import sqlite3
from slapos.cli.config import ConfigCommand from slapos.cli.config import ConfigCommand
from slapos.proxy import ProxyConfig from slapos.proxy import ProxyConfig
from slapos.proxy.db_version import DB_VERSION from slapos.proxy.db_version import DB_VERSION
from slapos.util import sqlite_connect from slapos.util import sqlite_connect, str2bytes
if bytes is str:
from io import BytesIO
class StringIO(BytesIO):
# Something between strict io.BytesIO and laxist/slow StringIO.StringIO
# (which starts returning unicode once unicode is written) for logging.
def write(self, b):
return BytesIO.write(self, b.encode('utf-8'))
else:
from io import StringIO
class ProxyShowCommand(ConfigCommand): class ProxyShowCommand(ConfigCommand):
""" """
...@@ -139,7 +147,7 @@ def log_params(logger, conn): ...@@ -139,7 +147,7 @@ def log_params(logger, conn):
if not row['connection_xml']: if not row['connection_xml']:
continue continue
xml = str(row['connection_xml']) xml = str2bytes(row['connection_xml'])
logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type']) logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type'])
instance = lxml.etree.fromstring(xml) instance = lxml.etree.fromstring(xml)
for parameter in list(instance): for parameter in list(instance):
...@@ -150,9 +158,12 @@ def log_params(logger, conn): ...@@ -150,9 +158,12 @@ def log_params(logger, conn):
# _ is usually json encoded - re-format to make it easier to read # _ is usually json encoded - re-format to make it easier to read
if name == '_': if name == '_':
try: try:
text = json.dumps(json.loads(text), indent=2) text = json.dumps(json.loads(text),
indent=2, sort_keys=True)
except ValueError: except ValueError:
pass pass
else: # to avoid differences between Py2 and Py3 in unit tests
text = '\n'.join(map(str.rstrip, text.splitlines()))
logger.info(' %s = %s', name, text) logger.info(' %s = %s', name, text)
...@@ -208,7 +219,7 @@ def do_show(conf): ...@@ -208,7 +219,7 @@ def do_show(conf):
# to paginate input, honoring $PAGER. # to paginate input, honoring $PAGER.
output = sys.stdout output = sys.stdout
if output.isatty(): if output.isatty():
output = StringIO.StringIO() output = StringIO()
proxy_show_logger = logging.getLogger(__name__) proxy_show_logger = logging.getLogger(__name__)
handler = logging.StreamHandler(output) handler = logging.StreamHandler(output)
handler.setLevel(logging.DEBUG) handler.setLevel(logging.DEBUG)
...@@ -221,7 +232,8 @@ def do_show(conf): ...@@ -221,7 +232,8 @@ def do_show(conf):
conn = sqlite_connect(conf.database_uri) conn = sqlite_connect(conf.database_uri)
conn.row_factory = sqlite3.Row conn.row_factory = sqlite3.Row
conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest()) conn.create_function('md5', 1,
lambda s: hashlib.md5(str2bytes(s)).hexdigest())
call_table = [ call_table = [
(conf.computers, log_computer_table), (conf.computers, log_computer_table),
...@@ -248,4 +260,4 @@ def do_show(conf): ...@@ -248,4 +260,4 @@ def do_show(conf):
close_fds=True, close_fds=True,
shell=True, shell=True,
stdin=subprocess.PIPE,) stdin=subprocess.PIPE,)
pager.communicate(output.getvalue().encode('utf-8')) pager.communicate(str2bytes(output.getvalue()))
...@@ -36,6 +36,7 @@ import sys ...@@ -36,6 +36,7 @@ import sys
import pkg_resources import pkg_resources
import requests import requests
import json import json
from six.moves import input
from slapos.cli.command import Command, must_be_root from slapos.cli.command import Command, must_be_root
...@@ -327,7 +328,7 @@ def gen_auth(conf): ...@@ -327,7 +328,7 @@ def gen_auth(conf):
else: else:
yield conf.login, getpass.getpass() yield conf.login, getpass.getpass()
while ask: while ask:
yield raw_input('SlapOS Master Login: '), getpass.getpass() yield input('SlapOS Master Login: '), getpass.getpass()
def do_register(conf): def do_register(conf):
...@@ -348,7 +349,7 @@ def do_register(conf): ...@@ -348,7 +349,7 @@ def do_register(conf):
password=password) password=password)
else: else:
while not conf.token: while not conf.token:
conf.token = raw_input('Computer security token: ').strip() conf.token = input('Computer security token: ').strip()
certificate, key = get_certificate_key_pair(conf.logger, certificate, key = get_certificate_key_pair(conf.logger,
conf.master_url_web, conf.master_url_web,
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
############################################################################## ##############################################################################
import atexit import atexit
import ConfigParser from six.moves import configparser
import os import os
import sys import sys
...@@ -52,7 +52,7 @@ class ClientConfig(object): ...@@ -52,7 +52,7 @@ class ClientConfig(object):
# Merges the arguments and configuration # Merges the arguments and configuration
try: try:
configuration_dict = dict(configp.items('slapconsole')) configuration_dict = dict(configp.items('slapconsole'))
except ConfigParser.NoSectionError: except configparser.NoSectionError:
pass pass
else: else:
for key in configuration_dict: for key in configuration_dict:
...@@ -119,7 +119,7 @@ def _getSoftwareReleaseFromSoftwareString(logger, software_string, product): ...@@ -119,7 +119,7 @@ def _getSoftwareReleaseFromSoftwareString(logger, software_string, product):
try: try:
return product.__getattr__(software_string[len(SOFTWARE_PRODUCT_NAMESPACE):]) return product.__getattr__(software_string[len(SOFTWARE_PRODUCT_NAMESPACE):])
except AttributeError as e: except AttributeError as e:
logger.error('Error: %s Exiting now.' % e.message) logger.error('Error: %s Exiting now.', e)
sys.exit(1) sys.exit(1)
def do_console(local): def do_console(local):
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
from psutil import process_iter, NoSuchProcess, AccessDenied from psutil import process_iter, NoSuchProcess, AccessDenied
from time import strftime from time import strftime
import shutil import shutil
...@@ -42,7 +43,7 @@ from slapos.collect.reporter import RawCSVDumper, \ ...@@ -42,7 +43,7 @@ from slapos.collect.reporter import RawCSVDumper, \
compressLogFolder, \ compressLogFolder, \
ConsumptionReport ConsumptionReport
from entity import get_user_list, Computer from .entity import get_user_list, Computer
def _get_time(): def _get_time():
return strftime("%Y-%m-%d -- %H:%M:%S").split(" -- ") return strftime("%Y-%m-%d -- %H:%M:%S").split(" -- ")
...@@ -157,5 +158,5 @@ def do_collect(conf): ...@@ -157,5 +158,5 @@ def do_collect(conf):
database.garbageCollect() database.garbageCollect()
except AccessDenied: except AccessDenied:
print "You HAVE TO execute this script with root permission." print("You HAVE TO execute this script with root permission.")
...@@ -74,19 +74,15 @@ class User(object): ...@@ -74,19 +74,15 @@ class User(object):
time_cycle = self.disk_snapshot_params.get('time_cycle', 0) time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
database.connect() database.connect()
if time_cycle: if time_cycle:
order = 'date DESC, time DESC' for date_time in database.select(table="folder", columns="date, time",
limit = 1 order='date DESC, time DESC', limit=1,
query = database.select(table="folder", columns="date, time", where="partition='%s'" % self.name):
order=order, limit=limit, latest_date = datetime.strptime('%s %s' % date_time,
where="partition='%s'" % self.name) "%Y-%m-%d %H:%M:%S")
query_result = zip(*query)
if len(query_result):
date, time = (query_result[0][0], query_result[1][0])
latest_date = datetime.strptime('%s %s' % (date, time),
"%Y-%m-%d %H:%M:%S")
if (datetime.now() - latest_date).seconds < time_cycle: if (datetime.now() - latest_date).seconds < time_cycle:
# wait the time cycle # wait the time cycle
return return
break
pid_file = self.disk_snapshot_params.get('pid_folder', None) pid_file = self.disk_snapshot_params.get('pid_folder', None)
if pid_file is not None: if pid_file is not None:
pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name) pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)
......
...@@ -40,6 +40,8 @@ import tarfile ...@@ -40,6 +40,8 @@ import tarfile
import time import time
import psutil import psutil
import six
log_file = False log_file = False
class Dumper(object): class Dumper(object):
...@@ -59,10 +61,10 @@ class SystemReporter(Dumper): ...@@ -59,10 +61,10 @@ class SystemReporter(Dumper):
""" Dump data """ """ Dump data """
_date = time.strftime("%Y-%m-%d") _date = time.strftime("%Y-%m-%d")
self.db.connect() self.db.connect()
for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems(): for item, collected_item_list in six.iteritems(self.db.exportSystemAsDict(_date)):
self.writeFile(item, folder, collected_item_list) self.writeFile(item, folder, collected_item_list)
for partition, collected_item_list in self.db.exportDiskAsDict(_date).iteritems(): for partition, collected_item_list in six.iteritems(self.db.exportDiskAsDict(_date)):
partition_id = "_".join(partition.split("-")[:-1]).replace("/", "_") partition_id = "_".join(partition.split("-")[:-1]).replace("/", "_")
item = "memory_%s" % partition.split("-")[-1] item = "memory_%s" % partition.split("-")[-1]
self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list) self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list)
...@@ -135,62 +137,53 @@ class ConsumptionReportBase(object): ...@@ -135,62 +137,53 @@ class ConsumptionReportBase(object):
def getPartitionCPULoadAverage(self, partition_id, date_scope): def getPartitionCPULoadAverage(self, partition_id, date_scope):
self.db.connect() self.db.connect()
query_result_cursor = self.db.select("user", date_scope, (cpu_percent_sum,), = self.db.select("user", date_scope,
columns="SUM(cpu_percent)", columns="SUM(cpu_percent)",
where="partition = '%s'" % partition_id) where="partition = '%s'" % partition_id)
cpu_percent_sum = zip(*query_result_cursor) if cpu_percent_sum is None:
if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
return return
query_result_cursor = self.db.select("user", date_scope, (sample_amount,), = self.db.select("user", date_scope,
columns="COUNT(DISTINCT time)", columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id) where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.db.close() self.db.close()
if len(sample_amount) and len(cpu_percent_sum): return cpu_percent_sum/sample_amount
return cpu_percent_sum[0][0]/sample_amount[0][0]
def getPartitionUsedMemoryAverage(self, partition_id, date_scope): def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
self.db.connect() self.db.connect()
query_result_cursor = self.db.select("user", date_scope, (memory_sum,), = self.db.select("user", date_scope,
columns="SUM(memory_rss)", columns="SUM(memory_rss)",
where="partition = '%s'" % partition_id) where="partition = '%s'" % partition_id)
memory_sum = zip(*query_result_cursor) if memory_sum is None:
if len(memory_sum) and memory_sum[0][0] is None:
return return
query_result_cursor = self.db.select("user", date_scope, (sample_amount,), = self.db.select("user", date_scope,
columns="COUNT(DISTINCT time)", columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id) where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.db.close() self.db.close()
if len(sample_amount) and len(memory_sum): return memory_sum/sample_amount
return memory_sum[0][0]/sample_amount[0][0]
def getPartitionDiskUsedAverage(self, partition_id, date_scope): def getPartitionDiskUsedAverage(self, partition_id, date_scope):
self.db.connect() self.db.connect()
query_result_cursor = self.db.select("folder", date_scope, (disk_used_sum,), = self.db.select("folder", date_scope,
columns="SUM(disk_used)", columns="SUM(disk_used)",
where="partition = '%s'" % partition_id) where="partition = '%s'" % partition_id)
disk_used_sum = zip(*query_result_cursor) if disk_used_sum is None:
if len(disk_used_sum) and disk_used_sum[0][0] is None:
return return
query_result_cursor = self.db.select("folder", date_scope, (collect_amount,), = self.db.select("folder", date_scope,
columns="COUNT(DISTINCT time)", columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id) where="partition = '%s'" % partition_id)
collect_amount = zip(*query_result_cursor)
self.db.close() self.db.close()
if len(collect_amount) and len(disk_used_sum): return disk_used_sum/collect_amount
return disk_used_sum[0][0]/collect_amount[0][0]
class ConsumptionReport(ConsumptionReportBase): class ConsumptionReport(ConsumptionReportBase):
...@@ -287,7 +280,7 @@ class ConsumptionReport(ConsumptionReportBase): ...@@ -287,7 +280,7 @@ class ConsumptionReport(ConsumptionReportBase):
reference=user, reference=user,
category="") category="")
with open(xml_report_path, 'w') as f: with open(xml_report_path, 'wb') as f:
f.write(journal.getXML()) f.write(journal.getXML())
f.close() f.close()
...@@ -298,23 +291,19 @@ class ConsumptionReport(ConsumptionReportBase): ...@@ -298,23 +291,19 @@ class ConsumptionReport(ConsumptionReportBase):
def _getCpuLoadAverageConsumption(self, date_scope): def _getCpuLoadAverageConsumption(self, date_scope):
self.db.connect() self.db.connect()
query_result_cursor = self.db.select("system", date_scope, (cpu_load_percent_list,), = self.db.select("system", date_scope,
columns="SUM(cpu_percent)/COUNT(cpu_percent)") columns="SUM(cpu_percent)/COUNT(cpu_percent)")
cpu_load_percent_list = zip(*query_result_cursor)
self.db.close() self.db.close()
if len(cpu_load_percent_list): return cpu_load_percent_list
return cpu_load_percent_list[0][0]
def _getMemoryAverageConsumption(self, date_scope): def _getMemoryAverageConsumption(self, date_scope):
self.db.connect() self.db.connect()
query_result_cursor = self.db.select("system", date_scope, (memory_used_list,), = self.db.select("system", date_scope,
columns="SUM(memory_used)/COUNT(memory_used)") columns="SUM(memory_used)/COUNT(memory_used)")
memory_used_list = zip(*query_result_cursor)
self.db.close() self.db.close()
if len(memory_used_list): return memory_used_list
return memory_used_list[0][0]
def _getZeroEmissionContribution(self): def _getZeroEmissionContribution(self):
self.db.connect() self.db.connect()
...@@ -329,7 +318,7 @@ class Journal(object): ...@@ -329,7 +318,7 @@ class Journal(object):
def getXML(self): def getXML(self):
report = ElementTree.tostring(self.root) report = ElementTree.tostring(self.root)
return "<?xml version='1.0' encoding='utf-8'?>%s" % report return b"<?xml version='1.0' encoding='utf-8'?>%s" % report
def newTransaction(self, portal_type="Sale Packing List"): def newTransaction(self, portal_type="Sale Packing List"):
transaction = ElementTree.SubElement(self.root, "transaction") transaction = ElementTree.SubElement(self.root, "transaction")
......
...@@ -27,13 +27,15 @@ ...@@ -27,13 +27,15 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import psutil import psutil
import os import os
import subprocess import subprocess
from temperature import collectComputerTemperature, \ from .temperature import collectComputerTemperature, launchTemperatureTest
launchTemperatureTest
from temperature.heating import get_contribution_ratio from .temperature.heating import get_contribution_ratio
import six
MEASURE_INTERVAL = 5 MEASURE_INTERVAL = 5
...@@ -159,7 +161,7 @@ class HeatingContributionSnapshot(_Snapshot): ...@@ -159,7 +161,7 @@ class HeatingContributionSnapshot(_Snapshot):
result = launchTemperatureTest(sensor_id) result = launchTemperatureTest(sensor_id)
if result is None: if result is None:
print "Impossible to test sensor: %s " % sensor_id print("Impossible to test sensor: %s " % sensor_id)
initial_temperature, final_temperature, duration = result initial_temperature, final_temperature, duration = result
...@@ -215,8 +217,7 @@ class ComputerSnapshot(_Snapshot): ...@@ -215,8 +217,7 @@ class ComputerSnapshot(_Snapshot):
# #
self.system_snapshot = SystemSnapshot() self.system_snapshot = SystemSnapshot()
self.temperature_snapshot_list = self._get_temperature_snapshot_list() self.temperature_snapshot_list = self._get_temperature_snapshot_list()
self.disk_snapshot_list = [] self._get_physical_disk_info()
self.partition_list = self._get_physical_disk_info()
if test_heating and model_id is not None \ if test_heating and model_id is not None \
and sensor_id is not None: and sensor_id is not None:
...@@ -231,16 +232,16 @@ class ComputerSnapshot(_Snapshot): ...@@ -231,16 +232,16 @@ class ComputerSnapshot(_Snapshot):
return temperature_snapshot_list return temperature_snapshot_list
def _get_physical_disk_info(self): def _get_physical_disk_info(self):
partition_dict = {} # XXX: merge the following 2 to avoid calling disk_usage() twice
self.disk_snapshot_list = []
self.partition_list = []
partition_set = set()
for partition in psutil.disk_partitions(): for partition in psutil.disk_partitions():
if partition.device not in partition_dict: dev = partition.device
if dev not in partition_set: # XXX: useful ?
partition_set.add(dev)
usage = psutil.disk_usage(partition.mountpoint) usage = psutil.disk_usage(partition.mountpoint)
partition_dict[partition.device] = usage.total self.partition_list.append((dev, usage.total))
self.disk_snapshot_list.append( self.disk_snapshot_list.append(
DiskPartitionSnapshot(partition.device, DiskPartitionSnapshot(dev, partition.mountpoint))
partition.mountpoint))
return [(k, v) for k, v in partition_dict.iteritems()]
from __future__ import print_function
from multiprocessing import Process, active_children, cpu_count, Pipe from multiprocessing import Process, active_children, cpu_count, Pipe
import subprocess import subprocess
...@@ -14,28 +15,21 @@ except NotImplementedError: ...@@ -14,28 +15,21 @@ except NotImplementedError:
DEFAULT_CPU = 1 DEFAULT_CPU = 1
def collectComputerTemperature(sensor_bin="sensors"): def collectComputerTemperature(sensor_bin="sensors"):
cmd = ["%s -u" % sensor_bin] stdout = subprocess.check_output((sensor_bin, '-u'), universal_newlines=True)
  • This seems to be a change in behavior, before this change, if sensors command is not available, the test fail. Which I believe is a good thing, it was hiding errors.

    Isn't the missing part that we need to include component/lmsensors/buildout.cfg in the PATH of the test here ? similar to what we do for socat for example ?

    /cc @rafael @tomo

  • Apparently this is also something we can get from psutil https://psutil.readthedocs.io/en/latest/index.html#psutil.sensors_temperatures

  • Test can fail because command is not available (like on my development environment), but also some tests fail because command exit with non 0, for example:

    test_computer_snapshot (slapos.tests.collect.TestCollectSnapshot) ... No sensors found!
    Make sure you loaded all the kernel drivers you need.
    Try sensors-detect to find out which these are.
    ERROR

    I noticed this on rapidspace-testnode-003-3Nodes-ERP5PROJECT3 , also on RAPIDCLOUD-VM-COMP-2867-3Nodes-ERP5PROJECT6

  • it should fail if sensor is not available.... and it should get sensors from the component like you said.

  • Thanks. So if sensors command is not available it should fail. OK, let's change the software profile to add the command from component in $PATH -> slapos!479 (merged)

    But what to do when sensors command is available and it does no find sensors (the No sensors found! error message from above) ? slapos!479 (merged) won't solve this case.

    Also according to https://github.com/lm-sensors/lm-sensors , this is linux only, psutil version seems a bit more portable but it's not something that's universally available, so slapos should maybe consider this as something optional - only required on linux for example

  • From my understanding of the discussions in slapos!479 (comment 71803) , here we should not use check_output but an approach which would tolerate a non 0 exit code here.

    In !73 (merged) we discuss adding subprocess32, so we can maybe switch to subprocess.run here ?

  • for the subprocess.run approach -> !81 (merged)

Please register or sign in to reply
sp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
stdout, stderr = sp.communicate()
sensor_output_list = stdout.splitlines() sensor_output_list = stdout.splitlines()
adapter_name = "" adapter_name = ""
sensor_temperature_list = [] sensor_temperature_list = []
for line_number in range(len(sensor_output_list)): for line_number, sensor_output in enumerate(sensor_output_list):
found_sensor = None stripped_line = sensor_output.strip()
stripped_line = sensor_output_list[line_number].strip()
if stripped_line.startswith("Adapter:"): if stripped_line.startswith("Adapter:"):
adapter_name = sensor_output_list[line_number-1] adapter_name = sensor_output_list[line_number-1]
elif stripped_line.startswith("temp") and "_input" in stripped_line: elif stripped_line.startswith("temp") and "_input" in stripped_line:
temperature = sensor_output_list[line_number].split()[-1] temperature = sensor_output.split()[-1]
found_sensor = ["%s %s" % (adapter_name, sensor_output_list[line_number-1]), float(temperature)] found_sensor = ["%s %s" % (adapter_name, sensor_output_list[line_number-1]), float(temperature)]
if found_sensor is not None:
critical = '1000' critical = '1000'
maximal = '1000' maximal = '1000'
for next_line in sensor_output_list[line_number+1:line_number+3]: for next_line in sensor_output_list[line_number+1:line_number+3]:
...@@ -120,7 +114,7 @@ def launchTemperatureTest(sensor_id, sensor_bin="sensors", timeout=600, interval ...@@ -120,7 +114,7 @@ def launchTemperatureTest(sensor_id, sensor_bin="sensors", timeout=600, interval
for connection in process_connection_list: for connection in process_connection_list:
try: try:
print connection.recv() print(connection.recv())
except EOFError: except EOFError:
continue continue
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
# #
############################################################################## ##############################################################################
import ConfigParser from six.moves import configparser
import errno import errno
import fcntl import fcntl
import grp import grp
...@@ -51,12 +51,13 @@ import time ...@@ -51,12 +51,13 @@ import time
import traceback import traceback
import zipfile import zipfile
import platform import platform
from urllib2 import urlopen from six.moves.urllib.request import urlopen
import six
import lxml.etree import lxml.etree
import xml_marshaller.xml_marshaller import xml_marshaller.xml_marshaller
from slapos.util import mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6 from slapos.util import dumps, mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6
import slapos.slap as slap import slapos.slap as slap
from slapos import version from slapos import version
from slapos import manager as slapmanager from slapos import manager as slapmanager
...@@ -87,7 +88,7 @@ class OS(object): ...@@ -87,7 +88,7 @@ class OS(object):
def _addWrapper(self, name): def _addWrapper(self, name):
def wrapper(*args, **kw): def wrapper(*args, **kw):
arg_list = [repr(x) for x in args] + [ arg_list = [repr(x) for x in args] + [
'%s=%r' % (x, y) for x, y in kw.iteritems() '%s=%r' % (x, y) for x, y in six.iteritems(kw)
] ]
self._logger.debug('%s(%s)' % (name, ', '.join(arg_list))) self._logger.debug('%s(%s)' % (name, ', '.join(arg_list)))
if not self._dry_run: if not self._dry_run:
...@@ -232,8 +233,8 @@ def _getDict(obj): ...@@ -232,8 +233,8 @@ def _getDict(obj):
return obj return obj
return { return {
key: _getDict(value) \ key: _getDict(value)
for key, value in dikt.iteritems() \ for key, value in six.iteritems(dikt)
# do not attempt to serialize logger: it is both useless and recursive. # do not attempt to serialize logger: it is both useless and recursive.
# do not serialize attributes starting with "_", let the classes have some privacy # do not serialize attributes starting with "_", let the classes have some privacy
if not key.startswith("_") if not key.startswith("_")
...@@ -336,7 +337,7 @@ class Computer(object): ...@@ -336,7 +337,7 @@ class Computer(object):
if conf.dry_run: if conf.dry_run:
return return
try: try:
slap_computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(_getDict(self))) slap_computer.updateConfiguration(dumps(_getDict(self)))
except slap.NotFoundError as error: except slap.NotFoundError as error:
raise slap.NotFoundError("%s\nERROR: This SlapOS node is not recognised by " raise slap.NotFoundError("%s\nERROR: This SlapOS node is not recognised by "
"SlapOS Master and/or computer_id and certificates don't match. " "SlapOS Master and/or computer_id and certificates don't match. "
...@@ -358,7 +359,7 @@ class Computer(object): ...@@ -358,7 +359,7 @@ class Computer(object):
with open(path_to_json, 'wb') as fout: with open(path_to_json, 'wb') as fout:
fout.write(json.dumps(computer_dict, sort_keys=True, indent=2)) fout.write(json.dumps(computer_dict, sort_keys=True, indent=2))
new_xml = xml_marshaller.xml_marshaller.dumps(computer_dict) new_xml = dumps(computer_dict)
new_pretty_xml = prettify_xml(new_xml) new_pretty_xml = prettify_xml(new_xml)
path_to_archive = path_to_xml + '.zip' path_to_archive = path_to_xml + '.zip'
...@@ -1197,7 +1198,7 @@ class Interface(object): ...@@ -1197,7 +1198,7 @@ class Interface(object):
def parse_computer_definition(conf, definition_path): def parse_computer_definition(conf, definition_path):
conf.logger.info('Using definition file %r' % definition_path) conf.logger.info('Using definition file %r' % definition_path)
computer_definition = ConfigParser.RawConfigParser({ computer_definition = configparser.RawConfigParser({
'software_user': 'slapsoft', 'software_user': 'slapsoft',
}) })
computer_definition.read(definition_path) computer_definition.read(definition_path)
...@@ -1308,7 +1309,7 @@ def parse_computer_xml(conf, xml_path): ...@@ -1308,7 +1309,7 @@ def parse_computer_xml(conf, xml_path):
def write_computer_definition(conf, computer): def write_computer_definition(conf, computer):
computer_definition = ConfigParser.RawConfigParser() computer_definition = configparser.RawConfigParser()
computer_definition.add_section('computer') computer_definition.add_section('computer')
if computer.address is not None and computer.netmask is not None: if computer.address is not None and computer.netmask is not None:
computer_definition.set('computer', 'address', '/'.join( computer_definition.set('computer', 'address', '/'.join(
......
...@@ -38,7 +38,7 @@ import subprocess ...@@ -38,7 +38,7 @@ import subprocess
import tarfile import tarfile
import tempfile import tempfile
import time import time
import xmlrpclib from six.moves import xmlrpc_client as xmlrpclib, range
from supervisor import xmlrpc from supervisor import xmlrpc
...@@ -51,6 +51,7 @@ from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError, ...@@ -51,6 +51,7 @@ from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
PathDoesNotExistError, DiskSpaceError) PathDoesNotExistError, DiskSpaceError)
from slapos.grid.networkcache import download_network_cached, upload_network_cached from slapos.grid.networkcache import download_network_cached, upload_network_cached
from slapos.human import bytes2human from slapos.human import bytes2human
from slapos.util import bytes2str
WATCHDOG_MARK = '-on-watch' WATCHDOG_MARK = '-on-watch'
...@@ -60,8 +61,8 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750 ...@@ -60,8 +61,8 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
CP_STORAGE_FOLDER_NAME = 'DATA' CP_STORAGE_FOLDER_NAME = 'DATA'
# XXX not very clean. this is changed when testing # XXX not very clean. this is changed when testing
PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__, PROGRAM_PARTITION_TEMPLATE = bytes2str(pkg_resources.resource_string(__name__,
'templates/program_partition_supervisord.conf.in').read() 'templates/program_partition_supervisord.conf.in'))
def free_space(path, fn): def free_space(path, fn):
...@@ -150,7 +151,7 @@ class Software(object): ...@@ -150,7 +151,7 @@ class Software(object):
self.software_min_free_space = software_min_free_space self.software_min_free_space = software_min_free_space
def check_free_space(self): def check_free_space(self):
required = self.software_min_free_space required = self.software_min_free_space or 0
available = free_space_nonroot(self.software_path) available = free_space_nonroot(self.software_path)
if available < required: if available < required:
...@@ -292,7 +293,7 @@ class Software(object): ...@@ -292,7 +293,7 @@ class Software(object):
f.close() f.close()
def _create_buildout_profile(self, buildout_cfg, url): def _create_buildout_profile(self, buildout_cfg, url):
with open(buildout_cfg, 'wb') as fout: with open(buildout_cfg, 'w') as fout:
fout.write('[buildout]\nextends = ' + url + '\n') fout.write('[buildout]\nextends = ' + url + '\n')
self._set_ownership(buildout_cfg) self._set_ownership(buildout_cfg)
...@@ -419,7 +420,7 @@ class Partition(object): ...@@ -419,7 +420,7 @@ class Partition(object):
def check_free_space(self): def check_free_space(self):
required = self.instance_min_free_space required = self.instance_min_free_space or 0
available = free_space_nonroot(self.instance_path) available = free_space_nonroot(self.instance_path)
if available < required: if available < required:
...@@ -481,8 +482,8 @@ class Partition(object): ...@@ -481,8 +482,8 @@ class Partition(object):
} }
def addCustomGroup(self, group_suffix, partition_id, program_list): def addCustomGroup(self, group_suffix, partition_id, program_list):
group_partition_template = pkg_resources.resource_stream(__name__, group_partition_template = bytes2str(pkg_resources.resource_string(__name__,
'templates/group_partition_supervisord.conf.in').read() 'templates/group_partition_supervisord.conf.in'))
group_id = '{}-{}'.format(partition_id, group_suffix) group_id = '{}-{}'.format(partition_id, group_suffix)
self.supervisor_configuration_group += group_partition_template % { self.supervisor_configuration_group += group_partition_template % {
...@@ -568,8 +569,8 @@ class Partition(object): ...@@ -568,8 +569,8 @@ class Partition(object):
# fill generated buildout with additional information # fill generated buildout with additional information
buildout_text = open(config_location).read() buildout_text = open(config_location).read()
buildout_text += '\n\n' + pkg_resources.resource_string(__name__, buildout_text += '\n\n' + bytes2str(pkg_resources.resource_string(__name__,
'templates/buildout-tail.cfg.in') % { 'templates/buildout-tail.cfg.in')) % {
'computer_id': self.computer_id, 'computer_id': self.computer_id,
'partition_id': self.partition_id, 'partition_id': self.partition_id,
'server_url': self.server_url, 'server_url': self.server_url,
...@@ -671,8 +672,8 @@ class Partition(object): ...@@ -671,8 +672,8 @@ class Partition(object):
os.unlink(self.supervisord_partition_configuration_path) os.unlink(self.supervisord_partition_configuration_path)
else: else:
partition_id = self.computer_partition.getId() partition_id = self.computer_partition.getId()
group_partition_template = pkg_resources.resource_stream(__name__, group_partition_template = bytes2str(pkg_resources.resource_string(__name__,
'templates/group_partition_supervisord.conf.in').read() 'templates/group_partition_supervisord.conf.in'))
self.supervisor_configuration_group = group_partition_template % { self.supervisor_configuration_group = group_partition_template % {
'instance_id': partition_id, 'instance_id': partition_id,
'program_list': ','.join(['_'.join([partition_id, runner]) 'program_list': ','.join(['_'.join([partition_id, runner])
...@@ -842,7 +843,7 @@ class Partition(object): ...@@ -842,7 +843,7 @@ class Partition(object):
self.logger.warning('Problem while stopping process %r, will try later' % gname) self.logger.warning('Problem while stopping process %r, will try later' % gname)
else: else:
self.logger.info('Stopped %r' % gname) self.logger.info('Stopped %r' % gname)
for i in xrange(0, 10): for i in range(0, 10):
# Some process may be still running, be nice and wait for them to be stopped. # Some process may be still running, be nice and wait for them to be stopped.
try: try:
supervisor.removeProcessGroup(gname) supervisor.removeProcessGroup(gname)
......
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
# FOR A PARTICULAR PURPOSE. # FOR A PARTICULAR PURPOSE.
# #
############################################################################## ##############################################################################
from __future__ import print_function
import ast import ast
import json import json
...@@ -29,10 +30,10 @@ try: ...@@ -29,10 +30,10 @@ try:
else: else:
LIBNETWORKCACHE_ENABLED = True LIBNETWORKCACHE_ENABLED = True
except: except:
print 'There was problem while trying to import slapos.libnetworkcache:'\ print('There was problem while trying to import slapos.libnetworkcache:\n%s'
'\n%s' % traceback.format_exc() % traceback.format_exc())
LIBNETWORKCACHE_ENABLED = False LIBNETWORKCACHE_ENABLED = False
print 'Networkcache forced to be disabled.' print('Networkcache forced to be disabled.')
...@@ -45,8 +46,8 @@ def fallback_call(function): ...@@ -45,8 +46,8 @@ def fallback_call(function):
try: try:
return function(self, *args, **kwd) return function(self, *args, **kwd)
except: # indeed, *any* exception is swallowed except: # indeed, *any* exception is swallowed
print 'There was problem while calling method %r:\n%s' % ( print('There was problem while calling method %r:\n%s' % (
function.__name__, traceback.format_exc()) function.__name__, traceback.format_exc()))
return False return False
wrapper.__doc__ = function.__doc__ wrapper.__doc__ = function.__doc__
return wrapper return wrapper
...@@ -107,7 +108,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root, ...@@ -107,7 +108,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
f.close() f.close()
file_descriptor.close() file_descriptor.close()
return True return True
except (IOError, DirectoryNotFound), e: except (IOError, DirectoryNotFound) as e:
logger.info('Failed to download from network cache %s: %s' % \ logger.info('Failed to download from network cache %s: %s' % \
(software_url, str(e))) (software_url, str(e)))
return False return False
...@@ -169,7 +170,7 @@ def upload_network_cached(software_root, software_url, cached_key, ...@@ -169,7 +170,7 @@ def upload_network_cached(software_root, software_url, cached_key,
try: try:
return nc.upload_generic(f, cached_key, **kw) return nc.upload_generic(f, cached_key, **kw)
except (IOError, UploadError), e: except (IOError, UploadError) as e:
logger.info('Failed to upload file. %s' % (str(e))) logger.info('Failed to upload file. %s' % (str(e)))
return False return False
finally: finally:
......
...@@ -38,7 +38,7 @@ import importlib ...@@ -38,7 +38,7 @@ import importlib
import traceback import traceback
import psutil import psutil
from multiprocessing import Process, Queue as MQueue from multiprocessing import Process, Queue as MQueue
import Queue from six.moves import queue, reload_module
from slapos.util import mkdir_p, chownDirectory from slapos.util import mkdir_p, chownDirectory
from slapos.grid.utils import dropPrivileges, killProcessTree from slapos.grid.utils import dropPrivileges, killProcessTree
from slapos.grid.promise import interface from slapos.grid.promise import interface
...@@ -168,7 +168,7 @@ class PromiseProcess(Process): ...@@ -168,7 +168,7 @@ class PromiseProcess(Process):
if not os.path.exists(init_file): if not os.path.exists(init_file):
with open(init_file, 'w') as f: with open(init_file, 'w') as f:
f.write("") f.write("")
os.chmod(init_file, 0644) os.chmod(init_file, 0o644)
# add promise folder to sys.path so we can import promise script # add promise folder to sys.path so we can import promise script
if sys.path[0] != promise_folder: if sys.path[0] != promise_folder:
sys.path[0:0] = [promise_folder] sys.path[0:0] = [promise_folder]
...@@ -184,9 +184,8 @@ class PromiseProcess(Process): ...@@ -184,9 +184,8 @@ class PromiseProcess(Process):
raise AttributeError("Class RunPromise not found in promise" \ raise AttributeError("Class RunPromise not found in promise" \
"%s" % self.name) "%s" % self.name)
if not interface.IPromise.implementedBy(promise_module.RunPromise): if not interface.IPromise.implementedBy(promise_module.RunPromise):
raise RuntimeError("RunPromise class in %s must implements 'IPromise'" \ raise RuntimeError("RunPromise class in %s must implement 'IPromise'"
" interface. zope_interface.implements(interface.IPromise) is" \ " interface. @implementer(interface.IPromise) is missing ?" % self.name)
" missing ?" % self.name)
from slapos.grid.promise.generic import GenericPromise from slapos.grid.promise.generic import GenericPromise
if not issubclass(promise_module.RunPromise, GenericPromise): if not issubclass(promise_module.RunPromise, GenericPromise):
...@@ -195,7 +194,7 @@ class PromiseProcess(Process): ...@@ -195,7 +194,7 @@ class PromiseProcess(Process):
if promise_module.__file__ != self.promise_path: if promise_module.__file__ != self.promise_path:
# cached module need to be updated # cached module need to be updated
promise_module = reload(promise_module) promise_module = reload_module(promise_module)
# load extra parameters # load extra parameters
self._loadPromiseParameterDict(promise_module) self._loadPromiseParameterDict(promise_module)
...@@ -208,7 +207,7 @@ class PromiseProcess(Process): ...@@ -208,7 +207,7 @@ class PromiseProcess(Process):
if not isinstance(extra_dict, dict): if not isinstance(extra_dict, dict):
raise ValueError("Extra parameter is not a dict") raise ValueError("Extra parameter is not a dict")
for key in extra_dict: for key in extra_dict:
if self.argument_dict.has_key(key): if key in self.argument_dict:
raise ValueError("Extra parameter name %r cannot be used.\n%s" % ( raise ValueError("Extra parameter name %r cannot be used.\n%s" % (
key, extra_dict)) key, extra_dict))
self.argument_dict[key] = extra_dict[key] self.argument_dict[key] = extra_dict[key]
...@@ -362,7 +361,7 @@ class PromiseLauncher(object): ...@@ -362,7 +361,7 @@ class PromiseLauncher(object):
try: try:
result = PromiseQueueResult() result = PromiseQueueResult()
result.load(json.loads(f.read())) result.load(json.loads(f.read()))
except ValueError, e: except ValueError as e:
result = None result = None
self.logger.warn('Bad promise JSON result at %r: %s' % ( self.logger.warn('Bad promise JSON result at %r: %s' % (
promise_output_file, promise_output_file,
...@@ -375,7 +374,7 @@ class PromiseLauncher(object): ...@@ -375,7 +374,7 @@ class PromiseLauncher(object):
while True: while True:
try: try:
self.queue_result.get_nowait() self.queue_result.get_nowait()
except Queue.Empty: except queue.Empty:
return return
def _updateFolderOwner(self, folder_path=None): def _updateFolderOwner(self, folder_path=None):
...@@ -443,7 +442,7 @@ class PromiseLauncher(object): ...@@ -443,7 +442,7 @@ class PromiseLauncher(object):
if not promise_process.is_alive(): if not promise_process.is_alive():
try: try:
queue_item = self.queue_result.get(True, 1) queue_item = self.queue_result.get(True, 1)
except Queue.Empty: except queue.Empty:
# no result found in process result Queue # no result found in process result Queue
pass pass
else: else:
......
...@@ -35,8 +35,10 @@ import time ...@@ -35,8 +35,10 @@ import time
import random import random
import traceback import traceback
import slapos.slap import slapos.slap
from slapos.util import mkdir_p from slapos.util import bytes2str, mkdir_p
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
import six
from six import PY3, with_metaclass
from datetime import datetime, timedelta from datetime import datetime, timedelta
PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise' PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise'
...@@ -46,6 +48,10 @@ PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log' ...@@ -46,6 +48,10 @@ PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
PROMISE_PARAMETER_NAME = 'extra_config_dict' PROMISE_PARAMETER_NAME = 'extra_config_dict'
PROMISE_PERIOD_FILE_NAME = '%s.periodicity' PROMISE_PERIOD_FILE_NAME = '%s.periodicity'
LOGLINE_RE = r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w{4,7})\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)"
matchLogStr = re.compile(LOGLINE_RE).match
matchLogBytes = re.compile(LOGLINE_RE.encode()).match if PY3 else matchLogStr
class BaseResult(object): class BaseResult(object):
def __init__(self, problem=False, message=None, date=None): def __init__(self, problem=False, message=None, date=None):
self.__problem = problem self.__problem = problem
...@@ -129,10 +135,7 @@ class PromiseQueueResult(object): ...@@ -129,10 +135,7 @@ class PromiseQueueResult(object):
self.path = data['path'] self.path = data['path']
self.execution_time = data['execution-time'] self.execution_time = data['execution-time']
class GenericPromise(object): class GenericPromise(with_metaclass(ABCMeta, object)):
# Abstract class
__metaclass__ = ABCMeta
def __init__(self, config): def __init__(self, config):
self.__config = config self.__config = config
...@@ -160,9 +163,7 @@ class GenericPromise(object): ...@@ -160,9 +163,7 @@ class GenericPromise(object):
for handler in self.logger.handlers: for handler in self.logger.handlers:
self.logger.removeHandler(handler) self.logger.removeHandler(handler)
if self.__log_folder is None: if self.__log_folder is None:
# configure logger with StringIO self.__logger_buffer = six.StringIO()
import cStringIO
self.__logger_buffer = cStringIO.StringIO()
logger_handler = logging.StreamHandler(self.__logger_buffer) logger_handler = logging.StreamHandler(self.__logger_buffer)
self.__log_file = None self.__log_file = None
else: else:
...@@ -230,9 +231,9 @@ class GenericPromise(object): ...@@ -230,9 +231,9 @@ class GenericPromise(object):
""" """
Call bang if requested Call bang if requested
""" """
if self.__config.has_key('master-url') and \ if 'master-url' in self.__config and \
self.__config.has_key('partition-id') and \ 'partition-in' in self.__config and \
self.__config.has_key('computer-id'): 'computer-id' in self.__config:
slap = slapos.slap.slap() slap = slapos.slap.slap()
slap.initializeConnection( slap.initializeConnection(
...@@ -247,18 +248,14 @@ class GenericPromise(object): ...@@ -247,18 +248,14 @@ class GenericPromise(object):
computer_partition.bang(message) computer_partition.bang(message)
self.logger.info("Bang with message %r." % message) self.logger.info("Bang with message %r." % message)
def __getLogRegex(self):
return re.compile(r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w{4,7})\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)")
def __getResultFromString(self, result_string, only_failure=False): def __getResultFromString(self, result_string, only_failure=False):
line_list = result_string.split('\n') line_list = result_string.split('\n')
result_list = [] result_list = []
line_part = "" line_part = ""
regex = self.__getLogRegex()
for line in line_list: for line in line_list:
if not line: if not line:
continue continue
match = regex.match(line) match = matchLogStr(line)
if match is not None: if match is not None:
if not only_failure or (only_failure and match.groups()[1] == 'ERROR'): if not only_failure or (only_failure and match.groups()[1] == 'ERROR'):
result_list.append({ result_list.append({
...@@ -297,37 +294,36 @@ class GenericPromise(object): ...@@ -297,37 +294,36 @@ class GenericPromise(object):
if not os.path.exists(self.__log_file): if not os.path.exists(self.__log_file):
return [] return []
regex = self.__getLogRegex()
max_date_string = ""
if latest_minute > 0: if latest_minute > 0:
date = datetime.now() - timedelta(minutes=latest_minute) date = datetime.now() - timedelta(minutes=latest_minute)
max_date_string = date.strftime('%Y-%m-%d %H:%M:%S') max_date_string = date.strftime('%Y-%m-%d %H:%M:%S')
else:
max_date_string = ""
line_list = [] line_list = []
result_list = [] result_list = []
transaction_id = None transaction_id = None
transaction_count = 0 transaction_count = 0
with open(self.__log_file, 'r') as f: with open(self.__log_file, 'rb') as f:
offset = 0
f.seek(0, 2) f.seek(0, 2)
size = f.tell() * -1 offset = f.tell()
line = line_part = "" line = b""
while offset > size: line_part = ""
while offset:
offset -= 1 offset -= 1
f.seek(offset, 2) f.seek(offset)
char = f.read(1) char = f.read(1)
if char != '\n': if char != b'\n':
line = char + line line = char + line
if char == '\n' or offset == size: if offset:
# Add new line
if offset == -1:
continue continue
if line != "": if line:
result = regex.match(line) result = matchLogBytes(line)
if result is not None: if result is not None:
if max_date_string and result.groups()[0] <= max_date_string: date, level, tid, msg = map(bytes2str, result.groups())
if max_date_string and date <= max_date_string:
break break
if transaction_id != result.groups()[2]: if transaction_id != tid:
if transaction_id is not None: if transaction_id is not None:
# append new result # append new result
result_list.append(line_list) result_list.append(line_list)
...@@ -335,20 +331,18 @@ class GenericPromise(object): ...@@ -335,20 +331,18 @@ class GenericPromise(object):
transaction_count += 1 transaction_count += 1
if transaction_count > result_count: if transaction_count > result_count:
break break
transaction_id = result.groups()[2] transaction_id = tid
if not only_failure or \ if not only_failure or level == 'ERROR':
(only_failure and result.groups()[1] == 'ERROR'):
line_list.insert(0, { line_list.insert(0, {
'date': datetime.strptime(result.groups()[0], 'date': datetime.strptime(date,
'%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S'),
'status': result.groups()[1], 'status': level,
'message': (result.groups()[3] + line_part).strip(), 'message': (msg + line_part).strip(),
}) })
line_part = ""
else: else:
line_part = '\n' + line + line_part line_part = '\n' + bytes2str(line) + line_part
line = "" line = b""
continue
line = line_part = ""
if len(line_list): if len(line_list):
result_list.append(line_list) result_list.append(line_list)
...@@ -410,7 +404,7 @@ class GenericPromise(object): ...@@ -410,7 +404,7 @@ class GenericPromise(object):
try: try:
self.__queue.put_nowait(result_item) self.__queue.put_nowait(result_item)
break break
except Queue.Full, e: except Queue.Full as e:
error = e error = e
time.sleep(0.5) time.sleep(0.5)
if error: if error:
...@@ -459,7 +453,7 @@ class GenericPromise(object): ...@@ -459,7 +453,7 @@ class GenericPromise(object):
""" """
try: try:
self.sense() self.sense()
except Exception, e: except Exception as e:
# log the result # log the result
self.logger.error(str(e)) self.logger.error(str(e))
if check_anomaly: if check_anomaly:
...@@ -468,7 +462,7 @@ class GenericPromise(object): ...@@ -468,7 +462,7 @@ class GenericPromise(object):
result = self.anomaly() result = self.anomaly()
if result is None: if result is None:
raise ValueError("Promise anomaly method returned 'None'") raise ValueError("Promise anomaly method returned 'None'")
except Exception, e: except Exception as e:
result = AnomalyResult(problem=True, message=str(e)) result = AnomalyResult(problem=True, message=str(e))
else: else:
if isinstance(result, AnomalyResult) and result.hasFailed() and can_bang: if isinstance(result, AnomalyResult) and result.hasFailed() and can_bang:
...@@ -482,7 +476,7 @@ class GenericPromise(object): ...@@ -482,7 +476,7 @@ class GenericPromise(object):
result = self.test() result = self.test()
if result is None: if result is None:
raise ValueError("Promise test method returned 'None'") raise ValueError("Promise test method returned 'None'")
except Exception, e: except Exception as e:
result = TestResult(problem=True, message=str(e)) result = TestResult(problem=True, message=str(e))
if self.__logger_buffer is not None: if self.__logger_buffer is not None:
......
...@@ -32,17 +32,16 @@ import subprocess ...@@ -32,17 +32,16 @@ import subprocess
import functools import functools
import signal import signal
import traceback import traceback
from zope import interface as zope_interface from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise from slapos.grid.promise.generic import GenericPromise
@implementer(interface.IPromise)
class WrapPromise(GenericPromise): class WrapPromise(GenericPromise):
""" """
A wrapper promise used to run old promises style and bash promises A wrapper promise used to run old promises style and bash promises
""" """
zope_interface.implements(interface.IPromise)
def __init__(self, config): def __init__(self, config):
GenericPromise.__init__(self, config) GenericPromise.__init__(self, config)
self.setPeriodicity(minute=2) self.setPeriodicity(minute=2)
...@@ -62,21 +61,19 @@ class WrapPromise(GenericPromise): ...@@ -62,21 +61,19 @@ class WrapPromise(GenericPromise):
[self.getPromiseFile()], [self.getPromiseFile()],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
cwd=self.getPartitionFolder() cwd=self.getPartitionFolder(),
universal_newlines=True,
) )
handler = functools.partial(self.terminate, self.getName(), self.logger, handler = functools.partial(self.terminate, self.getName(), self.logger,
promise_process) promise_process)
signal.signal(signal.SIGINT, handler) signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGTERM, handler)
output, error = promise_process.communicate() message = promise_process.communicate()[0].strip()
message = output or "" if promise_process.returncode:
if error: self.logger.error(message)
message += "\n" + error
if promise_process.returncode != 0:
self.logger.error(message.strip())
else: else:
self.logger.info(message.strip()) self.logger.info(message)
def test(self): def test(self):
# Fail if the latest promise result failed # Fail if the latest promise result failed
......
...@@ -32,7 +32,7 @@ import os ...@@ -32,7 +32,7 @@ import os
import pkg_resources import pkg_resources
import random import random
import socket import socket
import StringIO from io import BytesIO
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
...@@ -42,6 +42,7 @@ import warnings ...@@ -42,6 +42,7 @@ import warnings
import logging import logging
import json import json
import shutil import shutil
import six
if sys.version_info < (2, 6): if sys.version_info < (2, 6):
warnings.warn('Used python version (%s) is old and has problems with' warnings.warn('Used python version (%s) is old and has problems with'
...@@ -167,7 +168,7 @@ def merged_options(args, configp): ...@@ -167,7 +168,7 @@ def merged_options(args, configp):
if configp.has_section('networkcache'): if configp.has_section('networkcache'):
options.update(dict(configp.items('networkcache'))) options.update(dict(configp.items('networkcache')))
for key, value in vars(args).iteritems(): for key, value in six.iteritems(vars(args)):
if value is not None: if value is not None:
options[key] = value options[key] = value
...@@ -672,7 +673,7 @@ stderr_logfile_backups=1 ...@@ -672,7 +673,7 @@ stderr_logfile_backups=1
computer_partition.setComputerPartitionRelatedInstanceList( computer_partition.setComputerPartitionRelatedInstanceList(
[reference for reference in tf.read().split('\n') if reference] [reference for reference in tf.read().split('\n') if reference]
) )
except NotFoundError, e: except NotFoundError as e:
# Master doesn't implement this feature ? # Master doesn't implement this feature ?
self.logger.warning("NotFoundError: %s. \nCannot send requested instance "\ self.logger.warning("NotFoundError: %s. \nCannot send requested instance "\
"list to master. Please check if this feature is"\ "list to master. Please check if this feature is"\
...@@ -682,12 +683,12 @@ stderr_logfile_backups=1 ...@@ -682,12 +683,12 @@ stderr_logfile_backups=1
""" """
""" """
query_cmd = rule_command.replace('--add-rule', '--query-rule') query_cmd = rule_command.replace('--add-rule', '--query-rule')
process = FPopen(query_cmd) process = FPopen(query_cmd, universal_newlines=True)
result, stderr = process.communicate() result, stderr = process.communicate()
if result.strip() == 'no': if result.strip() == 'no':
# rule doesn't exist add to firewall # rule doesn't exist add to firewall
self.logger.debug(rule_command) self.logger.debug(rule_command)
process = FPopen(rule_command) process = FPopen(rule_command, universal_newlines=True)
rule_result, stderr = process.communicate() rule_result, stderr = process.communicate()
if process.returncode == 0: if process.returncode == 0:
if rule_result.strip() != 'success': if rule_result.strip() != 'success':
...@@ -705,13 +706,13 @@ stderr_logfile_backups=1 ...@@ -705,13 +706,13 @@ stderr_logfile_backups=1
""" """
""" """
query_cmd = rule_command.replace('--add-rule', '--query-rule') query_cmd = rule_command.replace('--add-rule', '--query-rule')
process = FPopen(query_cmd) process = FPopen(query_cmd, universal_newlines=True)
result, stderr = process.communicate() result, stderr = process.communicate()
if result.strip() == 'yes': if result.strip() == 'yes':
# The rule really exist, remove it # The rule really exist, remove it
remove_command = rule_command.replace('--add-rule', '--remove-rule') remove_command = rule_command.replace('--add-rule', '--remove-rule')
self.logger.debug(remove_command) self.logger.debug(remove_command)
process = FPopen(remove_command) process = FPopen(remove_command, universal_newlines=True)
rule_result, stderr = process.communicate() rule_result, stderr = process.communicate()
if process.returncode == 0: if process.returncode == 0:
if rule_result.strip() != 'success': if rule_result.strip() != 'success':
...@@ -764,7 +765,7 @@ stderr_logfile_backups=1 ...@@ -764,7 +765,7 @@ stderr_logfile_backups=1
# XXX - need to check firewalld reload instead of restart # XXX - need to check firewalld reload instead of restart
self.logger.info("Reloading firewall configuration...") self.logger.info("Reloading firewall configuration...")
reload_cmd = self.firewall_conf['reload_config_cmd'] reload_cmd = self.firewall_conf['reload_config_cmd']
reload_process = FPopen(reload_cmd) reload_process = FPopen(reload_cmd, universal_newlines=True)
stdout, stderr = reload_process.communicate() stdout, stderr = reload_process.communicate()
if reload_process.returncode != 0: if reload_process.returncode != 0:
raise Exception("Failed to load firewalld rules with command %s.\n%" % ( raise Exception("Failed to load firewalld rules with command %s.\n%" % (
...@@ -922,7 +923,7 @@ stderr_logfile_backups=1 ...@@ -922,7 +923,7 @@ stderr_logfile_backups=1
self._checkPromiseList(local_partition, self._checkPromiseList(local_partition,
check_anomaly=True, check_anomaly=True,
force=False) force=False)
except PromiseError, e: except PromiseError as e:
self.logger.error(e) self.logger.error(e)
if partition_access_status is None or not status_error: if partition_access_status is None or not status_error:
computer_partition.error(e, logger=self.logger) computer_partition.error(e, logger=self.logger)
...@@ -1140,7 +1141,7 @@ stderr_logfile_backups=1 ...@@ -1140,7 +1141,7 @@ stderr_logfile_backups=1
(computer_partition_id, computer_partition_state) (computer_partition_id, computer_partition_state)
computer_partition.error(error_string, logger=self.logger) computer_partition.error(error_string, logger=self.logger)
raise NotImplementedError(error_string) raise NotImplementedError(error_string)
except Exception, e: except Exception as e:
if not isinstance(e, PromiseError): if not isinstance(e, PromiseError):
with open(error_output_file, 'w') as error_file: with open(error_output_file, 'w') as error_file:
# Write error message in a log file assible to computer partition user # Write error message in a log file assible to computer partition user
...@@ -1309,7 +1310,7 @@ stderr_logfile_backups=1 ...@@ -1309,7 +1310,7 @@ stderr_logfile_backups=1
def validateXML(self, to_be_validated, xsd_model): def validateXML(self, to_be_validated, xsd_model):
"""Validates a given xml file""" """Validates a given xml file"""
#We retrieve the xsd model #We retrieve the xsd model
xsd_model = StringIO.StringIO(xsd_model) xsd_model = BytesIO(xsd_model)
xmlschema_doc = etree.parse(xsd_model) xmlschema_doc = etree.parse(xsd_model)
xmlschema = etree.XMLSchema(xmlschema_doc) xmlschema = etree.XMLSchema(xmlschema_doc)
......
...@@ -35,9 +35,10 @@ import subprocess ...@@ -35,9 +35,10 @@ import subprocess
import stat import stat
import sys import sys
import time import time
import xmlrpclib from six.moves import xmlrpc_client as xmlrpclib
from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile) from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile)
from slapos.util import bytes2str
from supervisor import xmlrpc, states from supervisor import xmlrpc, states
...@@ -89,8 +90,8 @@ def createSupervisordConfiguration(instance_root, watchdog_command=''): ...@@ -89,8 +90,8 @@ def createSupervisordConfiguration(instance_root, watchdog_command=''):
# Creates supervisord configuration # Creates supervisord configuration
updateFile(supervisord_configuration_file_path, updateFile(supervisord_configuration_file_path,
pkg_resources.resource_stream(__name__, bytes2str(pkg_resources.resource_string(__name__,
'templates/supervisord.conf.in').read() % { 'templates/supervisord.conf.in')) % {
'supervisord_configuration_directory': supervisord_configuration_directory, 'supervisord_configuration_directory': supervisord_configuration_directory,
'supervisord_socket': os.path.abspath(supervisord_socket), 'supervisord_socket': os.path.abspath(supervisord_socket),
'supervisord_loglevel': 'info', 'supervisord_loglevel': 'info',
......
...@@ -40,6 +40,8 @@ import logging ...@@ -40,6 +40,8 @@ import logging
import psutil import psutil
import time import time
import six
from slapos.grid.exception import BuildoutFailedError, WrongPermissionError from slapos.grid.exception import BuildoutFailedError, WrongPermissionError
# Such umask by default will create paths with full permission # Such umask by default will create paths with full permission
...@@ -123,20 +125,18 @@ class SlapPopen(subprocess.Popen): ...@@ -123,20 +125,18 @@ class SlapPopen(subprocess.Popen):
self.stdin.close() self.stdin.close()
self.stdin = None self.stdin = None
# XXX-Cedric: this algorithm looks overkill for simple logging.
output_lines = [] output_lines = []
while True: for line in self.stdout:
line = self.stdout.readline() if type(line) is not str:
if line == '' and self.poll() is not None: line = line.decode(errors='replace')
break output_lines.append(line)
if line: logger.info(line.rstrip('\n'))
output_lines.append(line) self.wait()
logger.info(line.rstrip('\n'))
self.output = ''.join(output_lines) self.output = ''.join(output_lines)
def md5digest(url): def md5digest(url):
return hashlib.md5(url).hexdigest() return hashlib.md5(url.encode('utf-8')).hexdigest()
def getCleanEnvironment(logger, home_path='/tmp'): def getCleanEnvironment(logger, home_path='/tmp'):
...@@ -150,7 +150,7 @@ def getCleanEnvironment(logger, home_path='/tmp'): ...@@ -150,7 +150,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
if old is not None: if old is not None:
removed_env.append(k) removed_env.append(k)
changed_env['HOME'] = env['HOME'] = home_path changed_env['HOME'] = env['HOME'] = home_path
for k in sorted(changed_env.iterkeys()): for k in sorted(six.iterkeys(changed_env)):
logger.debug('Overridden %s = %r' % (k, changed_env[k])) logger.debug('Overridden %s = %r' % (k, changed_env[k]))
if removed_env: if removed_env:
logger.debug('Removed from environment: %s' % ', '.join(sorted(removed_env))) logger.debug('Removed from environment: %s' % ', '.join(sorted(removed_env)))
...@@ -351,18 +351,20 @@ def launchBuildout(path, buildout_binary, logger, ...@@ -351,18 +351,20 @@ def launchBuildout(path, buildout_binary, logger,
def updateFile(file_path, content, mode=0o600): def updateFile(file_path, content, mode=0o600):
"""Creates or updates a file with "content" as content.""" """Creates or updates a file with "content" as content."""
altered = False content = content.encode('utf-8')
if not (os.path.isfile(file_path)) or \ try:
not (hashlib.md5(open(file_path).read()).digest() == with open(file_path, 'rb') as f:
hashlib.md5(content).digest()): if f.read(len(content) + 1) == content:
with open(file_path, 'w') as fout: if stat.S_IMODE(os.fstat(f.fileno()).st_mode) == mode:
fout.write(content) return False
altered = True os.fchmod(f.fileno(), mode)
os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) return True
if stat.S_IMODE(os.stat(file_path).st_mode) != mode: except IOError:
os.chmod(file_path, mode) pass
altered = True with open(file_path, 'wb') as f:
return altered os.fchmod(f.fileno(), mode)
f.write(content)
return True
def updateExecutable(executable_path, content): def updateExecutable(executable_path, content):
...@@ -399,7 +401,7 @@ def killProcessTree(pid, logger): ...@@ -399,7 +401,7 @@ def killProcessTree(pid, logger):
for child in running_process_list: for child in running_process_list:
try: try:
child.suspend() child.suspend()
except psutil.Error, e: except psutil.Error as e:
logger.debug(str(e)) logger.debug(str(e))
time.sleep(0.2) time.sleep(0.2)
...@@ -408,5 +410,5 @@ def killProcessTree(pid, logger): ...@@ -408,5 +410,5 @@ def killProcessTree(pid, logger):
for process in process_list: for process in process_list:
try: try:
process.kill() process.kill()
except psutil.Error, e: except psutil.Error as e:
logger.debug("Process kill: %s" % e) logger.debug("Process kill: %s" % e)
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
import argparse import argparse
import os.path import os.path
import sys import sys
import six
import slapos.slap.slap import slapos.slap.slap
from slapos.grid.slapgrid import COMPUTER_PARTITION_TIMESTAMP_FILENAME, \ from slapos.grid.slapgrid import COMPUTER_PARTITION_TIMESTAMP_FILENAME, \
...@@ -56,7 +57,7 @@ def parseArgumentTuple(): ...@@ -56,7 +57,7 @@ def parseArgumentTuple():
# Build option_dict # Build option_dict
option_dict = {} option_dict = {}
for argument_key, argument_value in vars(option).iteritems(): for argument_key, argument_value in six.iteritems(vars(option)):
option_dict.update({argument_key: argument_value}) option_dict.update({argument_key: argument_value})
return option_dict return option_dict
......
...@@ -5,12 +5,12 @@ import os.path ...@@ -5,12 +5,12 @@ import os.path
import pwd import pwd
import time import time
from zope import interface as zope_interface from zope.interface import implementer
from slapos.manager import interface from slapos.manager import interface
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@implementer(interface.IManager)
class Manager(object): class Manager(object):
"""Manage cgroup's cpuset in terms on initializing and runtime operations. """Manage cgroup's cpuset in terms on initializing and runtime operations.
...@@ -21,8 +21,6 @@ class Manager(object): ...@@ -21,8 +21,6 @@ class Manager(object):
TODO: there is no limit on number of reserved cores per user. TODO: there is no limit on number of reserved cores per user.
""" """
zope_interface.implements(interface.IManager)
cpu_exclusive_file = ".slapos-cpu-exclusive" cpu_exclusive_file = ".slapos-cpu-exclusive"
cpuset_path = "/sys/fs/cgroup/cpuset/" cpuset_path = "/sys/fs/cgroup/cpuset/"
task_write_mode = "wt" task_write_mode = "wt"
......
...@@ -5,7 +5,6 @@ import os ...@@ -5,7 +5,6 @@ import os
import pwd import pwd
import grp import grp
from .interface import IManager from .interface import IManager
from itertools import ifilter
from zope import interface from zope import interface
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
...@@ -5,8 +5,8 @@ import netaddr ...@@ -5,8 +5,8 @@ import netaddr
import os import os
from .interface import IManager from .interface import IManager
from itertools import ifilter from six.moves import filter
from zope import interface from zope.interface import implementer
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -23,9 +23,8 @@ def which(exename): ...@@ -23,9 +23,8 @@ def which(exename):
return full_path return full_path
return None return None
@implementer(IManager)
class Manager(object): class Manager(object):
interface.implements(IManager)
port_redirect_filename = '.slapos-port-redirect' port_redirect_filename = '.slapos-port-redirect'
def __init__(self, config): def __init__(self, config):
...@@ -89,7 +88,7 @@ class Manager(object): ...@@ -89,7 +88,7 @@ class Manager(object):
'full_ip_list', []) 'full_ip_list', [])
partition_ip_list = [tup[1] for tup in partition_ip_list] partition_ip_list = [tup[1] for tup in partition_ip_list]
partition_ipv6 = next(ifilter(lambda ip_addr: ':' in ip_addr, partition_ipv6 = next(filter(lambda ip_addr: ':' in ip_addr,
partition_ip_list), partition_ip_list),
None) None)
......
...@@ -4,17 +4,16 @@ import os ...@@ -4,17 +4,16 @@ import os
import sys import sys
import subprocess import subprocess
from zope import interface as zope_interface from zope.interface import implementer
from slapos.manager import interface from slapos.manager import interface
from slapos.grid.slapgrid import COMPUTER_PARTITION_WAIT_LIST_FILENAME from slapos.grid.slapgrid import COMPUTER_PARTITION_WAIT_LIST_FILENAME
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@implementer(interface.IManager)
class Manager(object): class Manager(object):
"""Manager is called in every step of preparation of the computer.""" """Manager is called in every step of preparation of the computer."""
zope_interface.implements(interface.IManager)
def __init__(self, config): def __init__(self, config):
"""Manager needs to know config for its functioning. """Manager needs to know config for its functioning.
""" """
......
...@@ -33,6 +33,8 @@ import logging ...@@ -33,6 +33,8 @@ import logging
from slapos.proxy.views import app from slapos.proxy.views import app
from slapos.util import sqlite_connect from slapos.util import sqlite_connect
import six
def _generateSoftwareProductListFromString(software_product_list_string): def _generateSoftwareProductListFromString(software_product_list_string):
""" """
Take a string as argument (which usually comes from the software_product_list Take a string as argument (which usually comes from the software_product_list
...@@ -72,7 +74,7 @@ class ProxyConfig(object): ...@@ -72,7 +74,7 @@ class ProxyConfig(object):
elif section.startswith('multimaster/'): elif section.startswith('multimaster/'):
# Merge multimaster configuration if any # Merge multimaster configuration if any
# XXX: check for duplicate SR entries # XXX: check for duplicate SR entries
for key, value in configuration_dict.iteritems(): for key, value in six.iteritems(configuration_dict):
if key == 'software_release_list': if key == 'software_release_list':
# Split multi-lines values # Split multi-lines values
configuration_dict[key] = [line.strip() for line in value.strip().split('\n')] configuration_dict[key] = [line.strip() for line in value.strip().split('\n')]
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import pkg_resources import pkg_resources
from slapos.util import bytes2str
DB_VERSION = pkg_resources.resource_stream('slapos.proxy', 'schema.sql').readline().strip().split(':')[1] DB_VERSION = bytes2str(pkg_resources.resource_stream('slapos.proxy', 'schema.sql').readline()).strip().split(':')[1]
...@@ -28,6 +28,8 @@ ...@@ -28,6 +28,8 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
from lxml import etree from lxml import etree
import random import random
import string import string
...@@ -35,12 +37,13 @@ from slapos.slap.slap import Computer, ComputerPartition, \ ...@@ -35,12 +37,13 @@ from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError SoftwareRelease, SoftwareInstance, NotFoundError
from slapos.proxy.db_version import DB_VERSION from slapos.proxy.db_version import DB_VERSION
import slapos.slap import slapos.slap
from slapos.util import sqlite_connect from slapos.util import bytes2str, str2bytes, unicode2str, sqlite_connect
from flask import g, Flask, request, abort from flask import g, Flask, request, abort
import xml_marshaller from slapos.util import loads, dumps
from xml_marshaller.xml_marshaller import loads
from xml_marshaller.xml_marshaller import dumps import six
from six.moves import range
app = Flask(__name__) app = Flask(__name__)
...@@ -50,19 +53,10 @@ class UnauthorizedError(Exception): ...@@ -50,19 +53,10 @@ class UnauthorizedError(Exception):
pass pass
# cast everything to string, utf-8 encoded
def to_str(v):
if isinstance(v, str):
return v
if not isinstance(v, unicode):
v = unicode(v)
return v.encode('utf-8')
def xml2dict(xml): def xml2dict(xml):
result_dict = {} result_dict = {}
if xml is not None and xml != '': if xml:
tree = etree.fromstring(to_str(xml)) tree = etree.fromstring(str2bytes(xml))
for element in tree.iter(tag=etree.Element): for element in tree.iter(tag=etree.Element):
if element.tag == 'parameter': if element.tag == 'parameter':
key = element.get('id') key = element.get('id')
...@@ -77,21 +71,16 @@ def xml2dict(xml): ...@@ -77,21 +71,16 @@ def xml2dict(xml):
def dict2xml(dictionary): def dict2xml(dictionary):
instance = etree.Element('instance') instance = etree.Element('instance')
for parameter_id, parameter_value in dictionary.iteritems(): for parameter_id, parameter_value in six.iteritems(dictionary):
# cast everything to string
parameter_value = unicode(parameter_value)
etree.SubElement(instance, "parameter", etree.SubElement(instance, "parameter",
attrib={'id': parameter_id}).text = parameter_value attrib={'id': parameter_id}).text = parameter_value
return etree.tostring(instance, return bytes2str(etree.tostring(instance,
pretty_print=True, pretty_print=True,
xml_declaration=True, xml_declaration=True,
encoding='utf-8') encoding='utf-8'))
def partitiondict2partition(partition): def partitiondict2partition(partition):
for key, value in partition.iteritems():
if type(value) is unicode:
partition[key] = value.encode()
slap_partition = ComputerPartition(partition['computer_reference'], slap_partition = ComputerPartition(partition['computer_reference'],
partition['reference']) partition['reference'])
slap_partition._software_release_document = None slap_partition._software_release_document = None
...@@ -121,7 +110,7 @@ def partitiondict2partition(partition): ...@@ -121,7 +110,7 @@ def partitiondict2partition(partition):
root_partition['partition_reference'] root_partition['partition_reference']
if partition['slave_instance_list'] is not None: if partition['slave_instance_list'] is not None:
slap_partition._parameter_dict['slave_instance_list'] = \ slap_partition._parameter_dict['slave_instance_list'] = \
xml_marshaller.xml_marshaller.loads(partition['slave_instance_list']) loads(partition['slave_instance_list'].encode('utf-8'))
else: else:
slap_partition._parameter_dict['slave_instance_list'] = [] slap_partition._parameter_dict['slave_instance_list'] = []
slap_partition._connection_dict = xml2dict(partition['connection_xml']) slap_partition._connection_dict = xml2dict(partition['connection_xml'])
...@@ -139,7 +128,7 @@ def execute_db(table, query, args=(), one=False, db_version=None, log=False, db= ...@@ -139,7 +128,7 @@ def execute_db(table, query, args=(), one=False, db_version=None, log=False, db=
db_version = DB_VERSION db_version = DB_VERSION
query = query % (table + db_version,) query = query % (table + db_version,)
if log: if log:
print query print(query)
try: try:
cur = db.execute(query, args) cur = db.execute(query, args)
except: except:
...@@ -184,8 +173,8 @@ def _upgradeDatabaseIfNeeded(): ...@@ -184,8 +173,8 @@ def _upgradeDatabaseIfNeeded():
if current_schema_version == DB_VERSION: if current_schema_version == DB_VERSION:
return return
schema = app.open_resource('schema.sql') with app.open_resource('schema.sql', 'r') as f:
schema = schema.read() % dict(version=DB_VERSION, computer=app.config['computer_id']) schema = f.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
g.db.cursor().executescript(schema) g.db.cursor().executescript(schema)
g.db.commit() g.db.commit()
...@@ -242,24 +231,20 @@ def getFullComputerInformation(): ...@@ -242,24 +231,20 @@ def getFullComputerInformation():
for partition in execute_db('partition', 'SELECT * FROM %s WHERE computer_reference=?', [computer_id]): for partition in execute_db('partition', 'SELECT * FROM %s WHERE computer_reference=?', [computer_id]):
slap_computer._computer_partition_list.append(partitiondict2partition( slap_computer._computer_partition_list.append(partitiondict2partition(
partition)) partition))
return xml_marshaller.xml_marshaller.dumps(slap_computer) return dumps(slap_computer)
@app.route('/setComputerPartitionConnectionXml', methods=['POST']) @app.route('/setComputerPartitionConnectionXml', methods=['POST'])
def setComputerPartitionConnectionXml(): def setComputerPartitionConnectionXml():
slave_reference = request.form.get('slave_reference', None) slave_reference = request.form.get('slave_reference', None)
computer_partition_id = request.form['computer_partition_id'].encode() computer_partition_id = unicode2str(request.form['computer_partition_id'])
computer_id = request.form['computer_id'].encode() computer_id = unicode2str(request.form['computer_id'])
connection_xml = request.form['connection_xml'].encode() connection_xml = dict2xml(loads(request.form['connection_xml'].encode('utf-8')))
connection_dict = xml_marshaller.xml_marshaller.loads(
connection_xml)
connection_xml = dict2xml(connection_dict)
if not slave_reference or slave_reference == 'None': if not slave_reference or slave_reference == 'None':
query = 'UPDATE %s SET connection_xml=? WHERE reference=? AND computer_reference=?' query = 'UPDATE %s SET connection_xml=? WHERE reference=? AND computer_reference=?'
argument_list = [connection_xml, computer_partition_id, computer_id] argument_list = [connection_xml, computer_partition_id, computer_id]
execute_db('partition', query, argument_list) execute_db('partition', query, argument_list)
return 'done' return 'done'
else: else:
slave_reference = slave_reference.encode()
query = 'UPDATE %s SET connection_xml=? , hosted_by=? WHERE reference=?' query = 'UPDATE %s SET connection_xml=? , hosted_by=? WHERE reference=?'
argument_list = [connection_xml, computer_partition_id, slave_reference] argument_list = [connection_xml, computer_partition_id, slave_reference]
execute_db('slave', query, argument_list) execute_db('slave', query, argument_list)
...@@ -304,7 +289,7 @@ def useComputer(): ...@@ -304,7 +289,7 @@ def useComputer():
@app.route('/loadComputerConfigurationFromXML', methods=['POST']) @app.route('/loadComputerConfigurationFromXML', methods=['POST'])
def loadComputerConfigurationFromXML(): def loadComputerConfigurationFromXML():
xml = request.form['xml'] xml = request.form['xml']
computer_dict = xml_marshaller.xml_marshaller.loads(str(xml)) computer_dict = loads(xml.encode('utf-8'))
execute_db('computer', 'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)', execute_db('computer', 'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)',
computer_dict) computer_dict)
for partition in computer_dict['partition_list']: for partition in computer_dict['partition_list']:
...@@ -322,14 +307,13 @@ def loadComputerConfigurationFromXML(): ...@@ -322,14 +307,13 @@ def loadComputerConfigurationFromXML():
@app.route('/registerComputerPartition', methods=['GET']) @app.route('/registerComputerPartition', methods=['GET'])
def registerComputerPartition(): def registerComputerPartition():
computer_reference = request.args['computer_reference'].encode() computer_reference = unicode2str(request.args['computer_reference'])
computer_partition_reference = request.args['computer_partition_reference'].encode() computer_partition_reference = unicode2str(request.args['computer_partition_reference'])
partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?', partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
[computer_partition_reference, computer_reference], one=True) [computer_partition_reference, computer_reference], one=True)
if partition is None: if partition is None:
raise UnauthorizedError raise UnauthorizedError
return xml_marshaller.xml_marshaller.dumps( return dumps(partitiondict2partition(partition))
partitiondict2partition(partition))
@app.route('/supplySupply', methods=['POST']) @app.route('/supplySupply', methods=['POST'])
def supplySupply(): def supplySupply():
...@@ -348,7 +332,7 @@ def requestComputerPartition(): ...@@ -348,7 +332,7 @@ def requestComputerPartition():
parsed_request_dict = parseRequestComputerPartitionForm(request.form) parsed_request_dict = parseRequestComputerPartitionForm(request.form)
# Is it a slave instance? # Is it a slave instance?
slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode()) slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode('utf-8'))
# Check first if instance is already allocated # Check first if instance is already allocated
if slave: if slave:
...@@ -391,20 +375,21 @@ def parseRequestComputerPartitionForm(form): ...@@ -391,20 +375,21 @@ def parseRequestComputerPartitionForm(form):
""" """
Parse without intelligence a form from a request(), return it. Parse without intelligence a form from a request(), return it.
""" """
parsed_dict = {} parsed_dict = {
parsed_dict['software_release'] = form['software_release'].encode() 'software_release': unicode2str(form['software_release']),
parsed_dict['software_type'] = form.get('software_type').encode() 'software_type': unicode2str(form.get('software_type')),
parsed_dict['partition_reference'] = form.get('partition_reference', '').encode() 'partition_reference': unicode2str(form.get('partition_reference', '')),
parsed_dict['partition_id'] = form.get('computer_partition_id', '').encode() 'partition_id': unicode2str(form.get('computer_partition_id', '')),
parsed_dict['partition_parameter_kw'] = loads(form.get('partition_parameter_xml', EMPTY_DICT_XML).encode()) 'partition_parameter_kw': loads(form.get('partition_parameter_xml', EMPTY_DICT_XML).encode('utf-8')),
parsed_dict['filter_kw'] = loads(form.get('filter_xml', EMPTY_DICT_XML).encode()) 'filter_kw': loads(form.get('filter_xml', EMPTY_DICT_XML).encode('utf-8')),
# Note: currently ignored for slave instance (slave instances # Note: currently ignored for slave instance (slave instances
# are always started). # are always started).
parsed_dict['requested_state'] = loads(form.get('state').encode()) 'requested_state': loads(form.get('state').encode('utf-8')),
}
return parsed_dict return parsed_dict
run_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(32)]) run_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in range(32)])
def checkIfMasterIsCurrentMaster(master_url): def checkIfMasterIsCurrentMaster(master_url):
""" """
Because there are several ways to contact this server, we can't easily check Because there are several ways to contact this server, we can't easily check
...@@ -421,12 +406,9 @@ def checkIfMasterIsCurrentMaster(master_url): ...@@ -421,12 +406,9 @@ def checkIfMasterIsCurrentMaster(master_url):
slap = slapos.slap.slap() slap = slapos.slap.slap()
slap.initializeConnection(master_url) slap.initializeConnection(master_url)
try: try:
master_run_id = slap._connection_helper.GET('/getRunId') return run_id == bytes2str(slap._connection_helper.GET('/getRunId'))
except: except:
return False return False
if master_run_id == run_id:
return True
return False
@app.route('/getRunId', methods=['GET']) @app.route('/getRunId', methods=['GET'])
def getRunId(): def getRunId():
...@@ -469,7 +451,7 @@ def isRequestToBeForwardedToExternalMaster(parsed_request_dict): ...@@ -469,7 +451,7 @@ def isRequestToBeForwardedToExternalMaster(parsed_request_dict):
return master_url return master_url
software_release = parsed_request_dict['software_release'] software_release = parsed_request_dict['software_release']
for mutimaster_url, mutimaster_entry in app.config.get('multimaster').iteritems(): for mutimaster_url, mutimaster_entry in six.iteritems(app.config.get('multimaster')):
if software_release in mutimaster_entry['software_release_list']: if software_release in mutimaster_entry['software_release_list']:
# Don't allocate the instance locally, but forward to specified master # Don't allocate the instance locally, but forward to specified master
return mutimaster_url return mutimaster_url
...@@ -495,20 +477,17 @@ def forwardRequestToExternalMaster(master_url, request_form): ...@@ -495,20 +477,17 @@ def forwardRequestToExternalMaster(master_url, request_form):
else: else:
slap.initializeConnection(master_url) slap.initializeConnection(master_url)
partition_reference = request_form['partition_reference'].encode() partition_reference = unicode2str(request_form['partition_reference'])
# Store in database # Store in database
execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)', execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
{'partition_reference':partition_reference, 'master_url': master_url}) {'partition_reference':partition_reference, 'master_url': master_url})
new_request_form = request_form.copy() new_request_form = request_form.copy()
filter_kw = loads(new_request_form['filter_xml'].encode()) filter_kw = loads(new_request_form['filter_xml'].encode('utf-8'))
filter_kw['source_instance_id'] = partition_reference filter_kw['source_instance_id'] = partition_reference
new_request_form['filter_xml'] = dumps(filter_kw) new_request_form['filter_xml'] = dumps(filter_kw)
xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form) xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form)
if type(xml) is unicode:
xml = str(xml)
xml.encode('utf-8')
partition = loads(xml) partition = loads(xml)
# XXX move to other end # XXX move to other end
...@@ -568,7 +547,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit ...@@ -568,7 +547,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
instance_xml = dict2xml(partition_parameter_kw) instance_xml = dict2xml(partition_parameter_kw)
requested_computer_id = filter_kw['computer_guid'] requested_computer_id = filter_kw['computer_guid']
instance_xml = dict2xml(partition_parameter_kw)
args = [] args = []
a = args.append a = args.append
q = 'SELECT * FROM %s WHERE partition_reference=?' q = 'SELECT * FROM %s WHERE partition_reference=?'
...@@ -599,7 +577,7 @@ def requestNotSlave(software_release, software_type, partition_reference, partit ...@@ -599,7 +577,7 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
software_type = 'RootSoftwareInstance' software_type = 'RootSoftwareInstance'
else: else:
# XXX Check if software_release should be updated # XXX Check if software_release should be updated
if partition['software_release'].encode() != software_release: if partition['software_release'] != software_release:
q += ' ,software_release=?' q += ' ,software_release=?'
a(software_release) a(software_release)
if partition['requested_by']: if partition['requested_by']:
...@@ -625,13 +603,13 @@ def requestNotSlave(software_release, software_type, partition_reference, partit ...@@ -625,13 +603,13 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
q += ' ,xml=?' q += ' ,xml=?'
a(instance_xml) a(instance_xml)
q += ' WHERE reference=? AND computer_reference=?' q += ' WHERE reference=? AND computer_reference=?'
a(partition['reference'].encode()) a(partition['reference'])
a(partition['computer_reference'].encode()) a(partition['computer_reference'])
execute_db('partition', q, args) execute_db('partition', q, args)
args = [] args = []
partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?', partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
[partition['reference'].encode(), partition['computer_reference'].encode()], one=True) [partition['reference'], partition['computer_reference']], one=True)
address_list = [] address_list = []
for address in execute_db('partition_network', 'SELECT * FROM %s WHERE partition_reference=?', [partition['reference']]): for address in execute_db('partition_network', 'SELECT * FROM %s WHERE partition_reference=?', [partition['reference']]):
address_list.append((address['reference'], address['address'])) address_list.append((address['reference'], address['address']))
...@@ -642,12 +620,12 @@ def requestNotSlave(software_release, software_type, partition_reference, partit ...@@ -642,12 +620,12 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
software_instance = SoftwareInstance(_connection_dict=xml2dict(partition['connection_xml']), software_instance = SoftwareInstance(_connection_dict=xml2dict(partition['connection_xml']),
_parameter_dict=xml2dict(partition['xml']), _parameter_dict=xml2dict(partition['xml']),
connection_xml=partition['connection_xml'], connection_xml=partition['connection_xml'],
slap_computer_id=partition['computer_reference'].encode(), slap_computer_id=partition['computer_reference'],
slap_computer_partition_id=partition['reference'], slap_computer_partition_id=partition['reference'],
slap_software_release_url=partition['software_release'], slap_software_release_url=partition['software_release'],
slap_server_url='slap_server_url', slap_server_url='slap_server_url',
slap_software_type=partition['software_type'], slap_software_type=partition['software_type'],
_instance_guid='%s-%s' % (partition['computer_reference'].encode(), partition['reference']), _instance_guid='%s-%s' % (partition['computer_reference'], partition['reference']),
_requested_state=requested_state, _requested_state=requested_state,
ip_list=address_list) ip_list=address_list)
return software_instance return software_instance
...@@ -704,28 +682,29 @@ def requestSlave(software_release, software_type, partition_reference, partition ...@@ -704,28 +682,29 @@ def requestSlave(software_release, software_type, partition_reference, partition
# Add slave to partition slave_list if not present else replace information # Add slave to partition slave_list if not present else replace information
slave_instance_list = partition['slave_instance_list'] slave_instance_list = partition['slave_instance_list']
if slave_instance_list is None: if slave_instance_list:
slave_instance_list = [] slave_instance_list = loads(slave_instance_list.encode('utf-8'))
else: for i, x in enumerate(slave_instance_list):
slave_instance_list = xml_marshaller.xml_marshaller.loads(slave_instance_list.encode())
for x in slave_instance_list:
if x['slave_reference'] == slave_reference: if x['slave_reference'] == slave_reference:
slave_instance_list.remove(x) slave_instance_list[i] = new_slave
break
slave_instance_list.append(new_slave) else:
slave_instance_list.append(new_slave)
else:
slave_instance_list = [new_slave]
# Update slave_instance_list in database # Update slave_instance_list in database
args = [] args = []
a = args.append a = args.append
q = 'UPDATE %s SET slave_instance_list=?' q = 'UPDATE %s SET slave_instance_list=?'
a(xml_marshaller.xml_marshaller.dumps(slave_instance_list)) a(bytes2str(dumps(slave_instance_list)))
q += ' WHERE reference=? and computer_reference=?' q += ' WHERE reference=? and computer_reference=?'
a(partition['reference'].encode()) a(partition['reference'])
a(requested_computer_id) a(requested_computer_id)
execute_db('partition', q, args) execute_db('partition', q, args)
args = [] args = []
partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?', partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
[partition['reference'].encode(), requested_computer_id], one=True) [partition['reference'], requested_computer_id], one=True)
# Add slave to slave table if not there # Add slave to slave table if not there
slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=? and computer_reference=?', slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
...@@ -757,9 +736,9 @@ def requestSlave(software_release, software_type, partition_reference, partition ...@@ -757,9 +736,9 @@ def requestSlave(software_release, software_type, partition_reference, partition
@app.route('/softwareInstanceRename', methods=['POST']) @app.route('/softwareInstanceRename', methods=['POST'])
def softwareInstanceRename(): def softwareInstanceRename():
new_name = request.form['new_name'].encode() new_name = unicode2str(request.form['new_name'])
computer_partition_id = request.form['computer_partition_id'].encode() computer_partition_id = unicode2str(request.form['computer_partition_id'])
computer_id = request.form['computer_id'].encode() computer_id = unicode2str(request.form['computer_id'])
q = 'UPDATE %s SET partition_reference = ? WHERE reference = ? AND computer_reference = ?' q = 'UPDATE %s SET partition_reference = ? WHERE reference = ? AND computer_reference = ?'
execute_db('partition', q, [new_name, computer_partition_id, computer_id]) execute_db('partition', q, [new_name, computer_partition_id, computer_id])
...@@ -767,16 +746,16 @@ def softwareInstanceRename(): ...@@ -767,16 +746,16 @@ def softwareInstanceRename():
@app.route('/getComputerPartitionStatus', methods=['GET']) @app.route('/getComputerPartitionStatus', methods=['GET'])
def getComputerPartitionStatus(): def getComputerPartitionStatus():
return xml_marshaller.xml_marshaller.dumps('Not implemented.') return dumps('Not implemented.')
@app.route('/computerBang', methods=['POST']) @app.route('/computerBang', methods=['POST'])
def computerBang(): def computerBang():
return xml_marshaller.xml_marshaller.dumps('') return dumps('')
@app.route('/getComputerPartitionCertificate', methods=['GET']) @app.route('/getComputerPartitionCertificate', methods=['GET'])
def getComputerPartitionCertificate(): def getComputerPartitionCertificate():
# proxy does not use partition certificate, but client calls this. # proxy does not use partition certificate, but client calls this.
return xml_marshaller.xml_marshaller.dumps({'certificate': '', 'key': ''}) return dumps({'certificate': '', 'key': ''})
@app.route('/getSoftwareReleaseListFromSoftwareProduct', methods=['GET']) @app.route('/getSoftwareReleaseListFromSoftwareProduct', methods=['GET'])
def getSoftwareReleaseListFromSoftwareProduct(): def getSoftwareReleaseListFromSoftwareProduct():
...@@ -788,10 +767,10 @@ def getSoftwareReleaseListFromSoftwareProduct(): ...@@ -788,10 +767,10 @@ def getSoftwareReleaseListFromSoftwareProduct():
raise NotImplementedError('software_release_url parameter is not supported yet.') raise NotImplementedError('software_release_url parameter is not supported yet.')
else: else:
assert(software_product_reference is not None) assert(software_product_reference is not None)
if app.config['software_product_list'].has_key(software_product_reference): if software_product_reference in app.config['software_product_list']:
software_release_url_list =\ software_release_url_list =\
[app.config['software_product_list'][software_product_reference]] [app.config['software_product_list'][software_product_reference]]
else: else:
software_release_url_list = [] software_release_url_list = []
return xml_marshaller.xml_marshaller.dumps(software_release_url_list) return dumps(software_release_url_list)
...@@ -30,4 +30,4 @@ if sys.version_info < (2, 6): ...@@ -30,4 +30,4 @@ if sys.version_info < (2, 6):
import warnings import warnings
warnings.warn('Used python version (%s) is old and has problems with' warnings.warn('Used python version (%s) is old and has problems with'
' IPv6 connections' % '.'.join([str(q) for q in sys.version_info[:3]])) ' IPv6 connections' % '.'.join([str(q) for q in sys.version_info[:3]]))
from slap import * from .slap import *
...@@ -40,15 +40,19 @@ import os ...@@ -40,15 +40,19 @@ import os
import json import json
import logging import logging
import re import re
import urlparse
import hashlib import hashlib
from util import xml2dict from functools import wraps
import six
from six.moves.urllib import parse
from .util import xml2dict
from slapos.util import loads, dumps, bytes2str
import netaddr import netaddr
from xml.sax import saxutils from xml.sax import saxutils
import zope.interface from zope.interface import implementer
from interface import slap as interface from .interface import slap as interface
from xml_marshaller import xml_marshaller
from uritemplate import expand from uritemplate import expand
...@@ -92,33 +96,29 @@ class SlapRequester(SlapDocument): ...@@ -92,33 +96,29 @@ class SlapRequester(SlapDocument):
request_dict=request_dict, request_dict=request_dict,
connection_helper=self._connection_helper, connection_helper=self._connection_helper,
) )
if type(xml) is unicode: software_instance = loads(xml)
xml = str(xml)
xml.encode('utf-8')
software_instance = xml_marshaller.loads(xml)
computer_partition = ComputerPartition( computer_partition = ComputerPartition(
software_instance.slap_computer_id.encode('UTF-8'), software_instance.slap_computer_id,
software_instance.slap_computer_partition_id.encode('UTF-8'), software_instance.slap_computer_partition_id,
connection_helper=self._connection_helper, connection_helper=self._connection_helper,
) )
# Hack to give all object attributes to the ComputerPartition instance # Hack to give all object attributes to the ComputerPartition instance
# XXX Should be removed by correctly specifying difference between # XXX Should be removed by correctly specifying difference between
# ComputerPartition and SoftwareInstance # ComputerPartition and SoftwareInstance
computer_partition.__dict__ = dict(computer_partition.__dict__.items() + computer_partition.__dict__.update(software_instance.__dict__)
software_instance.__dict__.items())
# XXX not generic enough. # XXX not generic enough.
if xml_marshaller.loads(request_dict['shared_xml']): if loads(request_dict['shared_xml']):
computer_partition._synced = True computer_partition._synced = True
computer_partition._connection_dict = software_instance._connection_dict computer_partition._connection_dict = software_instance._connection_dict
computer_partition._parameter_dict = software_instance._parameter_dict computer_partition._parameter_dict = software_instance._parameter_dict
return computer_partition return computer_partition
@implementer(interface.ISoftwareRelease)
class SoftwareRelease(SlapDocument): class SoftwareRelease(SlapDocument):
""" """
Contains Software Release information Contains Software Release information
""" """
zope.interface.implements(interface.ISoftwareRelease)
def __init__(self, software_release=None, computer_guid=None, **kw): def __init__(self, software_release=None, computer_guid=None, **kw):
""" """
...@@ -129,8 +129,6 @@ class SoftwareRelease(SlapDocument): ...@@ -129,8 +129,6 @@ class SoftwareRelease(SlapDocument):
SlapDocument.__init__(self, kw.pop('connection_helper', None), SlapDocument.__init__(self, kw.pop('connection_helper', None),
kw.pop('hateoas_navigator', None)) kw.pop('hateoas_navigator', None))
self._software_instance_list = [] self._software_instance_list = []
if software_release is not None:
software_release = software_release.encode('UTF-8')
self._software_release = software_release self._software_release = software_release
self._computer_guid = computer_guid self._computer_guid = computer_guid
...@@ -178,8 +176,8 @@ class SoftwareRelease(SlapDocument): ...@@ -178,8 +176,8 @@ class SoftwareRelease(SlapDocument):
return getattr(self, '_requested_state', 'available') return getattr(self, '_requested_state', 'available')
@implementer(interface.ISoftwareProductCollection)
class SoftwareProductCollection(object): class SoftwareProductCollection(object):
zope.interface.implements(interface.ISoftwareProductCollection)
def __init__(self, logger, slap): def __init__(self, logger, slap):
self.logger = logger self.logger = logger
...@@ -201,40 +199,41 @@ class SoftwareProductCollection(object): ...@@ -201,40 +199,41 @@ class SoftwareProductCollection(object):
# XXX What is this SoftwareInstance class? # XXX What is this SoftwareInstance class?
@implementer(interface.ISoftwareInstance)
class SoftwareInstance(SlapDocument): class SoftwareInstance(SlapDocument):
""" """
Contains Software Instance information Contains Software Instance information
""" """
zope.interface.implements(interface.ISoftwareInstance)
def __init__(self, **kwargs): def __init__(self, **kw):
""" """
Makes easy initialisation of class parameters Makes easy initialisation of class parameters
""" """
for k, v in kwargs.iteritems(): self.__dict__.update(kw)
setattr(self, k, v)
"""Exposed exceptions""" """Exposed exceptions"""
@implementer(interface.IResourceNotReady)
class ResourceNotReady(Exception): class ResourceNotReady(Exception):
zope.interface.implements(interface.IResourceNotReady) pass
@implementer(interface.IServerError)
class ServerError(Exception): class ServerError(Exception):
zope.interface.implements(interface.IServerError) pass
@implementer(interface.INotFoundError)
class NotFoundError(Exception): class NotFoundError(Exception):
zope.interface.implements(interface.INotFoundError) pass
class AuthenticationError(Exception): class AuthenticationError(Exception):
pass pass
@implementer(interface.IConnectionError)
class ConnectionError(Exception): class ConnectionError(Exception):
zope.interface.implements(interface.IConnectionError) pass
@implementer(interface.ISupply)
class Supply(SlapDocument): class Supply(SlapDocument):
zope.interface.implements(interface.ISupply)
def supply(self, software_release, computer_guid=None, state='available'): def supply(self, software_release, computer_guid=None, state='available'):
try: try:
...@@ -247,8 +246,8 @@ class Supply(SlapDocument): ...@@ -247,8 +246,8 @@ class Supply(SlapDocument):
% computer_guid) % computer_guid)
@implementer(interface.IOpenOrder)
class OpenOrder(SlapRequester): class OpenOrder(SlapRequester):
zope.interface.implements(interface.IOpenOrder)
def request(self, software_release, partition_reference, def request(self, software_release, partition_reference,
partition_parameter_kw=None, software_type=None, partition_parameter_kw=None, software_type=None,
...@@ -260,12 +259,12 @@ class OpenOrder(SlapRequester): ...@@ -260,12 +259,12 @@ class OpenOrder(SlapRequester):
request_dict = { request_dict = {
'software_release': software_release, 'software_release': software_release,
'partition_reference': partition_reference, 'partition_reference': partition_reference,
'partition_parameter_xml': xml_marshaller.dumps(partition_parameter_kw), 'partition_parameter_xml': dumps(partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw), 'filter_xml': dumps(filter_kw),
# XXX Cedric: Why state and shared are marshalled? First is a string # XXX Cedric: Why state and shared are marshalled? First is a string
# And second is a boolean. # And second is a boolean.
'state': xml_marshaller.dumps(state), 'state': dumps(state),
'shared_xml': xml_marshaller.dumps(shared), 'shared_xml': dumps(shared),
} }
if software_type is not None: if software_type is not None:
request_dict['software_type'] = software_type request_dict['software_type'] = software_type
...@@ -280,7 +279,7 @@ class OpenOrder(SlapRequester): ...@@ -280,7 +279,7 @@ class OpenOrder(SlapRequester):
raw_information = self._hateoas_navigator.getHostingSubscriptionRootSoftwareInstanceInformation(partition_reference) raw_information = self._hateoas_navigator.getHostingSubscriptionRootSoftwareInstanceInformation(partition_reference)
software_instance = SoftwareInstance() software_instance = SoftwareInstance()
# XXX redefine SoftwareInstance to be more consistent # XXX redefine SoftwareInstance to be more consistent
for key, value in raw_information.iteritems(): for key, value in six.iteritems(raw_information):
if key in ['_links']: if key in ['_links']:
continue continue
setattr(software_instance, '_%s' % key, value) setattr(software_instance, '_%s' % key, value)
...@@ -292,7 +291,7 @@ class OpenOrder(SlapRequester): ...@@ -292,7 +291,7 @@ class OpenOrder(SlapRequester):
Requests a computer. Requests a computer.
""" """
xml = self._connection_helper.POST('requestComputer', data={'computer_title': computer_reference}) xml = self._connection_helper.POST('requestComputer', data={'computer_title': computer_reference})
computer = xml_marshaller.loads(xml) computer = loads(xml)
computer._connection_helper = self._connection_helper computer._connection_helper = self._connection_helper
computer._hateoas_navigator = self._hateoas_navigator computer._hateoas_navigator = self._hateoas_navigator
return computer return computer
...@@ -303,24 +302,19 @@ def _syncComputerInformation(func): ...@@ -303,24 +302,19 @@ def _syncComputerInformation(func):
Synchronize computer object with server information Synchronize computer object with server information
""" """
def decorated(self, *args, **kw): def decorated(self, *args, **kw):
if getattr(self, '_synced', 0): if not getattr(self, '_synced', 0):
return func(self, *args, **kw) computer = self._connection_helper.getFullComputerInformation(
computer = self._connection_helper.getFullComputerInformation(self._computer_id) self._computer_id)
for key, value in computer.__dict__.items(): self.__dict__.update(computer.__dict__)
if isinstance(value, unicode): self._synced = True
# convert unicode to utf-8 for computer_partition in self.getComputerPartitionList():
setattr(self, key, value.encode('utf-8')) computer_partition._synced = True
else:
setattr(self, key, value)
setattr(self, '_synced', True)
for computer_partition in self.getComputerPartitionList():
setattr(computer_partition, '_synced', True)
return func(self, *args, **kw) return func(self, *args, **kw)
return decorated return wraps(func)(decorated)
@implementer(interface.IComputer)
class Computer(SlapDocument): class Computer(SlapDocument):
zope.interface.implements(interface.IComputer)
def __init__(self, computer_id, connection_helper=None, hateoas_navigator=None): def __init__(self, computer_id, connection_helper=None, hateoas_navigator=None):
SlapDocument.__init__(self, connection_helper, hateoas_navigator) SlapDocument.__init__(self, connection_helper, hateoas_navigator)
...@@ -366,7 +360,7 @@ class Computer(SlapDocument): ...@@ -366,7 +360,7 @@ class Computer(SlapDocument):
def getStatus(self): def getStatus(self):
xml = self._connection_helper.GET('getComputerStatus', params={'computer_id': self._computer_id}) xml = self._connection_helper.GET('getComputerStatus', params={'computer_id': self._computer_id})
return xml_marshaller.loads(xml) return loads(xml)
def revokeCertificate(self): def revokeCertificate(self):
self._connection_helper.POST('revokeComputerCertificate', data={ self._connection_helper.POST('revokeComputerCertificate', data={
...@@ -375,7 +369,7 @@ class Computer(SlapDocument): ...@@ -375,7 +369,7 @@ class Computer(SlapDocument):
def generateCertificate(self): def generateCertificate(self):
xml = self._connection_helper.POST('generateComputerCertificate', data={ xml = self._connection_helper.POST('generateComputerCertificate', data={
'computer_id': self._computer_id}) 'computer_id': self._computer_id})
return xml_marshaller.loads(xml) return loads(xml)
def parsed_error_message(status, body, path): def parsed_error_message(status, body, path):
...@@ -391,8 +385,8 @@ def parsed_error_message(status, body, path): ...@@ -391,8 +385,8 @@ def parsed_error_message(status, body, path):
return 'Server responded with wrong code %s with %s' % (status, path) return 'Server responded with wrong code %s with %s' % (status, path)
@implementer(interface.IComputerPartition)
class ComputerPartition(SlapRequester): class ComputerPartition(SlapRequester):
zope.interface.implements(interface.IComputerPartition)
def __init__(self, computer_id=None, partition_id=None, def __init__(self, computer_id=None, partition_id=None,
request_dict=None, connection_helper=None, request_dict=None, connection_helper=None,
...@@ -466,11 +460,10 @@ class ComputerPartition(SlapRequester): ...@@ -466,11 +460,10 @@ class ComputerPartition(SlapRequester):
'software_release': software_release, 'software_release': software_release,
'software_type': software_type, 'software_type': software_type,
'partition_reference': partition_reference, 'partition_reference': partition_reference,
'shared_xml': xml_marshaller.dumps(shared), 'shared_xml': dumps(shared),
'partition_parameter_xml': xml_marshaller.dumps( 'partition_parameter_xml': dumps(partition_parameter_kw),
partition_parameter_kw), 'filter_xml': dumps(filter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw), 'state': dumps(state),
'state': xml_marshaller.dumps(state),
} }
self._updateTransactionFile(partition_reference) self._updateTransactionFile(partition_reference)
return self._requestComputerPartition(request_dict) return self._requestComputerPartition(request_dict)
...@@ -529,7 +522,7 @@ class ComputerPartition(SlapRequester): ...@@ -529,7 +522,7 @@ class ComputerPartition(SlapRequester):
raw_information = self._hateoas_navigator.getRelatedInstanceInformation(partition_reference) raw_information = self._hateoas_navigator.getRelatedInstanceInformation(partition_reference)
software_instance = SoftwareInstance() software_instance = SoftwareInstance()
# XXX redefine SoftwareInstance to be more consistent # XXX redefine SoftwareInstance to be more consistent
for key, value in raw_information.iteritems(): for key, value in six.iteritems(raw_information):
if key in ['_links']: if key in ['_links']:
continue continue
setattr(software_instance, '_%s' % key, value) setattr(software_instance, '_%s' % key, value)
...@@ -615,7 +608,7 @@ class ComputerPartition(SlapRequester): ...@@ -615,7 +608,7 @@ class ComputerPartition(SlapRequester):
self._connection_helper.POST('setComputerPartitionConnectionXml', data={ self._connection_helper.POST('setComputerPartitionConnectionXml', data={
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id, 'computer_partition_id': self._partition_id,
'connection_xml': xml_marshaller.dumps(connection_dict), 'connection_xml': dumps(connection_dict),
'slave_reference': slave_reference}) 'slave_reference': slave_reference})
def getInstanceParameter(self, key): def getInstanceParameter(self, key):
...@@ -643,7 +636,7 @@ class ComputerPartition(SlapRequester): ...@@ -643,7 +636,7 @@ class ComputerPartition(SlapRequester):
'computer_partition_id': self._partition_id, 'computer_partition_id': self._partition_id,
} }
) )
return xml_marshaller.loads(xml) return loads(xml)
def getStatus(self): def getStatus(self):
xml = self._connection_helper.GET('getComputerPartitionStatus', xml = self._connection_helper.GET('getComputerPartitionStatus',
...@@ -652,7 +645,7 @@ class ComputerPartition(SlapRequester): ...@@ -652,7 +645,7 @@ class ComputerPartition(SlapRequester):
'computer_partition_id': self._partition_id, 'computer_partition_id': self._partition_id,
} }
) )
return xml_marshaller.loads(xml) return loads(xml)
def getFullHostingIpAddressList(self): def getFullHostingIpAddressList(self):
xml = self._connection_helper.GET('getHostingSubscriptionIpList', xml = self._connection_helper.GET('getHostingSubscriptionIpList',
...@@ -661,14 +654,14 @@ class ComputerPartition(SlapRequester): ...@@ -661,14 +654,14 @@ class ComputerPartition(SlapRequester):
'computer_partition_id': self._partition_id, 'computer_partition_id': self._partition_id,
} }
) )
return xml_marshaller.loads(xml) return loads(xml)
def setComputerPartitionRelatedInstanceList(self, instance_reference_list): def setComputerPartitionRelatedInstanceList(self, instance_reference_list):
self._connection_helper.POST('updateComputerPartitionRelatedInstanceList', self._connection_helper.POST('updateComputerPartitionRelatedInstanceList',
data={ data={
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id, 'computer_partition_id': self._partition_id,
'instance_reference_xml': xml_marshaller.dumps(instance_reference_list) 'instance_reference_xml': dumps(instance_reference_list)
} }
) )
...@@ -676,7 +669,7 @@ def _addIpv6Brackets(url): ...@@ -676,7 +669,7 @@ def _addIpv6Brackets(url):
# if master_url contains an ipv6 without bracket, add it # if master_url contains an ipv6 without bracket, add it
# Note that this is mostly to limit specific issues with # Note that this is mostly to limit specific issues with
# backward compatiblity, not to ensure generic detection. # backward compatiblity, not to ensure generic detection.
api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url) api_scheme, api_netloc, api_path, api_query, api_fragment = parse.urlsplit(url)
try: try:
ip = netaddr.IPAddress(api_netloc) ip = netaddr.IPAddress(api_netloc)
port = None port = None
...@@ -690,7 +683,7 @@ def _addIpv6Brackets(url): ...@@ -690,7 +683,7 @@ def _addIpv6Brackets(url):
api_netloc = '[%s]' % ip api_netloc = '[%s]' % ip
if port: if port:
api_netloc = '%s:%s' % (api_netloc, port) api_netloc = '%s:%s' % (api_netloc, port)
url = urlparse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment)) url = parse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment))
return url return url
class ConnectionHelper: class ConnectionHelper:
...@@ -709,7 +702,7 @@ class ConnectionHelper: ...@@ -709,7 +702,7 @@ class ConnectionHelper:
def getComputerInformation(self, computer_id): def getComputerInformation(self, computer_id):
xml = self.GET('getComputerInformation', params={'computer_id': computer_id}) xml = self.GET('getComputerInformation', params={'computer_id': computer_id})
return xml_marshaller.loads(xml) return loads(xml)
def getFullComputerInformation(self, computer_id): def getFullComputerInformation(self, computer_id):
""" """
...@@ -728,13 +721,10 @@ class ConnectionHelper: ...@@ -728,13 +721,10 @@ class ConnectionHelper:
# We should stablise slap library soon. # We should stablise slap library soon.
xml = self.GET('getComputerInformation', params=params) xml = self.GET('getComputerInformation', params=params)
if type(xml) is unicode: return loads(xml)
xml = str(xml)
xml.encode('utf-8')
return xml_marshaller.loads(xml)
def do_request(self, method, path, params=None, data=None, headers=None): def do_request(self, method, path, params=None, data=None, headers=None):
url = urlparse.urljoin(self.slapgrid_uri, path) url = parse.urljoin(self.slapgrid_uri, path)
if headers is None: if headers is None:
headers = {} headers = {}
headers.setdefault('Accept', '*/*') headers.setdefault('Accept', '*/*')
...@@ -754,7 +744,7 @@ class ConnectionHelper: ...@@ -754,7 +744,7 @@ class ConnectionHelper:
# Behavior kept for compatibility with old slapproxies (< v1.3.3). # Behavior kept for compatibility with old slapproxies (< v1.3.3).
# Can be removed when old slapproxies are no longer in use. # Can be removed when old slapproxies are no longer in use.
if data: if data:
for k, v in data.iteritems(): for k, v in six.iteritems(data):
if v is None: if v is None:
data[k] = 'None' data[k] = 'None'
...@@ -765,7 +755,12 @@ class ConnectionHelper: ...@@ -765,7 +755,12 @@ class ConnectionHelper:
data=data, data=data,
headers=headers, headers=headers,
timeout=self.timeout) timeout=self.timeout)
req.raise_for_status() try:
req.raise_for_status()
except TypeError:
# In Py3, a comparison between NoneType and int can occur if req has no
# status_code (= None).
pass
  • Isn't it better to let the TypeError propagate in this case ? Without knowing under which conditions these response without status code occur, it sounds like a error case, wasn't it ?

    The concern is about the try: except: pass that might catch too much error.

Please register or sign in to reply
except (requests.Timeout, requests.ConnectionError) as exc: except (requests.Timeout, requests.ConnectionError) as exc:
raise ConnectionError("Couldn't connect to the server. Please " raise ConnectionError("Couldn't connect to the server. Please "
...@@ -816,8 +811,8 @@ class ConnectionHelper: ...@@ -816,8 +811,8 @@ class ConnectionHelper:
getHateoasUrl_cache = {} getHateoasUrl_cache = {}
@implementer(interface.slap)
class slap: class slap:
zope.interface.implements(interface.slap)
def initializeConnection(self, slapgrid_uri, def initializeConnection(self, slapgrid_uri,
key_file=None, cert_file=None, key_file=None, cert_file=None,
...@@ -837,7 +832,8 @@ class slap: ...@@ -837,7 +832,8 @@ class slap:
pass pass
if not slapgrid_rest_uri: if not slapgrid_rest_uri:
try: try:
slapgrid_rest_uri = getHateoasUrl_cache[getHateoasUrl_cache_key] = self._connection_helper.GET('getHateoasUrl') slapgrid_rest_uri = getHateoasUrl_cache[getHateoasUrl_cache_key] = \
bytes2str(self._connection_helper.GET('getHateoasUrl'))
except: except:
pass pass
if slapgrid_rest_uri: if slapgrid_rest_uri:
...@@ -885,10 +881,7 @@ class slap: ...@@ -885,10 +881,7 @@ class slap:
'computer_partition_reference': partition_id, 'computer_partition_reference': partition_id,
} }
) )
if type(xml) is unicode: result = loads(xml)
xml = str(xml)
xml.encode('utf-8')
result = xml_marshaller.loads(xml)
# XXX: dirty hack to make computer partition usable. xml_marshaller is too # XXX: dirty hack to make computer partition usable. xml_marshaller is too
# low-level for our needs here. # low-level for our needs here.
result._connection_helper = self._connection_helper result._connection_helper = self._connection_helper
...@@ -923,10 +916,7 @@ class slap: ...@@ -923,10 +916,7 @@ class slap:
params['software_release_url'] = software_release_url params['software_release_url'] = software_release_url
xml = self._connection_helper.GET(url, params=params) xml = self._connection_helper.GET(url, params=params)
if type(xml) is unicode: result = loads(xml)
xml = str(xml)
xml.encode('utf-8')
result = xml_marshaller.loads(xml)
assert(type(result) == list) assert(type(result) == list)
return result return result
...@@ -1046,7 +1036,7 @@ class SlapHateoasNavigator(HateoasNavigator): ...@@ -1046,7 +1036,7 @@ class SlapHateoasNavigator(HateoasNavigator):
raw_information = self.getHostingSubscriptionRootSoftwareInstanceInformation(hosting_subscription_link['title']) raw_information = self.getHostingSubscriptionRootSoftwareInstanceInformation(hosting_subscription_link['title'])
software_instance = SoftwareInstance() software_instance = SoftwareInstance()
# XXX redefine SoftwareInstance to be more consistent # XXX redefine SoftwareInstance to be more consistent
for key, value in raw_information.iteritems(): for key, value in six.iteritems(raw_information):
if key in ['_links']: if key in ['_links']:
continue continue
setattr(software_instance, '_%s' % key, value) setattr(software_instance, '_%s' % key, value)
......
...@@ -29,23 +29,21 @@ import logging ...@@ -29,23 +29,21 @@ import logging
import pprint import pprint
import unittest import unittest
import tempfile import tempfile
import StringIO
import sys import sys
import os import os
import sqlite3
import pkg_resources import pkg_resources
from contextlib import contextmanager
from mock import patch, create_autospec from mock import patch, create_autospec
import mock import mock
from slapos.util import sqlite_connect from slapos.util import sqlite_connect, bytes2str
import slapos.cli.console import slapos.cli.console
import slapos.cli.entry import slapos.cli.entry
import slapos.cli.info import slapos.cli.info
import slapos.cli.list import slapos.cli.list
import slapos.cli.supervisorctl import slapos.cli.supervisorctl
import slapos.cli.proxy_show from slapos.cli.proxy_show import do_show, StringIO
from slapos.client import ClientConfig from slapos.client import ClientConfig
import slapos.grid.svcbackend import slapos.grid.svcbackend
import slapos.proxy import slapos.proxy
...@@ -99,9 +97,9 @@ class TestCliProxyShow(CliMixin): ...@@ -99,9 +97,9 @@ class TestCliProxyShow(CliMixin):
self.conf.logger = self.logger self.conf.logger = self.logger
# load database # load database
schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_11.sql') schema = bytes2str(pkg_resources.resource_string('slapos.tests.slapproxy', 'database_dump_version_11.sql'))
db = sqlite_connect(self.db_file.name) db = sqlite_connect(self.db_file.name)
db.cursor().executescript(schema.read()) db.cursor().executescript(schema)
db.commit() db.commit()
# by default we simulate being invoked with "show all" arguments # by default we simulate being invoked with "show all" arguments
...@@ -121,7 +119,7 @@ class TestCliProxyShow(CliMixin): ...@@ -121,7 +119,7 @@ class TestCliProxyShow(CliMixin):
with mock.patch( with mock.patch(
'slapos.cli.proxy_show.logging.getLogger', 'slapos.cli.proxy_show.logging.getLogger',
return_value=logger): return_value=logger):
slapos.cli.proxy_show.do_show(self.conf) do_show(self.conf)
# installed softwares are listed # installed softwares are listed
logger.info.assert_any_call( logger.info.assert_any_call(
...@@ -132,7 +130,7 @@ class TestCliProxyShow(CliMixin): ...@@ -132,7 +130,7 @@ class TestCliProxyShow(CliMixin):
logger.info.assert_any_call( logger.info.assert_any_call(
' %s = %s', ' %s = %s',
'_', '_',
'{\n "url": "memcached://10.0.30.235:2003/", \n "monitor-base-url": ""\n}') '{\n "monitor-base-url": "",\n "url": "memcached://10.0.30.235:2003/"\n}')
# other parameters are displayed as simple string # other parameters are displayed as simple string
logger.info.assert_any_call( logger.info.assert_any_call(
...@@ -152,10 +150,10 @@ class TestCliProxyShow(CliMixin): ...@@ -152,10 +150,10 @@ class TestCliProxyShow(CliMixin):
def test_proxy_show_displays_on_stdout(self): def test_proxy_show_displays_on_stdout(self):
saved_stderr = sys.stderr saved_stderr = sys.stderr
saved_stdout = sys.stdout saved_stdout = sys.stdout
sys.stderr = stderr = StringIO.StringIO() sys.stderr = stderr = StringIO()
sys.stdout = stdout = StringIO.StringIO() sys.stdout = stdout = StringIO()
try: try:
slapos.cli.proxy_show.do_show(self.conf) do_show(self.conf)
finally: finally:
sys.stderr = saved_stderr sys.stderr = saved_stderr
sys.stdout = saved_stdout sys.stdout = saved_stdout
...@@ -169,8 +167,8 @@ class TestCliProxyShow(CliMixin): ...@@ -169,8 +167,8 @@ class TestCliProxyShow(CliMixin):
def test_proxy_show_use_pager(self): def test_proxy_show_use_pager(self):
saved_stderr = sys.stderr saved_stderr = sys.stderr
saved_stdout = sys.stdout saved_stdout = sys.stdout
sys.stderr = stderr = StringIO.StringIO() sys.stderr = stderr = StringIO()
sys.stdout = stdout = StringIO.StringIO() sys.stdout = stdout = StringIO()
stdout.isatty = lambda *args: True stdout.isatty = lambda *args: True
# use a pager that just output to a file. # use a pager that just output to a file.
...@@ -179,7 +177,7 @@ class TestCliProxyShow(CliMixin): ...@@ -179,7 +177,7 @@ class TestCliProxyShow(CliMixin):
os.environ['PAGER'] = 'cat > {}'.format(tmp.name) os.environ['PAGER'] = 'cat > {}'.format(tmp.name)
try: try:
slapos.cli.proxy_show.do_show(self.conf) do_show(self.conf)
finally: finally:
sys.stderr = saved_stderr sys.stderr = saved_stderr
sys.stdout = saved_stdout sys.stdout = saved_stdout
...@@ -327,53 +325,34 @@ class TestCliSupervisorctl(CliMixin): ...@@ -327,53 +325,34 @@ class TestCliSupervisorctl(CliMixin):
class TestCliConsole(unittest.TestCase): class TestCliConsole(unittest.TestCase):
def setUp(self):
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._parameter_dict = {'parameter_name': 'parameter_value'}
request_patch = patch.object(slapos.slap.OpenOrder, 'request', return_value = cp) script = """\
self.mock_request = request_patch.start() print(request('software_release', 'instance').getInstanceParameterDict()['parameter_name'])
"""
self.config_file = tempfile.NamedTemporaryFile() @contextmanager
self.config_file.write('''[slapos] def _test_console(self):
master_url=null cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
''') cp._parameter_dict = {'parameter_name': 'parameter_value'}
self.config_file.flush() with patch.object(slapos.slap.OpenOrder, 'request',
return_value = cp) as mock_request, \
def tearDown(self): patch.object(sys, 'stdout', StringIO()) as app_stdout, \
self.mock_request.stop() tempfile.NamedTemporaryFile() as config_file:
self.config_file.close() config_file.write(b'[slapos]\nmaster_url=null\n')
config_file.flush()
yield slapos.cli.entry.SlapOSApp(), config_file.name
mock_request.assert_called_once_with(
'software_release', 'instance')
self.assertIn('parameter_value', app_stdout.getvalue())
def test_console_interactive(self): def test_console_interactive(self):
app = slapos.cli.entry.SlapOSApp() with self._test_console() as (app, config_file), \
saved_stdin = sys.stdin patch.object(sys, 'stdin', StringIO(self.script)):
saved_stdout = sys.stdout app.run(('console', '--cfg', config_file))
try:
sys.stdin = app_stdin = StringIO.StringIO(
"""print request('software_release', 'instance').getInstanceParameterDict()['parameter_name']\n""")
sys.stdout = app_stdout = StringIO.StringIO()
app.run(('console', '--cfg', self.config_file.name))
finally:
sys.stdin = saved_stdin
sys.stdout = saved_stdout
self.mock_request.assert_called_once_with('software_release', 'instance')
self.assertIn('parameter_value', app_stdout.getvalue())
def test_console_script(self): def test_console_script(self):
with tempfile.NamedTemporaryFile() as script: with self._test_console() as (app, config_file), \
script.write( tempfile.NamedTemporaryFile('w') as script:
"""print request('software_release', 'instance').getInstanceParameterDict()['parameter_name']\n""") script.write(self.script)
script.flush() script.flush()
app.run(('console', '--cfg', config_file, script.name))
app = slapos.cli.entry.SlapOSApp()
saved_stdout = sys.stdout
try:
sys.stdout = app_stdout = StringIO.StringIO()
app.run(('console', '--cfg', self.config_file.name, script.name))
finally:
sys.stdout = saved_stdout
self.mock_request.assert_called_once_with('software_release', 'instance')
self.assertIn('parameter_value', app_stdout.getvalue())
...@@ -36,7 +36,7 @@ import psutil ...@@ -36,7 +36,7 @@ import psutil
from time import strftime from time import strftime
from slapos.collect import entity, snapshot, db, reporter from slapos.collect import entity, snapshot, db, reporter
from slapos.cli.entry import SlapOSApp from slapos.cli.entry import SlapOSApp
from ConfigParser import ConfigParser from six.moves.configparser import ConfigParser
class FakeDatabase(object): class FakeDatabase(object):
def __init__(self): def __init__(self):
...@@ -364,7 +364,7 @@ class TestCollectReport(unittest.TestCase): ...@@ -364,7 +364,7 @@ class TestCollectReport(unittest.TestCase):
with tarfile.open("%s.tar.gz" % dump_folder) as tf: with tarfile.open("%s.tar.gz" % dump_folder) as tf:
self.assertEqual(tf.getmembers()[0].name, "1990-01-01") self.assertEqual(tf.getmembers()[0].name, "1990-01-01")
self.assertEqual(tf.getmembers()[1].name, "1990-01-01/test.txt") self.assertEqual(tf.getmembers()[1].name, "1990-01-01/test.txt")
self.assertEqual(tf.extractfile(tf.getmembers()[1]).read(), 'hi') self.assertEqual(tf.extractfile(tf.getmembers()[1]).read(), b'hi')
class TestCollectSnapshot(unittest.TestCase): class TestCollectSnapshot(unittest.TestCase):
...@@ -482,10 +482,10 @@ class TestCollectEntity(unittest.TestCase): ...@@ -482,10 +482,10 @@ class TestCollectEntity(unittest.TestCase):
config.set('slapos', 'instance_root', self.instance_root) config.set('slapos', 'instance_root', self.instance_root)
user_dict = entity.get_user_list(config) user_dict = entity.get_user_list(config)
username_list = ['slapuser0', 'slapuser1', 'slapuser2'] username_set = {'slapuser0', 'slapuser1', 'slapuser2'}
self.assertEqual(username_list, user_dict.keys()) self.assertEquals(username_set, set(user_dict))
for name in username_list: for name in username_set:
self.assertEqual(user_dict[name].name, name) self.assertEqual(user_dict[name].name, name)
self.assertEqual(user_dict[name].snapshot_list, []) self.assertEqual(user_dict[name].snapshot_list, [])
expected_path = "%s/slappart%s" % (self.instance_root, name.strip("slapuser")) expected_path = "%s/slappart%s" % (self.instance_root, name.strip("slapuser"))
...@@ -508,11 +508,11 @@ class TestCollectEntity(unittest.TestCase): ...@@ -508,11 +508,11 @@ class TestCollectEntity(unittest.TestCase):
self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot") self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot")
self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",)) self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",))
self.assertEqual(database.invoked_method_list[1][1][1].keys(), self.assertEqual(set(database.invoked_method_list[1][1][1]),
['cpu_time', 'cpu_percent', 'process', {'cpu_time', 'cpu_percent', 'process',
'memory_rss', 'pid', 'memory_percent', 'memory_rss', 'pid', 'memory_percent',
'io_rw_counter', 'insertion_date', 'insertion_time', 'io_rw_counter', 'insertion_date', 'insertion_time',
'io_cycles_counter', 'cpu_num_threads']) 'io_cycles_counter', 'cpu_num_threads'})
self.assertEqual(database.invoked_method_list[2], ("commit", "")) self.assertEqual(database.invoked_method_list[2], ("commit", ""))
self.assertEqual(database.invoked_method_list[3], ("close", "")) self.assertEqual(database.invoked_method_list[3], ("close", ""))
...@@ -527,19 +527,19 @@ class TestCollectEntity(unittest.TestCase): ...@@ -527,19 +527,19 @@ class TestCollectEntity(unittest.TestCase):
self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot") self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot")
self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",)) self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",))
self.assertEqual(database.invoked_method_list[1][1][1].keys(), self.assertEqual(set(database.invoked_method_list[1][1][1]),
['cpu_time', 'cpu_percent', 'process', {'cpu_time', 'cpu_percent', 'process',
'memory_rss', 'pid', 'memory_percent', 'memory_rss', 'pid', 'memory_percent',
'io_rw_counter', 'insertion_date', 'insertion_time', 'io_rw_counter', 'insertion_date', 'insertion_time',
'io_cycles_counter', 'cpu_num_threads']) 'io_cycles_counter', 'cpu_num_threads'})
self.assertEqual(database.invoked_method_list[2], ("commit", "")) self.assertEquals(database.invoked_method_list[2], ("commit", ""))
self.assertEqual(database.invoked_method_list[3], ("close", "")) self.assertEquals(database.invoked_method_list[3], ("close", ""))
self.assertEqual(database.invoked_method_list[4], ("connect", "")) self.assertEqual(database.invoked_method_list[4], ("connect", ""))
self.assertEqual(database.invoked_method_list[5][0], "inserFolderSnapshot") self.assertEqual(database.invoked_method_list[5][0], "inserFolderSnapshot")
self.assertEqual(database.invoked_method_list[5][1][0], ("fakeuser0",)) self.assertEqual(database.invoked_method_list[5][1][0], ("fakeuser0",))
self.assertEqual(database.invoked_method_list[5][1][1].keys(), self.assertEqual(set(database.invoked_method_list[5][1][1]),
['insertion_date', 'disk_usage', 'insertion_time']) {'insertion_date', 'disk_usage', 'insertion_time'})
self.assertEqual(database.invoked_method_list[6], ("commit", "")) self.assertEqual(database.invoked_method_list[6], ("commit", ""))
self.assertEqual(database.invoked_method_list[7], ("close", "")) self.assertEqual(database.invoked_method_list[7], ("close", ""))
...@@ -554,23 +554,23 @@ class TestCollectEntity(unittest.TestCase): ...@@ -554,23 +554,23 @@ class TestCollectEntity(unittest.TestCase):
self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot") self.assertEqual(database.invoked_method_list[1][0], "insertUserSnapshot")
self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",)) self.assertEqual(database.invoked_method_list[1][1][0], ("fakeuser0",))
self.assertEqual(database.invoked_method_list[1][1][1].keys(), self.assertEqual(set(database.invoked_method_list[1][1][1]),
['cpu_time', 'cpu_percent', 'process', {'cpu_time', 'cpu_percent', 'process',
'memory_rss', 'pid', 'memory_percent', 'memory_rss', 'pid', 'memory_percent',
'io_rw_counter', 'insertion_date', 'insertion_time', 'io_rw_counter', 'insertion_date', 'insertion_time',
'io_cycles_counter', 'cpu_num_threads']) 'io_cycles_counter', 'cpu_num_threads'})
self.assertEqual(database.invoked_method_list[2], ("commit", "")) self.assertEqual(database.invoked_method_list[2], ("commit", ""))
self.assertEqual(database.invoked_method_list[3], ("close", "")) self.assertEqual(database.invoked_method_list[3], ("close", ""))
self.assertEqual(database.invoked_method_list[4], ("connect", "")) self.assertEqual(database.invoked_method_list[4], ("connect", ""))
self.assertEqual(database.invoked_method_list[5][0], "select") self.assertEqual(database.invoked_method_list[5][0], "select")
self.assertEqual(database.invoked_method_list[5][1][0], ()) self.assertEqual(database.invoked_method_list[5][1][0], ())
self.assertEqual(database.invoked_method_list[5][1][1].keys(), self.assertEqual(set(database.invoked_method_list[5][1][1]),
['table', 'where', 'limit', 'order', 'columns']) {'table', 'where', 'limit', 'order', 'columns'})
self.assertEqual(database.invoked_method_list[6][0], "inserFolderSnapshot") self.assertEqual(database.invoked_method_list[6][0], "inserFolderSnapshot")
self.assertEqual(database.invoked_method_list[6][1][0], ("fakeuser0",)) self.assertEqual(database.invoked_method_list[6][1][0], ("fakeuser0",))
self.assertEqual(database.invoked_method_list[6][1][1].keys(), self.assertEqual(set(database.invoked_method_list[6][1][1]),
['insertion_date', 'disk_usage', 'insertion_time']) {'insertion_date', 'disk_usage', 'insertion_time'})
self.assertEqual(database.invoked_method_list[7], ("commit", "")) self.assertEqual(database.invoked_method_list[7], ("commit", ""))
self.assertEqual(database.invoked_method_list[8], ("close", "")) self.assertEqual(database.invoked_method_list[8], ("close", ""))
...@@ -583,14 +583,14 @@ class TestCollectEntity(unittest.TestCase): ...@@ -583,14 +583,14 @@ class TestCollectEntity(unittest.TestCase):
self.assertEqual(database.invoked_method_list[1][0], "insertComputerSnapshot") self.assertEqual(database.invoked_method_list[1][0], "insertComputerSnapshot")
self.assertEqual(database.invoked_method_list[1][1][0], ()) self.assertEqual(database.invoked_method_list[1][1][0], ())
self.assertEqual(database.invoked_method_list[1][1][1].keys(), self.assertEqual(set(database.invoked_method_list[1][1][1]),
['insertion_time', 'insertion_date', 'cpu_num_core', {'insertion_time', 'insertion_date', 'cpu_num_core',
'partition_list', 'cpu_frequency', 'memory_size', 'partition_list', 'cpu_frequency', 'memory_size',
'cpu_type', 'memory_type']) 'cpu_type', 'memory_type'})
self.assertEqual(database.invoked_method_list[2][0], "insertSystemSnapshot") self.assertEqual(database.invoked_method_list[2][0], "insertSystemSnapshot")
self.assertEqual(database.invoked_method_list[2][1][0], ()) self.assertEqual(database.invoked_method_list[2][1][0], ())
self.assertEqual(set(database.invoked_method_list[2][1][1].keys()), self.assertEqual(set(database.invoked_method_list[2][1][1]),
set([ 'memory_used', 'cpu_percent', 'insertion_date', 'insertion_time', set([ 'memory_used', 'cpu_percent', 'insertion_date', 'insertion_time',
'loadavg', 'memory_free', 'net_in_bytes', 'net_in_dropped', 'loadavg', 'memory_free', 'net_in_bytes', 'net_in_dropped',
'net_in_errors', 'net_out_bytes', 'net_out_dropped', 'net_in_errors', 'net_out_bytes', 'net_out_dropped',
...@@ -598,7 +598,7 @@ class TestCollectEntity(unittest.TestCase): ...@@ -598,7 +598,7 @@ class TestCollectEntity(unittest.TestCase):
self.assertEqual(database.invoked_method_list[3][0], "insertDiskPartitionSnapshot") self.assertEqual(database.invoked_method_list[3][0], "insertDiskPartitionSnapshot")
self.assertEqual(database.invoked_method_list[3][1][0], ()) self.assertEqual(database.invoked_method_list[3][1][0], ())
self.assertEqual(set(database.invoked_method_list[3][1][1].keys()), self.assertEqual(set(database.invoked_method_list[3][1][1]),
set([ 'used', 'insertion_date', 'partition', 'free', set([ 'used', 'insertion_date', 'partition', 'free',
'mountpoint', 'insertion_time' ])) 'mountpoint', 'insertion_time' ]))
......
...@@ -34,7 +34,7 @@ import slapos.cli.configure_local ...@@ -34,7 +34,7 @@ import slapos.cli.configure_local
from slapos.cli.configure_local import ConfigureLocalCommand, _createConfigurationDirectory from slapos.cli.configure_local import ConfigureLocalCommand, _createConfigurationDirectory
from slapos.cli.entry import SlapOSApp from slapos.cli.entry import SlapOSApp
from argparse import Namespace from argparse import Namespace
from ConfigParser import ConfigParser from six.moves.configparser import ConfigParser
# Disable any command to launch slapformat and supervisor # Disable any command to launch slapformat and supervisor
slapos.cli.configure_local._runFormat = lambda x: "Do nothing" slapos.cli.configure_local._runFormat = lambda x: "Do nothing"
......
...@@ -47,7 +47,7 @@ class SlapPopenTestCase(unittest.TestCase): ...@@ -47,7 +47,7 @@ class SlapPopenTestCase(unittest.TestCase):
def test_exec(self): def test_exec(self):
"""Test command execution with SlapPopen. """Test command execution with SlapPopen.
""" """
self.script.write('#!/bin/sh\necho "hello"\nexit 123') self.script.write(b'#!/bin/sh\necho "hello"\nexit 123')
self.script.close() self.script.close()
logger = mock.MagicMock() logger = mock.MagicMock()
...@@ -65,7 +65,7 @@ class SlapPopenTestCase(unittest.TestCase): ...@@ -65,7 +65,7 @@ class SlapPopenTestCase(unittest.TestCase):
def test_debug(self): def test_debug(self):
"""Test debug=True, which keeps interactive. """Test debug=True, which keeps interactive.
""" """
self.script.write('#!/bin/sh\necho "exit code?"\nread rc\nexit $rc') self.script.write(b'#!/bin/sh\necho "exit code?"\nread rc\nexit $rc')
self.script.close() self.script.close()
# keep a reference to stdin and stdout to restore them later # keep a reference to stdin and stdout to restore them later
...@@ -74,7 +74,7 @@ class SlapPopenTestCase(unittest.TestCase): ...@@ -74,7 +74,7 @@ class SlapPopenTestCase(unittest.TestCase):
# replace stdin with a pipe that will write 123 # replace stdin with a pipe that will write 123
child_stdin_r, child_stdin_w = os.pipe() child_stdin_r, child_stdin_w = os.pipe()
os.write(child_stdin_w, "123") os.write(child_stdin_w, b"123")
os.close(child_stdin_w) os.close(child_stdin_w)
os.dup2(child_stdin_r, sys.stdin.fileno()) os.dup2(child_stdin_r, sys.stdin.fileno())
...@@ -88,7 +88,7 @@ class SlapPopenTestCase(unittest.TestCase): ...@@ -88,7 +88,7 @@ class SlapPopenTestCase(unittest.TestCase):
debug=True, debug=True,
logger=logging.getLogger()) logger=logging.getLogger())
# program output # program output
self.assertEqual('exit code?\n', os.read(child_stdout_r, 1024)) self.assertEqual(b'exit code?\n', os.read(child_stdout_r, 1024))
self.assertEqual(123, program.returncode) self.assertEqual(123, program.returncode)
self.assertEqual('(output not captured in debug mode)', program.output) self.assertEqual('(output not captured in debug mode)', program.output)
......
...@@ -28,17 +28,15 @@ import unittest ...@@ -28,17 +28,15 @@ import unittest
from zope.interface.verify import verifyClass from zope.interface.verify import verifyClass
import zope.interface import zope.interface
import types from six import class_types
from slapos import slap from slapos import slap
def getOnlyImplementationAssertionMethod(klass, method_list): def getOnlyImplementationAssertionMethod(klass, method_list):
"""Returns method which verifies if a klass only implements its interfaces""" """Returns method which verifies if a klass only implements its interfaces"""
def testMethod(self): def testMethod(self):
implemented_method_list = [x for x in dir(klass) \ implemented_method_list = {x for x in dir(klass)
if ((not x.startswith('_')) and callable(getattr(klass, x)))] if not x.startswith('_') and callable(getattr(klass, x))}
for interface_method in method_list: implemented_method_list.difference_update(method_list)
if interface_method in implemented_method_list:
implemented_method_list.remove(interface_method)
if implemented_method_list: if implemented_method_list:
raise AssertionError("Unexpected methods %s" % implemented_method_list) raise AssertionError("Unexpected methods %s" % implemented_method_list)
...@@ -61,7 +59,7 @@ def generateTestMethodListOnClass(klass, module): ...@@ -61,7 +59,7 @@ def generateTestMethodListOnClass(klass, module):
"""Generate test method on klass""" """Generate test method on klass"""
for class_id in dir(module): for class_id in dir(module):
implementing_class = getattr(module, class_id) implementing_class = getattr(module, class_id)
if type(implementing_class) not in (types.ClassType, types.TypeType): if not isinstance(implementing_class, class_types):
continue continue
# add methods to assert that publicly available classes are defining # add methods to assert that publicly available classes are defining
# interfaces # interfaces
...@@ -69,7 +67,7 @@ def generateTestMethodListOnClass(klass, module): ...@@ -69,7 +67,7 @@ def generateTestMethodListOnClass(klass, module):
setattr(klass, method_name, getDeclarationAssertionMethod( setattr(klass, method_name, getDeclarationAssertionMethod(
implementing_class)) implementing_class))
implemented_method_list = [] implemented_method_list = ['with_traceback']
for interface in list(zope.interface.implementedBy(implementing_class)): for interface in list(zope.interface.implementedBy(implementing_class)):
# for each interface which class declares add a method which verify # for each interface which class declares add a method which verify
# implementation # implementation
......
...@@ -34,8 +34,8 @@ import json ...@@ -34,8 +34,8 @@ import json
import random import random
import logging import logging
from datetime import datetime, timedelta from datetime import datetime, timedelta
import Queue import six
from zope import interface as zope_interface from six.moves import queue
from slapos.grid.promise import interface, PromiseLauncher, PromiseProcess, PromiseError from slapos.grid.promise import interface, PromiseLauncher, PromiseProcess, PromiseError
from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult, from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult,
PromiseQueueResult, PROMISE_STATE_FOLDER_NAME, PromiseQueueResult, PROMISE_STATE_FOLDER_NAME,
...@@ -59,7 +59,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase): ...@@ -59,7 +59,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
def writeInit(self): def writeInit(self):
with open(os.path.join(self.plugin_dir, '__init__'), 'w') as f: with open(os.path.join(self.plugin_dir, '__init__'), 'w') as f:
f.write('') f.write('')
os.chmod(os.path.join(self.plugin_dir, '__init__'), 0644) os.chmod(os.path.join(self.plugin_dir, '__init__'), 0o644)
if sys.path[0] != self.plugin_dir: if sys.path[0] != self.plugin_dir:
sys.path[0:0] = [self.plugin_dir] sys.path[0:0] = [self.plugin_dir]
...@@ -111,7 +111,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase): ...@@ -111,7 +111,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
'debug': False, 'debug': False,
'name': promise_name, 'name': promise_name,
'path': os.path.join(self.plugin_dir, promise_name), 'path': os.path.join(self.plugin_dir, promise_name),
'queue': Queue.Queue(), 'queue': queue.Queue(),
} }
def createPromiseProcess(self, promise_name, check_anomaly=False, wrap=False): def createPromiseProcess(self, promise_name, check_anomaly=False, wrap=False):
...@@ -128,21 +128,20 @@ class TestSlapOSPromiseMixin(unittest.TestCase): ...@@ -128,21 +128,20 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
wrap=wrap, wrap=wrap,
) )
def writeFile(self, path, content, mode=0644): def writeFile(self, path, content, mode=0o644):
with open(path, 'w') as f: with open(path, 'w') as f:
f.write(content) f.write(content)
os.chmod(path, mode) os.chmod(path, mode)
def generatePromiseScript(self, name, success=True, failure_count=1, content="", def generatePromiseScript(self, name, success=True, failure_count=1, content="",
periodicity=0.03): periodicity=0.03):
promise_content = """from zope import interface as zope_interface promise_content = """from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise from slapos.grid.promise import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise): class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def __init__(self, config): def __init__(self, config):
GenericPromise.__init__(self, config) GenericPromise.__init__(self, config)
self.setPeriodicity(minute=%(periodicity)s) self.setPeriodicity(minute=%(periodicity)s)
...@@ -186,7 +185,9 @@ class TestSlapOSPromiseLauncher(TestSlapOSPromiseMixin): ...@@ -186,7 +185,9 @@ class TestSlapOSPromiseLauncher(TestSlapOSPromiseMixin):
with self.assertRaises(ImportError) as exc: with self.assertRaises(ImportError) as exc:
promise_module = promise_process._loadPromiseModule() promise_module = promise_process._loadPromiseModule()
self.assertEqual(exc.exception.message, 'No module named my_promise_no_py') self.assertEqual(str(exc.exception), 'No module named %s' %
("'%s'" % promise_name if six.PY3 else
promise_name))
def test_promise_match_interface_no_implement(self): def test_promise_match_interface_no_implement(self):
promise_name = 'my_promise_noimplement.py' promise_name = 'my_promise_noimplement.py'
...@@ -208,19 +209,18 @@ class RunPromise(GenericPromise): ...@@ -208,19 +209,18 @@ class RunPromise(GenericPromise):
with self.assertRaises(RuntimeError) as exc: with self.assertRaises(RuntimeError) as exc:
promise_module = promise_process._loadPromiseModule() promise_module = promise_process._loadPromiseModule()
message = "RunPromise class in my_promise_noimplement.py must implements" \ message = "RunPromise class in my_promise_noimplement.py must implement" \
" 'IPromise' interface. zope_interface.implements(interface.IPromise) is missing ?" " 'IPromise' interface. @implementer(interface.IPromise) is missing ?"
self.assertEqual(exc.exception.message, message) self.assertEqual(str(exc.exception), message)
def test_promise_match_interface_no_generic(self): def test_promise_match_interface_no_generic(self):
promise_name = 'my_promise_nogeneric.py' promise_name = 'my_promise_nogeneric.py'
promise_content = """from zope import interface as zope_interface promise_content = """from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
@implementer(interface.IPromise)
class RunPromise(object): class RunPromise(object):
zope_interface.implements(interface.IPromise)
def __init__(self, config): def __init__(self, config):
pass pass
...@@ -236,18 +236,17 @@ class RunPromise(object): ...@@ -236,18 +236,17 @@ class RunPromise(object):
with self.assertRaises(RuntimeError) as exc: with self.assertRaises(RuntimeError) as exc:
promise_module = promise_process._loadPromiseModule() promise_module = promise_process._loadPromiseModule()
self.assertEqual(exc.exception.message, 'RunPromise class is not a subclass of GenericPromise class.') self.assertEqual(str(exc.exception), 'RunPromise class is not a subclass of GenericPromise class.')
def test_promise_match_interface_no_sense(self): def test_promise_match_interface_no_sense(self):
promise_name = 'my_promise_nosense.py' promise_name = 'my_promise_nosense.py'
promise_content = """from zope import interface as zope_interface promise_content = """from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise from slapos.grid.promise import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise): class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def __init__(self, config): def __init__(self, config):
pass pass
...@@ -263,23 +262,22 @@ class RunPromise(GenericPromise): ...@@ -263,23 +262,22 @@ class RunPromise(GenericPromise):
with self.assertRaises(TypeError) as exc: with self.assertRaises(TypeError) as exc:
promise_module = promise_process._loadPromiseModule() promise_module = promise_process._loadPromiseModule()
promise = promise_module.RunPromise({}) promise = promise_module.RunPromise({})
self.assertEqual(exc.exception.message, self.assertEqual(str(exc.exception),
"Can't instantiate abstract class RunPromise with abstract methods sense") "Can't instantiate abstract class RunPromise with abstract methods sense")
def test_promise_extra_config(self): def test_promise_extra_config(self):
promise_name = 'my_promise_extra.py' promise_name = 'my_promise_extra.py'
config_dict = {'foo': 'bar', 'my-config': 4522111, config_dict = {'foo': 'bar', 'my-config': 4522111,
'text': 'developers \ninformation, sample\n\ner'} 'text': 'developers \ninformation, sample\n\ner'}
promise_content = """from zope import interface as zope_interface promise_content = """from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise from slapos.grid.promise import GenericPromise
%(config_name)s = %(config_content)s %(config_name)s = %(config_content)s
@implementer(interface.IPromise)
class RunPromise(GenericPromise): class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def sense(self): def sense(self):
pass pass
...@@ -298,17 +296,19 @@ class RunPromise(GenericPromise): ...@@ -298,17 +296,19 @@ class RunPromise(GenericPromise):
def test_promise_extra_config_reserved_name(self): def test_promise_extra_config_reserved_name(self):
promise_name = 'my_promise_extra.py' promise_name = 'my_promise_extra.py'
config_dict = {'name': 'bar', 'my-config': 4522111} from collections import OrderedDict
promise_content = """from zope import interface as zope_interface config_dict = OrderedDict([('name', 'bar'), ('my-config', 4522111)])
promise_content = """from collections import OrderedDict
from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise from slapos.grid.promise import GenericPromise
%(config_name)s = %(config_content)s %(config_name)s = %(config_content)s
@implementer(interface.IPromise)
class RunPromise(GenericPromise): class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def sense(self): def sense(self):
pass pass
...@@ -321,7 +321,7 @@ class RunPromise(GenericPromise): ...@@ -321,7 +321,7 @@ class RunPromise(GenericPromise):
with self.assertRaises(ValueError) as exc: with self.assertRaises(ValueError) as exc:
promise_module = promise_process._loadPromiseModule() promise_module = promise_process._loadPromiseModule()
self.assertEqual(exc.exception.message, "Extra parameter name 'name' cannot be used.\n%s" % config_dict) self.assertEqual(str(exc.exception), "Extra parameter name 'name' cannot be used.\n%s" % config_dict)
def test_runpromise(self): def test_runpromise(self):
promise_name = 'my_promise.py' promise_name = 'my_promise.py'
...@@ -540,7 +540,7 @@ class RunPromise(GenericPromise): ...@@ -540,7 +540,7 @@ class RunPromise(GenericPromise):
# run promise will fail when promise fail (usefull for slapgrid) # run promise will fail when promise fail (usefull for slapgrid)
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() self.launcher.run()
self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
# force to reload the module without rerun python # force to reload the module without rerun python
os.system('rm %s/*.pyc' % self.plugin_dir) os.system('rm %s/*.pyc' % self.plugin_dir)
...@@ -643,7 +643,7 @@ class RunPromise(GenericPromise): ...@@ -643,7 +643,7 @@ class RunPromise(GenericPromise):
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() self.launcher.run()
self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
self.assertTrue(os.path.exists(first_state_file)) self.assertTrue(os.path.exists(first_state_file))
self.assertTrue(os.path.exists(second_state_file)) self.assertTrue(os.path.exists(second_state_file))
...@@ -658,7 +658,7 @@ class RunPromise(GenericPromise): ...@@ -658,7 +658,7 @@ class RunPromise(GenericPromise):
time.sleep(2) time.sleep(2)
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() # only my_first_promise will run but second_promise still failing self.launcher.run() # only my_first_promise will run but second_promise still failing
self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
first_result = json.load(open(first_state_file)) first_result = json.load(open(first_state_file))
second_result = json.load(open(second_state_file)) second_result = json.load(open(second_state_file))
...@@ -670,7 +670,7 @@ class RunPromise(GenericPromise): ...@@ -670,7 +670,7 @@ class RunPromise(GenericPromise):
self.configureLauncher() self.configureLauncher()
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() self.launcher.run()
self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
first_result = json.load(open(first_state_file)) first_result = json.load(open(first_state_file))
second_result = json.load(open(second_state_file)) second_result = json.load(open(second_state_file))
...@@ -693,7 +693,7 @@ class RunPromise(GenericPromise): ...@@ -693,7 +693,7 @@ class RunPromise(GenericPromise):
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() self.launcher.run()
self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
self.assertTrue(os.path.exists(first_state_file)) self.assertTrue(os.path.exists(first_state_file))
self.assertTrue(os.path.exists(second_state_file)) self.assertTrue(os.path.exists(second_state_file))
...@@ -708,7 +708,7 @@ class RunPromise(GenericPromise): ...@@ -708,7 +708,7 @@ class RunPromise(GenericPromise):
time.sleep(2) time.sleep(2)
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() # only my_first_promise will run but second_promise still failing self.launcher.run() # only my_first_promise will run but second_promise still failing
self.assertEqual(exc.exception.message, 'Promise %r failed.' % second_promise) self.assertEqual(str(exc.exception), 'Promise %r failed.' % second_promise)
first_result = json.load(open(first_state_file)) first_result = json.load(open(first_state_file))
second_result = json.load(open(second_state_file)) second_result = json.load(open(second_state_file))
...@@ -781,7 +781,7 @@ class RunPromise(GenericPromise): ...@@ -781,7 +781,7 @@ class RunPromise(GenericPromise):
f.write("""#!/bin/bash f.write("""#!/bin/bash
echo "success" echo "success"
""") """)
os.chmod(promise_path, 0744) os.chmod(promise_path, 0o744)
def test_method(result): def test_method(result):
self.called = True self.called = True
...@@ -810,13 +810,13 @@ echo "success" ...@@ -810,13 +810,13 @@ echo "success"
echo "This promise failed" echo "This promise failed"
exit 1 exit 1
""") """)
os.chmod(promise_path, 0744) os.chmod(promise_path, 0o744)
self.configureLauncher() self.configureLauncher()
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME) state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
with self.assertRaises(PromiseError) as exc: with self.assertRaises(PromiseError) as exc:
self.launcher.run() self.launcher.run()
self.assertEqual(exc.exception.message, 'Promise %r failed.' % promise_name) self.assertEqual(str(exc.exception), 'Promise %r failed.' % promise_name)
def test_runpromise_wrapped_mixed(self): def test_runpromise_wrapped_mixed(self):
self.called = 0 self.called = 0
...@@ -841,13 +841,13 @@ exit 1 ...@@ -841,13 +841,13 @@ exit 1
echo "promise 1 succeeded" echo "promise 1 succeeded"
exit 0 exit 0
""") """)
os.chmod(promise_path, 0744) os.chmod(promise_path, 0o744)
with open(promise_path2, 'w') as f: with open(promise_path2, 'w') as f:
f.write("""#!/bin/bash f.write("""#!/bin/bash
echo "promise 2 succeeded" echo "promise 2 succeeded"
exit 0 exit 0
""") """)
os.chmod(promise_path2, 0744) os.chmod(promise_path2, 0o744)
self.generatePromiseScript("first_promise.py", success=True) self.generatePromiseScript("first_promise.py", success=True)
self.generatePromiseScript("second_promise.py", success=True) self.generatePromiseScript("second_promise.py", success=True)
...@@ -989,7 +989,7 @@ exit 0 ...@@ -989,7 +989,7 @@ exit 0
sleep 20 sleep 20
echo "success" echo "success"
""") """)
os.chmod(promise_path, 0744) os.chmod(promise_path, 0o744)
def test_method(result): def test_method(result):
self.called = True self.called = True
...@@ -1025,7 +1025,7 @@ sleep 20 ...@@ -1025,7 +1025,7 @@ sleep 20
echo "success" echo "success"
exit 1 exit 1
""") """)
os.chmod(promise_path, 0744) os.chmod(promise_path, 0o744)
createPromise(first_promise_path) createPromise(first_promise_path)
createPromise(second_promise_path) createPromise(second_promise_path)
...@@ -1073,7 +1073,7 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1073,7 +1073,7 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
self.configureLauncher() self.configureLauncher()
self.generatePromiseScript(self.promise_name, periodicity=1, content=promise_content, success=success) self.generatePromiseScript(self.promise_name, periodicity=1, content=promise_content, success=success)
self.writeInit() self.writeInit()
self.queue = Queue.Queue() self.queue = queue.Queue()
self.promise_config = { self.promise_config = {
'log-folder': self.log_dir, 'log-folder': self.log_dir,
'partition-folder': self.partition_dir, 'partition-folder': self.partition_dir,
......
...@@ -25,16 +25,18 @@ ...@@ -25,16 +25,18 @@
# #
############################################################################## ##############################################################################
from __future__ import print_function
import logging import logging
import os import os
import unittest import unittest
import urlparse from six.moves.urllib import parse
import tempfile import tempfile
import httmock import httmock
import slapos.slap import slapos.slap
import xml_marshaller from slapos.util import dumps
class UndefinedYetException(Exception): class UndefinedYetException(Exception):
...@@ -51,11 +53,10 @@ class SlapMixin(unittest.TestCase): ...@@ -51,11 +53,10 @@ class SlapMixin(unittest.TestCase):
self.server_url = 'http://localhost/' self.server_url = 'http://localhost/'
else: else:
self.server_url = self._server_url self.server_url = self._server_url
print 'Testing against SLAP server %r' % self.server_url print('Testing against SLAP server %r' % self.server_url)
self.slap = slapos.slap.slap() self.slap = slapos.slap.slap()
self.partition_id = 'PARTITION_01' self.partition_id = 'PARTITION_01'
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'): os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
del os.environ['SLAPGRID_INSTANCE_ROOT']
def tearDown(self): def tearDown(self):
pass pass
...@@ -185,7 +186,7 @@ class TestSlap(SlapMixin): ...@@ -185,7 +186,7 @@ class TestSlap(SlapMixin):
self.slap.registerComputer(computer_guid) self.slap.registerComputer(computer_guid)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and qs == { and qs == {
'computer_reference': [computer_guid], 'computer_reference': [computer_guid],
...@@ -194,7 +195,7 @@ class TestSlap(SlapMixin): ...@@ -194,7 +195,7 @@ class TestSlap(SlapMixin):
partition = slapos.slap.ComputerPartition(computer_guid, partition_id) partition = slapos.slap.ComputerPartition(computer_guid, partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition) 'content': dumps(partition)
} }
else: else:
return {'status_code': 400} return {'status_code': 400}
...@@ -226,7 +227,7 @@ class TestSlap(SlapMixin): ...@@ -226,7 +227,7 @@ class TestSlap(SlapMixin):
partition_id = 'PARTITION_01' partition_id = 'PARTITION_01'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and qs == { and qs == {
'computer_reference': [computer_guid], 'computer_reference': [computer_guid],
...@@ -318,12 +319,12 @@ class TestSlap(SlapMixin): ...@@ -318,12 +319,12 @@ class TestSlap(SlapMixin):
software_release_url_list = ['1', '2'] software_release_url_list = ['1', '2']
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/getSoftwareReleaseListFromSoftwareProduct' if (url.path == '/getSoftwareReleaseListFromSoftwareProduct'
and qs == {'software_product_reference': [software_product_reference]}): and qs == {'software_product_reference': [software_product_reference]}):
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(software_release_url_list) 'content': dumps(software_release_url_list)
} }
with httmock.HTTMock(handler): with httmock.HTTMock(handler):
...@@ -344,12 +345,12 @@ class TestSlap(SlapMixin): ...@@ -344,12 +345,12 @@ class TestSlap(SlapMixin):
software_release_url_list = ['1', '2'] software_release_url_list = ['1', '2']
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/getSoftwareReleaseListFromSoftwareProduct' if (url.path == '/getSoftwareReleaseListFromSoftwareProduct'
and qs == {'software_release_url': [software_release_url]}): and qs == {'software_release_url': [software_release_url]}):
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(software_release_url_list) 'content': dumps(software_release_url_list)
} }
with httmock.HTTMock(handler): with httmock.HTTMock(handler):
...@@ -385,7 +386,7 @@ class TestSlap(SlapMixin): ...@@ -385,7 +386,7 @@ class TestSlap(SlapMixin):
""" """
hateoas_url = 'foo' hateoas_url = 'foo'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/getHateoasUrl'): if (url.path == '/getHateoasUrl'):
return { return {
'status_code': 200, 'status_code': 200,
...@@ -406,7 +407,7 @@ class TestSlap(SlapMixin): ...@@ -406,7 +407,7 @@ class TestSlap(SlapMixin):
""" """
hateoas_url = 'foo' hateoas_url = 'foo'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/getHateoasUrl'): if (url.path == '/getHateoasUrl'):
self.fail('slap should not have contacted master to get Hateoas URL.') self.fail('slap should not have contacted master to get Hateoas URL.')
...@@ -424,7 +425,7 @@ class TestSlap(SlapMixin): ...@@ -424,7 +425,7 @@ class TestSlap(SlapMixin):
""" """
hateoas_url = 'foo' hateoas_url = 'foo'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/getHateoasUrl'): if (url.path == '/getHateoasUrl'):
return { return {
'status_code': 404, 'status_code': 404,
...@@ -450,7 +451,7 @@ class TestComputer(SlapMixin): ...@@ -450,7 +451,7 @@ class TestComputer(SlapMixin):
slap.initializeConnection(self.server_url) slap.initializeConnection(self.server_url)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs and 'computer_reference' in qs
and 'computer_partition_reference' in qs): and 'computer_partition_reference' in qs):
...@@ -459,7 +460,7 @@ class TestComputer(SlapMixin): ...@@ -459,7 +460,7 @@ class TestComputer(SlapMixin):
qs['computer_partition_reference'][0]) qs['computer_partition_reference'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
elif (url.path == '/getFullComputerInformation' elif (url.path == '/getFullComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
...@@ -468,7 +469,7 @@ class TestComputer(SlapMixin): ...@@ -468,7 +469,7 @@ class TestComputer(SlapMixin):
slap_computer._computer_partition_list = [] slap_computer._computer_partition_list = []
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/requestComputerPartition': elif url.path == '/requestComputerPartition':
return {'status_code': 408} return {'status_code': 408}
...@@ -520,7 +521,7 @@ class TestComputer(SlapMixin): ...@@ -520,7 +521,7 @@ class TestComputer(SlapMixin):
self.slap.initializeConnection(self.server_url) self.slap.initializeConnection(self.server_url)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and qs == { and qs == {
'computer_reference': [self.computer_guid], 'computer_reference': [self.computer_guid],
...@@ -529,7 +530,7 @@ class TestComputer(SlapMixin): ...@@ -529,7 +530,7 @@ class TestComputer(SlapMixin):
partition = slapos.slap.ComputerPartition(self.computer_guid, partition_id) partition = slapos.slap.ComputerPartition(self.computer_guid, partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition) 'content': dumps(partition)
} }
elif (url.path == '/getFullComputerInformation' elif (url.path == '/getFullComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
...@@ -537,7 +538,7 @@ class TestComputer(SlapMixin): ...@@ -537,7 +538,7 @@ class TestComputer(SlapMixin):
slap_computer._computer_partition_list = [] slap_computer._computer_partition_list = []
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
else: else:
return {'status_code': 400} return {'status_code': 400}
...@@ -602,7 +603,7 @@ class TestComputerPartition(SlapMixin): ...@@ -602,7 +603,7 @@ class TestComputerPartition(SlapMixin):
partition_id = 'PARTITION_01' partition_id = 'PARTITION_01'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs and 'computer_reference' in qs
and 'computer_partition_reference' in qs): and 'computer_partition_reference' in qs):
...@@ -611,7 +612,7 @@ class TestComputerPartition(SlapMixin): ...@@ -611,7 +612,7 @@ class TestComputerPartition(SlapMixin):
qs['computer_partition_reference'][0]) qs['computer_partition_reference'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
elif (url.path == '/getComputerInformation' elif (url.path == '/getComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
...@@ -623,7 +624,7 @@ class TestComputerPartition(SlapMixin): ...@@ -623,7 +624,7 @@ class TestComputerPartition(SlapMixin):
slap_computer._computer_partition_list = [slap_partition] slap_computer._computer_partition_list = [slap_partition]
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/requestComputerPartition': elif url.path == '/requestComputerPartition':
raise RequestWasCalled raise RequestWasCalled
...@@ -647,7 +648,7 @@ class TestComputerPartition(SlapMixin): ...@@ -647,7 +648,7 @@ class TestComputerPartition(SlapMixin):
partition_id = 'PARTITION_01' partition_id = 'PARTITION_01'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs and 'computer_reference' in qs
and 'computer_partition_reference' in qs): and 'computer_partition_reference' in qs):
...@@ -656,7 +657,7 @@ class TestComputerPartition(SlapMixin): ...@@ -656,7 +657,7 @@ class TestComputerPartition(SlapMixin):
qs['computer_partition_reference'][0]) qs['computer_partition_reference'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
elif (url.path == '/getComputerInformation' elif (url.path == '/getComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
...@@ -668,7 +669,7 @@ class TestComputerPartition(SlapMixin): ...@@ -668,7 +669,7 @@ class TestComputerPartition(SlapMixin):
slap_computer._computer_partition_list = [slap_partition] slap_computer._computer_partition_list = [slap_partition]
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/requestComputerPartition': elif url.path == '/requestComputerPartition':
return {'status_code': 408} return {'status_code': 408}
...@@ -691,7 +692,7 @@ class TestComputerPartition(SlapMixin): ...@@ -691,7 +692,7 @@ class TestComputerPartition(SlapMixin):
partition_id = 'PARTITION_01' partition_id = 'PARTITION_01'
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and if (url.path == '/registerComputerPartition' and
'computer_reference' in qs and 'computer_reference' in qs and
'computer_partition_reference' in qs): 'computer_partition_reference' in qs):
...@@ -700,7 +701,7 @@ class TestComputerPartition(SlapMixin): ...@@ -700,7 +701,7 @@ class TestComputerPartition(SlapMixin):
qs['computer_partition_reference'][0]) qs['computer_partition_reference'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
elif (url.path == '/getComputerInformation' elif (url.path == '/getComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
...@@ -712,7 +713,7 @@ class TestComputerPartition(SlapMixin): ...@@ -712,7 +713,7 @@ class TestComputerPartition(SlapMixin):
slap_computer._computer_partition_list = [slap_partition] slap_computer._computer_partition_list = [slap_partition]
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/requestComputerPartition': elif url.path == '/requestComputerPartition':
return {'status_code': 408} return {'status_code': 408}
...@@ -740,7 +741,7 @@ class TestComputerPartition(SlapMixin): ...@@ -740,7 +741,7 @@ class TestComputerPartition(SlapMixin):
computer_guid = self._getTestComputerId() computer_guid = self._getTestComputerId()
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and if (url.path == '/registerComputerPartition' and
'computer_reference' in qs and 'computer_reference' in qs and
'computer_partition_reference' in qs): 'computer_partition_reference' in qs):
...@@ -749,7 +750,7 @@ class TestComputerPartition(SlapMixin): ...@@ -749,7 +750,7 @@ class TestComputerPartition(SlapMixin):
qs['computer_partition_reference'][0]) qs['computer_partition_reference'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
elif (url.path == '/getComputerInformation' and 'computer_id' in qs): elif (url.path == '/getComputerInformation' and 'computer_id' in qs):
slap_computer = slapos.slap.Computer(qs['computer_id'][0]) slap_computer = slapos.slap.Computer(qs['computer_id'][0])
...@@ -760,7 +761,7 @@ class TestComputerPartition(SlapMixin): ...@@ -760,7 +761,7 @@ class TestComputerPartition(SlapMixin):
slap_computer._computer_partition_list = [slap_partition] slap_computer._computer_partition_list = [slap_partition]
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/requestComputerPartition': elif url.path == '/requestComputerPartition':
from slapos.slap.slap import SoftwareInstance from slapos.slap.slap import SoftwareInstance
...@@ -769,7 +770,7 @@ class TestComputerPartition(SlapMixin): ...@@ -769,7 +770,7 @@ class TestComputerPartition(SlapMixin):
slap_computer_partition_id=requested_partition_id) slap_computer_partition_id=requested_partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
else: else:
return {'status_code': 404} return {'status_code': 404}
...@@ -801,7 +802,7 @@ class TestComputerPartition(SlapMixin): ...@@ -801,7 +802,7 @@ class TestComputerPartition(SlapMixin):
transaction_file_path = os.path.join(partition_root, transaction_file_name) transaction_file_path = os.path.join(partition_root, transaction_file_name)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs and 'computer_reference' in qs
and 'computer_partition_reference' in qs): and 'computer_partition_reference' in qs):
...@@ -810,7 +811,7 @@ class TestComputerPartition(SlapMixin): ...@@ -810,7 +811,7 @@ class TestComputerPartition(SlapMixin):
qs['computer_partition_reference'][0]) qs['computer_partition_reference'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
elif (url.path == '/getComputerInformation' elif (url.path == '/getComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
...@@ -822,7 +823,7 @@ class TestComputerPartition(SlapMixin): ...@@ -822,7 +823,7 @@ class TestComputerPartition(SlapMixin):
slap_computer._computer_partition_list = [slap_partition] slap_computer._computer_partition_list = [slap_partition]
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/requestComputerPartition': elif url.path == '/requestComputerPartition':
raise RequestWasCalled raise RequestWasCalled
...@@ -848,7 +849,7 @@ class TestComputerPartition(SlapMixin): ...@@ -848,7 +849,7 @@ class TestComputerPartition(SlapMixin):
'software_type', 'myref') 'software_type', 'myref')
self.assertTrue(os.path.exists(transaction_file_path)) self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f: with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n') content_list = f.read().splitlines()
self.assertEqual(content_list, ['myref']) self.assertEqual(content_list, ['myref'])
# Not override # Not override
...@@ -856,7 +857,7 @@ class TestComputerPartition(SlapMixin): ...@@ -856,7 +857,7 @@ class TestComputerPartition(SlapMixin):
self.computer_guid, partition_id) self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path)) self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f: with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n') content_list = f.read().splitlines()
self.assertEqual(content_list, ['myref']) self.assertEqual(content_list, ['myref'])
# Request a second instance # Request a second instance
...@@ -865,8 +866,8 @@ class TestComputerPartition(SlapMixin): ...@@ -865,8 +866,8 @@ class TestComputerPartition(SlapMixin):
'http://server/new/' + self._getTestComputerId(), 'http://server/new/' + self._getTestComputerId(),
'software_type', 'mysecondref') 'software_type', 'mysecondref')
with open(transaction_file_path, 'r') as f: with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n') content_list = f.read().splitlines()
self.assertEqual(list(set(content_list)), ['myref', 'mysecondref']) self.assertEqual(sorted(content_list), ['myref', 'mysecondref'])
def _test_new_computer_partition_state(self, state): def _test_new_computer_partition_state(self, state):
""" """
...@@ -879,7 +880,7 @@ class TestComputerPartition(SlapMixin): ...@@ -879,7 +880,7 @@ class TestComputerPartition(SlapMixin):
slap.initializeConnection(self.server_url) slap.initializeConnection(self.server_url)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and if (url.path == '/registerComputerPartition' and
qs['computer_reference'][0] == computer_guid and qs['computer_reference'][0] == computer_guid and
qs['computer_partition_reference'][0] == partition_id): qs['computer_partition_reference'][0] == partition_id):
...@@ -887,7 +888,7 @@ class TestComputerPartition(SlapMixin): ...@@ -887,7 +888,7 @@ class TestComputerPartition(SlapMixin):
computer_guid, partition_id) computer_guid, partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition) 'content': dumps(partition)
} }
else: else:
return {'status_code': 404} return {'status_code': 404}
...@@ -923,7 +924,7 @@ class TestComputerPartition(SlapMixin): ...@@ -923,7 +924,7 @@ class TestComputerPartition(SlapMixin):
slap.initializeConnection(self.server_url) slap.initializeConnection(self.server_url)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and if (url.path == '/registerComputerPartition' and
qs['computer_reference'][0] == computer_guid and qs['computer_reference'][0] == computer_guid and
qs['computer_partition_reference'][0] == partition_id): qs['computer_partition_reference'][0] == partition_id):
...@@ -931,10 +932,10 @@ class TestComputerPartition(SlapMixin): ...@@ -931,10 +932,10 @@ class TestComputerPartition(SlapMixin):
computer_guid, partition_id) computer_guid, partition_id)
return { return {
'statu_code': 200, 'statu_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition) 'content': dumps(partition)
} }
elif url.path == '/softwareInstanceError': elif url.path == '/softwareInstanceError':
parsed_qs_body = urlparse.parse_qs(req.body) parsed_qs_body = parse.parse_qs(req.body)
# XXX: why do we have computer_id and not computer_reference? # XXX: why do we have computer_id and not computer_reference?
# XXX: why do we have computer_partition_id and not # XXX: why do we have computer_partition_id and not
# computer_partition_reference? # computer_partition_reference?
...@@ -995,7 +996,7 @@ class TestSoftwareRelease(SlapMixin): ...@@ -995,7 +996,7 @@ class TestSoftwareRelease(SlapMixin):
slap.initializeConnection(self.server_url) slap.initializeConnection(self.server_url)
def handler(url, req): def handler(url, req):
qs = urlparse.parse_qs(req.body) qs = parse.parse_qs(req.body)
if (url.path == '/softwareReleaseError' and if (url.path == '/softwareReleaseError' and
qs['computer_id'][0] == computer_guid and qs['computer_id'][0] == computer_guid and
qs['url'][0] == software_release_uri and qs['url'][0] == software_release_uri and
...@@ -1079,7 +1080,7 @@ class TestOpenOrder(SlapMixin): ...@@ -1079,7 +1080,7 @@ class TestOpenOrder(SlapMixin):
slap_computer_partition_id=requested_partition_id) slap_computer_partition_id=requested_partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
with httmock.HTTMock(handler): with httmock.HTTMock(handler):
...@@ -1106,7 +1107,7 @@ class TestOpenOrder(SlapMixin): ...@@ -1106,7 +1107,7 @@ class TestOpenOrder(SlapMixin):
slap_computer_partition_id=requested_partition_id) slap_computer_partition_id=requested_partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
...@@ -1139,7 +1140,7 @@ class TestOpenOrder(SlapMixin): ...@@ -1139,7 +1140,7 @@ class TestOpenOrder(SlapMixin):
slap_computer_partition_id=requested_partition_id) slap_computer_partition_id=requested_partition_id)
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition) 'content': dumps(slap_partition)
} }
with httmock.HTTMock(handler): with httmock.HTTMock(handler):
...@@ -1205,7 +1206,7 @@ class TestSoftwareProductCollection(SlapMixin): ...@@ -1205,7 +1206,7 @@ class TestSoftwareProductCollection(SlapMixin):
self.assertEqual(self.product_collection.foo, '0') self.assertEqual(self.product_collection.foo, '0')
if __name__ == '__main__': if __name__ == '__main__':
print 'You can point to any SLAP server by setting TEST_SLAP_SERVER_URL '\ print('You can point to any SLAP server by setting TEST_SLAP_SERVER_URL'
'environment variable' ' environment variable')
unittest.main() unittest.main()
...@@ -26,6 +26,8 @@ ...@@ -26,6 +26,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import print_function
import glob import glob
import logging import logging
import slapos.format import slapos.format
...@@ -49,6 +51,8 @@ import mock ...@@ -49,6 +51,8 @@ import mock
from .slapgrid import DummyManager from .slapgrid import DummyManager
import six
USER_LIST = [] USER_LIST = []
GROUP_LIST = [] GROUP_LIST = []
INTERFACE_DICT = {} INTERFACE_DICT = {}
...@@ -89,7 +93,7 @@ class FakeCallAndRead: ...@@ -89,7 +93,7 @@ class FakeCallAndRead:
retval = 0, 'UP' retval = 0, 'UP'
global INTERFACE_DICT global INTERFACE_DICT
if 'useradd' in argument_list: if 'useradd' in argument_list:
print argument_list print(argument_list)
global USER_LIST global USER_LIST
username = argument_list[-1] username = argument_list[-1]
if username == '-r': if username == '-r':
...@@ -130,7 +134,7 @@ class LoggableWrapper: ...@@ -130,7 +134,7 @@ class LoggableWrapper:
def __call__(self, *args, **kwargs): def __call__(self, *args, **kwargs):
arg_list = [repr(x) for x in args] + [ arg_list = [repr(x) for x in args] + [
'%s=%r' % (x, y) for x, y in kwargs.iteritems()] '%s=%r' % (x, y) for x, y in six.iteritems(kwargs)]
self.__logger.debug('%s(%s)' % (self.__name, ', '.join(arg_list))) self.__logger.debug('%s(%s)' % (self.__name, ', '.join(arg_list)))
...@@ -197,6 +201,8 @@ class SlapformatMixin(unittest.TestCase): ...@@ -197,6 +201,8 @@ class SlapformatMixin(unittest.TestCase):
self.netifaces = NetifacesMock() self.netifaces = NetifacesMock()
self.saved_netifaces = {} self.saved_netifaces = {}
for fake in vars(NetifacesMock): for fake in vars(NetifacesMock):
if fake.startswith("__"):
continue
self.saved_netifaces[fake] = getattr(netifaces, fake, None) self.saved_netifaces[fake] = getattr(netifaces, fake, None)
setattr(netifaces, fake, getattr(self.netifaces, fake)) setattr(netifaces, fake, getattr(self.netifaces, fake))
...@@ -208,6 +214,8 @@ class SlapformatMixin(unittest.TestCase): ...@@ -208,6 +214,8 @@ class SlapformatMixin(unittest.TestCase):
def patchPwd(self): def patchPwd(self):
self.saved_pwd = {} self.saved_pwd = {}
for fake in vars(PwdMock): for fake in vars(PwdMock):
if fake.startswith("__"):
continue
self.saved_pwd[fake] = getattr(pwd, fake, None) self.saved_pwd[fake] = getattr(pwd, fake, None)
setattr(pwd, fake, getattr(PwdMock, fake)) setattr(pwd, fake, getattr(PwdMock, fake))
...@@ -219,6 +227,8 @@ class SlapformatMixin(unittest.TestCase): ...@@ -219,6 +227,8 @@ class SlapformatMixin(unittest.TestCase):
def patchTime(self): def patchTime(self):
self.saved_time = {} self.saved_time = {}
for fake in vars(TimeMock): for fake in vars(TimeMock):
if fake.startswith("__"):
continue
self.saved_time[fake] = getattr(time, fake, None) self.saved_time[fake] = getattr(time, fake, None)
setattr(time, fake, getattr(TimeMock, fake)) setattr(time, fake, getattr(TimeMock, fake))
...@@ -230,6 +240,8 @@ class SlapformatMixin(unittest.TestCase): ...@@ -230,6 +240,8 @@ class SlapformatMixin(unittest.TestCase):
def patchGrp(self): def patchGrp(self):
self.saved_grp = {} self.saved_grp = {}
for fake in vars(GrpMock): for fake in vars(GrpMock):
if fake.startswith("__"):
continue
self.saved_grp[fake] = getattr(grp, fake, None) self.saved_grp[fake] = getattr(grp, fake, None)
setattr(grp, fake, getattr(GrpMock, fake)) setattr(grp, fake, getattr(GrpMock, fake))
......
...@@ -38,13 +38,13 @@ import tempfile ...@@ -38,13 +38,13 @@ import tempfile
import textwrap import textwrap
import time import time
import unittest import unittest
import urlparse import six
from six.moves.urllib import parse
import json import json
import re import re
import xml_marshaller
from mock import patch from mock import patch
from zope import interface from zope.interface import implementer
import slapos.slap.slap import slapos.slap.slap
import slapos.grid.utils import slapos.grid.utils
...@@ -57,6 +57,7 @@ from slapos.manager.interface import IManager ...@@ -57,6 +57,7 @@ from slapos.manager.interface import IManager
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
import slapos.grid.SlapObject import slapos.grid.SlapObject
from slapos import manager as slapmanager from slapos import manager as slapmanager
from slapos.util import dumps
import httmock import httmock
...@@ -113,7 +114,7 @@ class BasicMixin(object): ...@@ -113,7 +114,7 @@ class BasicMixin(object):
self.manager_list = [] self.manager_list = []
self.software_root = os.path.join(self._tempdir, 'software') self.software_root = os.path.join(self._tempdir, 'software')
self.instance_root = os.path.join(self._tempdir, 'instance') self.instance_root = os.path.join(self._tempdir, 'instance')
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'): if 'SLAPGRID_INSTANCE_ROOT' in os.environ:
del os.environ['SLAPGRID_INSTANCE_ROOT'] del os.environ['SLAPGRID_INSTANCE_ROOT']
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
self.setSlapgrid() self.setSlapgrid()
...@@ -197,7 +198,7 @@ class BasicMixin(object): ...@@ -197,7 +198,7 @@ class BasicMixin(object):
instance_list.append('etc') instance_list.append('etc')
instance_list.append('var') instance_list.append('var')
instance_list.append('supervisord.socket') instance_list.append('supervisord.socket')
self.assertItemsEqual(os.listdir(self.instance_root), instance_list) six.assertCountEqual(self, os.listdir(self.instance_root), instance_list)
def tearDown(self): def tearDown(self):
# XXX: Hardcoded pid, as it is not configurable in slapos # XXX: Hardcoded pid, as it is not configurable in slapos
...@@ -314,21 +315,21 @@ class ComputerForTest(object): ...@@ -314,21 +315,21 @@ class ComputerForTest(object):
""" """
self.sequence.append(url.path) self.sequence.append(url.path)
if req.method == 'GET': if req.method == 'GET':
qs = urlparse.parse_qs(url.query) qs = parse.parse_qs(url.query)
else: else:
qs = urlparse.parse_qs(req.body) qs = parse.parse_qs(req.body)
if (url.path == '/getFullComputerInformation' if (url.path == '/getFullComputerInformation'
and 'computer_id' in qs): and 'computer_id' in qs):
slap_computer = self.getComputer(qs['computer_id'][0]) slap_computer = self.getComputer(qs['computer_id'][0])
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer) 'content': dumps(slap_computer)
} }
elif url.path == '/getHostingSubscriptionIpList': elif url.path == '/getHostingSubscriptionIpList':
ip_address_list = self.ip_address_list ip_address_list = self.ip_address_list
return { return {
'status_code': 200, 'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(ip_address_list) 'content': dumps(ip_address_list)
} }
if req.method == 'POST' and 'computer_partition_id' in qs: if req.method == 'POST' and 'computer_partition_id' in qs:
instance = self.instance_list[int(qs['computer_partition_id'][0])] instance = self.instance_list[int(qs['computer_partition_id'][0])]
...@@ -549,9 +550,8 @@ touch worked"""): ...@@ -549,9 +550,8 @@ touch worked"""):
fout.write(str(periodicity)) fout.write(str(periodicity))
@implementer(IManager)
class DummyManager(object): class DummyManager(object):
interface.implements(IManager)
def __init__(self): def __init__(self):
self.sequence = [] self.sequence = []
...@@ -584,7 +584,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase): ...@@ -584,7 +584,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
with httmock.HTTMock(computer.request_handler): with httmock.HTTMock(computer.request_handler):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual([]) self.assertInstanceDirectoryListEqual([])
self.assertItemsEqual(os.listdir(self.software_root), []) six.assertCountEqual(self, os.listdir(self.software_root), [])
st = os.stat(os.path.join(self.instance_root, 'var')) st = os.stat(os.path.join(self.instance_root, 'var'))
self.assertEqual(stat.S_IMODE(st.st_mode), 0o755) self.assertEqual(stat.S_IMODE(st.st_mode), 0o755)
...@@ -595,9 +595,9 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase): ...@@ -595,9 +595,9 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg', six.assertCountEqual(self, os.listdir(partition), ['.slapgrid', 'buildout.cfg',
'software_release', 'worked', '.slapos-retention-lock-delay']) 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
'/stoppedComputerPartition']) '/stoppedComputerPartition'])
...@@ -613,9 +613,9 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase): ...@@ -613,9 +613,9 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg', six.assertCountEqual(self, os.listdir(partition), ['.slapgrid', 'buildout.cfg',
'software_release', 'worked', '.slapos-retention-lock-delay']) 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
'/stoppedComputerPartition']) '/stoppedComputerPartition'])
...@@ -632,8 +632,8 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase): ...@@ -632,8 +632,8 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
partition.requested_state = 'destroyed' partition.requested_state = 'destroyed'
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), []) six.assertCountEqual(self, os.listdir(partition.partition_path), [])
self.assertItemsEqual(os.listdir(self.software_root), []) six.assertCountEqual(self, os.listdir(self.software_root), [])
self.assertEqual(partition.sequence, []) self.assertEqual(partition.sequence, [])
def test_one_partition_started(self): def test_one_partition_started(self):
...@@ -644,12 +644,12 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase): ...@@ -644,12 +644,12 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
partition.software.setBuildout(WRAPPER_CONTENT) partition.software.setBuildout(WRAPPER_CONTENT)
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
'/startedComputerPartition']) '/startedComputerPartition'])
...@@ -663,12 +663,12 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase): ...@@ -663,12 +663,12 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
partition.software.setBuildout(WRAPPER_CONTENT) partition.software.setBuildout(WRAPPER_CONTENT)
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
'/startedComputerPartition']) '/startedComputerPartition'])
...@@ -680,7 +680,7 @@ exit 1 ...@@ -680,7 +680,7 @@ exit 1
""") """)
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', '.slapgrid-0-error.log']) '.slapos-retention-lock-delay', '.slapgrid-0-error.log'])
...@@ -702,27 +702,28 @@ mkdir -p etc/run && ...@@ -702,27 +702,28 @@ mkdir -p etc/run &&
( (
cat <<'HEREDOC' cat <<'HEREDOC'
#!%(python)s #!%(python)s
from __future__ import print_function
import signal import signal
def handler(signum, frame): def handler(signum, frame):
for i in range(30): for i in range(30):
print 'Signal handler called with signal', signum print('Signal handler called with signal', signum)
raise SystemExit raise SystemExit
signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGTERM, handler)
while True: while True:
print "Working" print("Working")
HEREDOC HEREDOC
)> etc/run/wrapper && )> etc/run/wrapper &&
chmod 755 etc/run/wrapper chmod 755 etc/run/wrapper
""" % {'python': sys.executable}) """ % {'python': sys.executable})
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
'/startedComputerPartition']) '/startedComputerPartition'])
...@@ -732,7 +733,7 @@ chmod 755 etc/run/wrapper ...@@ -732,7 +733,7 @@ chmod 755 etc/run/wrapper
instance.requested_state = 'stopped' instance.requested_state = 'stopped'
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertLogContent(wrapper_log, 'Signal handler called with signal 15') self.assertLogContent(wrapper_log, 'Signal handler called with signal 15')
...@@ -758,27 +759,28 @@ mkdir -p etc/run && ...@@ -758,27 +759,28 @@ mkdir -p etc/run &&
( (
cat <<'HEREDOC' cat <<'HEREDOC'
#!%(python)s #!%(python)s
from __future__ import print_function
import signal import signal
def handler(signum, frame): def handler(signum, frame):
for i in range(30): for i in range(30):
print 'Signal handler called with signal', signum print('Signal handler called with signal', signum)
raise SystemExit raise SystemExit
signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGTERM, handler)
while True: while True:
print "Working" print("Working")
HEREDOC HEREDOC
)> etc/run/wrapper && )> etc/run/wrapper &&
chmod 755 etc/run/wrapper chmod 755 etc/run/wrapper
""" % {'python': sys.executable}) """ % {'python': sys.executable})
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[instance.software.software_hash]) [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
...@@ -792,7 +794,7 @@ exit 1 ...@@ -792,7 +794,7 @@ exit 1
""") """)
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', '.slapgrid-0-error.log']) '.slapos-retention-lock-delay', '.slapgrid-0-error.log'])
...@@ -812,9 +814,9 @@ exit 1 ...@@ -812,9 +814,9 @@ exit 1
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) ['.slapgrid', 'buildout.cfg', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[instance.software.software_hash]) [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
...@@ -826,10 +828,10 @@ exit 1 ...@@ -826,10 +828,10 @@ exit 1
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.0_wrapper.log', 'etc', ['.slapgrid', '.0_wrapper.log', 'etc',
'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[instance.software.software_hash]) [instance.software.software_hash])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
...@@ -856,8 +858,8 @@ exit 1 ...@@ -856,8 +858,8 @@ exit 1
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), ['.slapgrid', dummy_file_name]) six.assertCountEqual(self, os.listdir(partition), ['.slapgrid', dummy_file_name])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
'/stoppedComputerPartition']) '/stoppedComputerPartition'])
...@@ -900,7 +902,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase): ...@@ -900,7 +902,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', '.0_daemon.log', 'buildout.cfg', ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
daemon_log = os.path.join(partition.partition_path, '.0_daemon.log') daemon_log = os.path.join(partition.partition_path, '.0_daemon.log')
...@@ -946,7 +948,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase): ...@@ -946,7 +948,7 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
time.sleep(1) time.sleep(1)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', '.0_daemon.log', 'buildout.cfg', ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
'launched', 'crashed']) 'launched', 'crashed'])
...@@ -1229,9 +1231,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1229,9 +1231,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay']) ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
timestamp_path = os.path.join(instance.partition_path, '.timestamp') timestamp_path = os.path.join(instance.partition_path, '.timestamp')
self.setSlapgrid() self.setSlapgrid()
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
...@@ -1249,10 +1251,10 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1249,10 +1251,10 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', ['.slapgrid', '.timestamp', 'buildout.cfg',
'software_release', 'worked', '.slapos-retention-lock-delay']) 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(self.launchSlapgrid(develop=True), self.assertEqual(self.launchSlapgrid(develop=True),
slapgrid.SLAPGRID_SUCCESS) slapgrid.SLAPGRID_SUCCESS)
...@@ -1272,9 +1274,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1272,9 +1274,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay']) ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
instance.timestamp = str(int(timestamp) - 1) instance.timestamp = str(int(timestamp) - 1)
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
self.assertEqual(instance.sequence, self.assertEqual(instance.sequence,
...@@ -1290,9 +1292,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1290,9 +1292,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay']) ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
instance.timestamp = str(int(timestamp) + 1) instance.timestamp = str(int(timestamp) + 1)
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
...@@ -1315,9 +1317,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1315,9 +1317,9 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self.launchSlapgrid() self.launchSlapgrid()
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay']) ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[instance.software.software_hash]) [instance.software.software_hash])
instance.timestamp = None instance.timestamp = None
self.launchSlapgrid() self.launchSlapgrid()
...@@ -1344,7 +1346,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1344,7 +1346,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self.launchSlapgrid() self.launchSlapgrid()
partition = os.path.join(self.instance_root, '0') partition = os.path.join(self.instance_root, '0')
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', ['.slapgrid', '.timestamp', 'buildout.cfg',
'software_release', 'worked', '.slapos-retention-lock-delay']) 'software_release', 'worked', '.slapos-retention-lock-delay'])
...@@ -1354,7 +1356,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase): ...@@ -1354,7 +1356,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
instance.install = lambda: None instance.install = lambda: None
self.launchSlapgrid() self.launchSlapgrid()
self.assertItemsEqual(os.listdir(partition), six.assertCountEqual(self, os.listdir(partition),
['.slapgrid', '.timestamp', 'buildout.cfg', ['.slapgrid', '.timestamp', 'buildout.cfg',
'software_release', 'worked', '.slapos-retention-lock-delay']) 'software_release', 'worked', '.slapos-retention-lock-delay'])
...@@ -1638,12 +1640,12 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1638,12 +1640,12 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
instance.software.setBuildout(WRAPPER_CONTENT) instance.software.setBuildout(WRAPPER_CONTENT)
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
...@@ -1656,8 +1658,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1656,8 +1658,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), []) six.assertCountEqual(self, os.listdir(instance.partition_path), [])
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[instance.software.software_hash]) [instance.software.software_hash])
# Assert supervisor stopped process # Assert supervisor stopped process
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
...@@ -1687,8 +1689,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1687,8 +1689,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), []) six.assertCountEqual(self, os.listdir(instance.partition_path), [])
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[instance.software.software_hash]) [instance.software.software_hash])
# Assert supervisor stopped process # Assert supervisor stopped process
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
...@@ -1709,12 +1711,12 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1709,12 +1711,12 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
instance.software.setBuildout(WRAPPER_CONTENT) instance.software.setBuildout(WRAPPER_CONTENT)
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', ['/getFullComputerInformation',
...@@ -1728,14 +1730,14 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1728,14 +1730,14 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file]) '.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file]) '.slapos-retention-lock-delay', request_list_file])
...@@ -1760,8 +1762,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1760,8 +1762,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), []) six.assertCountEqual(self, os.listdir(instance.partition_path), [])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, ['/getFullComputerInformation']) self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
def test_slapgrid_report_ignore_free_instance(self): def test_slapgrid_report_ignore_free_instance(self):
...@@ -1779,8 +1781,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1779,8 +1781,8 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), []) six.assertCountEqual(self, os.listdir(instance.partition_path), [])
self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash]) six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
self.assertEqual(computer.sequence, ['/getFullComputerInformation']) self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
...@@ -2033,7 +2035,7 @@ exit 1 ...@@ -2033,7 +2035,7 @@ exit 1
self.assertEqual(self.grid.processComputerPartitionList(), self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_FAIL) slapos.grid.slapgrid.SLAPGRID_FAIL)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', 'buildout.cfg', 'software_release', ['.slapgrid', 'buildout.cfg', 'software_release',
'.slapgrid-0-error.log']) '.slapgrid-0-error.log'])
...@@ -2059,7 +2061,7 @@ exit 1 ...@@ -2059,7 +2061,7 @@ exit 1
self.assertEqual(self.grid.processComputerPartitionList(), self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_FAIL) slapos.grid.slapgrid.SLAPGRID_FAIL)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), six.assertCountEqual(self, os.listdir(instance.partition_path),
['.slapgrid', 'buildout.cfg', 'software_release', ['.slapgrid', 'buildout.cfg', 'software_release',
'.slapgrid-0-error.log']) '.slapgrid-0-error.log'])
...@@ -2151,8 +2153,8 @@ echo "ERROR: $var" ...@@ -2151,8 +2153,8 @@ echo "ERROR: $var"
exit 1 exit 1
""") """)
os.chmod(self.firewall_cmd_add, 0755) os.chmod(self.firewall_cmd_add, 0o755)
os.chmod(self.firewall_cmd_remove, 0755) os.chmod(self.firewall_cmd_remove, 0o755)
firewall_conf= dict( firewall_conf= dict(
authorized_sources=source_ip, authorized_sources=source_ip,
...@@ -2543,9 +2545,9 @@ exit 0 ...@@ -2543,9 +2545,9 @@ exit 0
os.makedirs(pre_delete_dir, 0o700) os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f: with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content) f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754) os.chmod(pre_delete_script, 0o754)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
...@@ -2559,12 +2561,12 @@ exit 0 ...@@ -2559,12 +2561,12 @@ exit 0
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running) # Assert partition directory is not destroyed (pre-delete is running)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list', '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
'.slapos-request-transaction-0']) '.slapos-request-transaction-0'])
self.assertItemsEqual(os.listdir(self.software_root), six.assertCountEqual(self, os.listdir(self.software_root),
[partition.software.software_hash]) [partition.software.software_hash])
# wait until the pre-delete script is finished # wait until the pre-delete script is finished
...@@ -2573,7 +2575,7 @@ exit 0 ...@@ -2573,7 +2575,7 @@ exit 0
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), []) six.assertCountEqual(self, os.listdir(partition.partition_path), [])
def test_partition_destroy_pre_remove_with_retention_lock(self): def test_partition_destroy_pre_remove_with_retention_lock(self):
computer = ComputerForTest(self.software_root, self.instance_root) computer = ComputerForTest(self.software_root, self.instance_root)
...@@ -2593,7 +2595,7 @@ exit 0 ...@@ -2593,7 +2595,7 @@ exit 0
os.makedirs(pre_delete_dir, 0o700) os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f: with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content) f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754) os.chmod(pre_delete_script, 0o754)
self.assertTrue(os.path.exists(pre_delete_script)) self.assertTrue(os.path.exists(pre_delete_script))
manager_list = slapmanager.from_config({'manager_list': 'prerm'}) manager_list = slapmanager.from_config({'manager_list': 'prerm'})
...@@ -2602,7 +2604,7 @@ exit 0 ...@@ -2602,7 +2604,7 @@ exit 0
partition.requested_state = 'destroyed' partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (retention-delay-lock) # Assert partition directory is not destroyed (retention-delay-lock)
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release', ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', 'worked', '.slapos-retention-lock-delay',
'.slapos-retention-lock-date', '.slapos-request-transaction-0']) '.slapos-retention-lock-date', '.slapos-request-transaction-0'])
...@@ -2616,7 +2618,7 @@ exit 0 ...@@ -2616,7 +2618,7 @@ exit 0
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running) # Assert partition directory is not destroyed (pre-delete is running)
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release', ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', '.slapos-retention-lock-date', 'worked', '.slapos-retention-lock-delay', '.slapos-retention-lock-date',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list', '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
...@@ -2627,7 +2629,7 @@ exit 0 ...@@ -2627,7 +2629,7 @@ exit 0
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertItemsEqual(os.listdir(partition.partition_path), []) six.assertCountEqual(self, os.listdir(partition.partition_path), [])
def test_partition_destroy_pre_remove_script_not_stopped(self): def test_partition_destroy_pre_remove_script_not_stopped(self):
computer = ComputerForTest(self.software_root, self.instance_root) computer = ComputerForTest(self.software_root, self.instance_root)
...@@ -2640,7 +2642,7 @@ exit 0 ...@@ -2640,7 +2642,7 @@ exit 0
os.makedirs(pre_delete_dir, 0o700) os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f: with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content) f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754) os.chmod(pre_delete_script, 0o754)
self.assertEqual(partition.state, 'started') self.assertEqual(partition.state, 'started')
manager_list = slapmanager.from_config({'manager_list': 'prerm'}) manager_list = slapmanager.from_config({'manager_list': 'prerm'})
self.grid._manager_list = manager_list self.grid._manager_list = manager_list
...@@ -2648,7 +2650,7 @@ exit 0 ...@@ -2648,7 +2650,7 @@ exit 0
partition.requested_state = 'destroyed' partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running) # Assert partition directory is not destroyed (pre-delete is running)
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release', ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0', 'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list']) '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
...@@ -2662,7 +2664,7 @@ exit 0 ...@@ -2662,7 +2664,7 @@ exit 0
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), []) six.assertCountEqual(self, os.listdir(partition.partition_path), [])
def test_partition_destroy_pre_remove_script_run_as_partition_user(self): def test_partition_destroy_pre_remove_script_run_as_partition_user(self):
computer = ComputerForTest(self.software_root, self.instance_root) computer = ComputerForTest(self.software_root, self.instance_root)
...@@ -2675,7 +2677,7 @@ exit 0 ...@@ -2675,7 +2677,7 @@ exit 0
os.makedirs(pre_delete_dir, 0o700) os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f: with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content) f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754) os.chmod(pre_delete_script, 0o754)
manager_list = slapmanager.from_config({'manager_list': 'prerm'}) manager_list = slapmanager.from_config({'manager_list': 'prerm'})
self.grid._manager_list = manager_list self.grid._manager_list = manager_list
...@@ -2683,7 +2685,7 @@ exit 0 ...@@ -2683,7 +2685,7 @@ exit 0
partition.requested_state = 'destroyed' partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running) # Assert partition directory is not destroyed (pre-delete is running)
self.assertItemsEqual(os.listdir(partition.partition_path), six.assertCountEqual(self, os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release', ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0', 'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list']) '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
...@@ -2714,7 +2716,7 @@ exit 0 ...@@ -2714,7 +2716,7 @@ exit 0
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), []) six.assertCountEqual(self, os.listdir(partition.partition_path), [])
# test that slapgrid commands, like e.g. 'slapos node software' do not leak # test that slapgrid commands, like e.g. 'slapos node software' do not leak
......
...@@ -28,7 +28,8 @@ ...@@ -28,7 +28,8 @@
# #
############################################################################## ##############################################################################
import ConfigParser import six
from six.moves import configparser
import os import os
import logging import logging
import shutil import shutil
...@@ -38,14 +39,12 @@ import sys ...@@ -38,14 +39,12 @@ import sys
import tempfile import tempfile
import time import time
import unittest import unittest
import xml_marshaller
from xml_marshaller.xml_marshaller import loads, dumps
import slapos.proxy import slapos.proxy
import slapos.proxy.views as views import slapos.proxy.views as views
import slapos.slap import slapos.slap
import slapos.slap.slap import slapos.slap.slap
from slapos.util import sqlite_connect from slapos.util import loads, dumps, sqlite_connect, bytes2str
import sqlite3 import sqlite3
import pkg_resources import pkg_resources
...@@ -77,7 +76,8 @@ class BasicMixin(object): ...@@ -77,7 +76,8 @@ class BasicMixin(object):
self.startProxy() self.startProxy()
def createSlapOSConfigurationFile(self): def createSlapOSConfigurationFile(self):
open(self.slapos_cfg, 'w').write("""[slapos] with open(self.slapos_cfg, 'w') as f:
f.write("""[slapos]
software_root = %(tempdir)s/opt/slapgrid software_root = %(tempdir)s/opt/slapgrid
instance_root = %(tempdir)s/srv/slapgrid instance_root = %(tempdir)s/srv/slapgrid
master_url = %(proxyaddr)s master_url = %(proxyaddr)s
...@@ -106,7 +106,7 @@ database_uri = %(tempdir)s/lib/proxy.db ...@@ -106,7 +106,7 @@ database_uri = %(tempdir)s/lib/proxy.db
Set config for slapproxy and start it Set config for slapproxy and start it
""" """
conf = slapos.proxy.ProxyConfig(logger=logging.getLogger()) conf = slapos.proxy.ProxyConfig(logger=logging.getLogger())
configp = ConfigParser.SafeConfigParser() configp = configparser.SafeConfigParser()
configp.read(self.slapos_cfg) configp.read(self.slapos_cfg)
conf.mergeConfig(ProxyOption(self.proxy_db), configp) conf.mergeConfig(ProxyOption(self.proxy_db), configp)
conf.setConfig() conf.setConfig()
...@@ -142,7 +142,7 @@ database_uri = %(tempdir)s/lib/proxy.db ...@@ -142,7 +142,7 @@ database_uri = %(tempdir)s/lib/proxy.db
request_dict = { request_dict = {
'computer_id': self.computer_id, 'computer_id': self.computer_id,
'xml': xml_marshaller.xml_marshaller.dumps(computer_dict), 'xml': dumps(computer_dict),
} }
rv = self.app.post('/loadComputerConfigurationFromXML', rv = self.app.post('/loadComputerConfigurationFromXML',
data=request_dict) data=request_dict)
...@@ -168,7 +168,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -168,7 +168,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
""" """
rv = self.app.get('/getComputerInformation?computer_id=%s' % self.computer_id) rv = self.app.get('/getComputerInformation?computer_id=%s' % self.computer_id)
self.assertIsInstance( self.assertIsInstance(
xml_marshaller.xml_marshaller.loads(rv.data), loads(rv.data),
slapos.slap.Computer) slapos.slap.Computer)
self.assertTrue(os.path.exists(self.proxy_db)) self.assertTrue(os.path.exists(self.proxy_db))
...@@ -179,7 +179,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -179,7 +179,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
""" """
rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id) rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
self.assertIsInstance( self.assertIsInstance(
xml_marshaller.xml_marshaller.loads(rv.data), loads(rv.data),
slapos.slap.Computer) slapos.slap.Computer)
self.assertTrue(os.path.exists(self.proxy_db)) self.assertTrue(os.path.exists(self.proxy_db))
...@@ -197,7 +197,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -197,7 +197,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
""" """
self.add_free_partition(10) self.add_free_partition(10)
rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id) rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
computer = xml_marshaller.xml_marshaller.loads(rv.data) computer = loads(rv.data)
for slap_partition in computer._computer_partition_list: for slap_partition in computer._computer_partition_list:
self.assertIsNone(slap_partition._software_release_document) self.assertIsNone(slap_partition._software_release_document)
self.assertEqual(slap_partition._requested_state, 'destroyed') self.assertEqual(slap_partition._requested_state, 'destroyed')
...@@ -216,8 +216,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -216,8 +216,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
response = self.app.get('/getSoftwareReleaseListFromSoftwareProduct' response = self.app.get('/getSoftwareReleaseListFromSoftwareProduct'
'?software_product_reference=%s' %\ '?software_product_reference=%s' %\
software_product_reference) software_product_reference)
software_release_url_list = xml_marshaller.xml_marshaller.loads( software_release_url_list = loads(response.data)
response.data)
self.assertEqual( self.assertEqual(
software_release_url_list, software_release_url_list,
[software_release_url] [software_release_url]
...@@ -232,8 +231,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -232,8 +231,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
self.app_config['software_product_list'] = {'random': 'random'} self.app_config['software_product_list'] = {'random': 'random'}
response = self.app.get('/getSoftwareReleaseListFromSoftwareProduct' response = self.app.get('/getSoftwareReleaseListFromSoftwareProduct'
'?software_product_reference=idonotexist') '?software_product_reference=idonotexist')
software_release_url_list = xml_marshaller.xml_marshaller.loads( software_release_url_list = loads(response.data)
response.data)
self.assertEqual( self.assertEqual(
software_release_url_list, software_release_url_list,
[] []
...@@ -269,7 +267,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -269,7 +267,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
rv = self.app.get( rv = self.app.get(
'/getComputerPartitionCertificate?computer_id=%s&computer_partition_id=%s' % ( '/getComputerPartitionCertificate?computer_id=%s&computer_partition_id=%s' % (
self.computer_id, 'slappart0')) self.computer_id, 'slappart0'))
response = xml_marshaller.xml_marshaller.loads(rv.data) response = loads(rv.data)
self.assertEqual({'certificate': '', 'key': ''}, response) self.assertEqual({'certificate': '', 'key': ''}, response)
def test_computerBang(self): def test_computerBang(self):
...@@ -277,7 +275,7 @@ class TestInformation(BasicMixin, unittest.TestCase): ...@@ -277,7 +275,7 @@ class TestInformation(BasicMixin, unittest.TestCase):
Tests that computerBang method is implemented in slapproxy. Tests that computerBang method is implemented in slapproxy.
""" """
rv = self.app.post( '/computerBang?computer_id=%s' % ( self.computer_id)) rv = self.app.post( '/computerBang?computer_id=%s' % ( self.computer_id))
response = xml_marshaller.xml_marshaller.loads(rv.data) response = loads(rv.data)
self.assertEqual('', response) self.assertEqual('', response)
class MasterMixin(BasicMixin, unittest.TestCase): class MasterMixin(BasicMixin, unittest.TestCase):
...@@ -305,11 +303,10 @@ class MasterMixin(BasicMixin, unittest.TestCase): ...@@ -305,11 +303,10 @@ class MasterMixin(BasicMixin, unittest.TestCase):
'software_release': software_release, 'software_release': software_release,
'software_type': software_type, 'software_type': software_type,
'partition_reference': partition_reference, 'partition_reference': partition_reference,
'shared_xml': xml_marshaller.xml_marshaller.dumps(shared), 'shared_xml': dumps(shared),
'partition_parameter_xml': xml_marshaller.xml_marshaller.dumps( 'partition_parameter_xml': dumps(partition_parameter_kw),
partition_parameter_kw), 'filter_xml': dumps(filter_kw),
'filter_xml': xml_marshaller.xml_marshaller.dumps(filter_kw), 'state': dumps(state),
'state': xml_marshaller.xml_marshaller.dumps(state),
} }
return self.app.post('/requestComputerPartition', data=request_dict) return self.app.post('/requestComputerPartition', data=request_dict)
...@@ -321,7 +318,7 @@ class MasterMixin(BasicMixin, unittest.TestCase): ...@@ -321,7 +318,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
rv = self._requestComputerPartition(*args, **kwargs) rv = self._requestComputerPartition(*args, **kwargs)
self.assertEqual(rv._status_code, 200) self.assertEqual(rv._status_code, 200)
xml = rv.data xml = rv.data
software_instance = xml_marshaller.xml_marshaller.loads(xml) software_instance = loads(xml)
computer_partition = slapos.slap.ComputerPartition( computer_partition = slapos.slap.ComputerPartition(
software_instance.slap_computer_id, software_instance.slap_computer_id,
...@@ -343,7 +340,7 @@ class MasterMixin(BasicMixin, unittest.TestCase): ...@@ -343,7 +340,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
self.app.post('/setComputerPartitionConnectionXml', data={ self.app.post('/setComputerPartitionConnectionXml', data={
'computer_id': self.computer_id, 'computer_id': self.computer_id,
'computer_partition_id': partition_id, 'computer_partition_id': partition_id,
'connection_xml': xml_marshaller.xml_marshaller.dumps(connection_dict), 'connection_xml': dumps(connection_dict),
'slave_reference': slave_reference}) 'slave_reference': slave_reference})
def getPartitionInformation(self, computer_partition_id): def getPartitionInformation(self, computer_partition_id):
...@@ -351,7 +348,7 @@ class MasterMixin(BasicMixin, unittest.TestCase): ...@@ -351,7 +348,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
Return computer information as stored in proxy for corresponding id Return computer information as stored in proxy for corresponding id
""" """
rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id) rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
computer = xml_marshaller.xml_marshaller.loads(rv.data) computer = loads(rv.data)
for instance in computer._computer_partition_list: for instance in computer._computer_partition_list:
if instance._partition_id == computer_partition_id: if instance._partition_id == computer_partition_id:
return instance return instance
...@@ -1006,7 +1003,8 @@ class TestMultiMasterSupport(MasterMixin): ...@@ -1006,7 +1003,8 @@ class TestMultiMasterSupport(MasterMixin):
super(TestMultiMasterSupport, self).tearDown() super(TestMultiMasterSupport, self).tearDown()
def createExternalProxyConfigurationFile(self): def createExternalProxyConfigurationFile(self):
open(self.external_slapproxy_configuration_file_location, 'w').write("""[slapos] with open(self.external_slapproxy_configuration_file_location, 'w') as f:
f.write("""[slapos]
computer_id = %(external_computer_id)s computer_id = %(external_computer_id)s
[slapproxy] [slapproxy]
host = %(host)s host = %(host)s
...@@ -1040,7 +1038,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1040,7 +1038,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
self.external_proxy_slap._connection_helper.GET('/') self.external_proxy_slap._connection_helper.GET('/')
except slapos.slap.NotFoundError: except slapos.slap.NotFoundError:
break break
except slapos.slap.ConnectionError, socket.error: except (slapos.slap.ConnectionError, socket.error):
attempts = attempts + 1 attempts = attempts + 1
time.sleep(0.1) time.sleep(0.1)
else: else:
...@@ -1054,9 +1052,9 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1054,9 +1052,9 @@ database_uri = %(tempdir)s/lib/external_proxy.db
Overwrite default slapos configuration file to enable specific multimaster Overwrite default slapos configuration file to enable specific multimaster
behaviours. behaviours.
""" """
configuration = pkg_resources.resource_stream( configuration = bytes2str(pkg_resources.resource_string(
'slapos.tests.slapproxy', 'slapos_multimaster.cfg.in' 'slapos.tests.slapproxy', 'slapos_multimaster.cfg.in'
).read() % { )) % {
'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr, 'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr,
'external_proxy_host': self.external_proxy_host, 'external_proxy_host': self.external_proxy_host,
'external_proxy_port': self.external_proxy_port 'external_proxy_port': self.external_proxy_port
...@@ -1089,7 +1087,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1089,7 +1087,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
request_dict = { request_dict = {
'computer_id': self.computer_id, 'computer_id': self.computer_id,
'xml': xml_marshaller.xml_marshaller.dumps(computer_dict), 'xml': dumps(computer_dict),
} }
self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML', self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
data=request_dict) data=request_dict)
...@@ -1121,7 +1119,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1121,7 +1119,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
external_slap.initializeConnection(self.external_master_url) external_slap.initializeConnection(self.external_master_url)
external_computer = external_slap.registerComputer(self.external_computer_id) external_computer = external_slap.registerComputer(self.external_computer_id)
external_partition = external_computer.getComputerPartitionList()[0] external_partition = external_computer.getComputerPartitionList()[0]
for k, v in partition_parameter_kw.iteritems(): for k, v in six.iteritems(partition_parameter_kw):
self.assertEqual( self.assertEqual(
external_partition.getInstanceParameter(k), external_partition.getInstanceParameter(k),
v v
...@@ -1146,7 +1144,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1146,7 +1144,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
'/getFullComputerInformation?computer_id=%s' % self.computer_id '/getFullComputerInformation?computer_id=%s' % self.computer_id
).data) ).data)
partition = computer._computer_partition_list[0] partition = computer._computer_partition_list[0]
for k, v in partition_parameter_kw.iteritems(): for k, v in six.iteritems(partition_parameter_kw):
self.assertEqual( self.assertEqual(
partition.getInstanceParameter(k), partition.getInstanceParameter(k),
v v
...@@ -1251,7 +1249,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1251,7 +1249,7 @@ database_uri = %(tempdir)s/lib/external_proxy.db
'/getFullComputerInformation?computer_id=%s' % self.computer_id '/getFullComputerInformation?computer_id=%s' % self.computer_id
).data) ).data)
partition = computer._computer_partition_list[0] partition = computer._computer_partition_list[0]
for k, v in dummy_parameter_dict.iteritems(): for k, v in six.iteritems(dummy_parameter_dict):
self.assertEqual( self.assertEqual(
partition.getInstanceParameter(k), partition.getInstanceParameter(k),
v v
...@@ -1271,8 +1269,10 @@ class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, T ...@@ -1271,8 +1269,10 @@ class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, T
""" """
def setUp(self): def setUp(self):
super(TestMigrateVersion10To11, self).setUp() super(TestMigrateVersion10To11, self).setUp()
schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_10.sql') schema = bytes2str(pkg_resources.resource_string(
schema = schema.read() % dict(version='11') 'slapos.tests.slapproxy',
'database_dump_version_10.sql'
)) % dict(version='11')
self.db = sqlite_connect(self.proxy_db) self.db = sqlite_connect(self.proxy_db)
self.db.cursor().executescript(schema) self.db.cursor().executescript(schema)
self.db.commit() self.db.commit()
......
...@@ -65,9 +65,9 @@ class TestUtil(unittest.TestCase): ...@@ -65,9 +65,9 @@ class TestUtil(unittest.TestCase):
wanted_directory0 = os.path.join(root_slaptest, 'slap-write0') wanted_directory0 = os.path.join(root_slaptest, 'slap-write0')
wanted_directory1 = os.path.join(root_slaptest, 'slap-write0', 'write-slap1') wanted_directory1 = os.path.join(root_slaptest, 'slap-write0', 'write-slap1')
wanted_directory2 = os.path.join(root_slaptest, 'slap-write0', 'write-slap1', 'write-teste2') wanted_directory2 = os.path.join(root_slaptest, 'slap-write0', 'write-slap1', 'write-teste2')
wanted_directory_mkdir0 = os.makedirs(wanted_directory0, mode=0777) wanted_directory_mkdir0 = os.makedirs(wanted_directory0, mode=0o777)
wanted_directory_mkdir1 = os.makedirs(wanted_directory1, mode=0777) wanted_directory_mkdir1 = os.makedirs(wanted_directory1, mode=0o777)
wanted_directory_mkdir2 = os.makedirs(wanted_directory2, mode=0777) wanted_directory_mkdir2 = os.makedirs(wanted_directory2, mode=0o777)
create_file_txt = tempfile.mkstemp(suffix='.txt', prefix='tmp', dir=wanted_directory2, text=True) create_file_txt = tempfile.mkstemp(suffix='.txt', prefix='tmp', dir=wanted_directory2, text=True)
user = 'nobody' user = 'nobody'
try: try:
...@@ -109,23 +109,14 @@ class TestUtil(unittest.TestCase): ...@@ -109,23 +109,14 @@ class TestUtil(unittest.TestCase):
shutil.rmtree(root_slaptest) shutil.rmtree(root_slaptest)
def test_string_to_boolean_with_true_values(self): def test_string_to_boolean_with_true_values(self):
"""
Check that mkdir_p doesn't raise if directory already exist.
"""
for value in ['true', 'True', 'TRUE']: for value in ['true', 'True', 'TRUE']:
self.assertTrue(string_to_boolean(value)) self.assertTrue(string_to_boolean(value))
def test_string_to_boolean_with_false_values(self): def test_string_to_boolean_with_false_values(self):
"""
Check that mkdir_p doesn't raise if directory already exist.
"""
for value in ['false', 'False', 'False']: for value in ['false', 'False', 'False']:
self.assertFalse(string_to_boolean(value)) self.assertFalse(string_to_boolean(value))
def test_string_to_boolean_with_incorrect_values(self): def test_string_to_boolean_with_incorrect_values(self):
"""
Check that mkdir_p doesn't raise if directory already exist.
"""
for value in [True, False, 1, '1', 't', 'tru', 'truelle', 'f', 'fals', 'falsey']: for value in [True, False, 1, '1', 't', 'tru', 'truelle', 'f', 'fals', 'falsey']:
self.assertRaises(ValueError, string_to_boolean, value) self.assertRaises(ValueError, string_to_boolean, value)
......
...@@ -33,6 +33,7 @@ import socket ...@@ -33,6 +33,7 @@ import socket
import struct import struct
import subprocess import subprocess
import sqlite3 import sqlite3
from xml_marshaller.xml_marshaller import dumps, loads
def mkdir_p(path, mode=0o700): def mkdir_p(path, mode=0o700):
...@@ -86,16 +87,9 @@ def string_to_boolean(string): ...@@ -86,16 +87,9 @@ def string_to_boolean(string):
The parser is completely arbitrary, see code for actual implementation. The parser is completely arbitrary, see code for actual implementation.
""" """
if not isinstance(string, str) and not isinstance(string, unicode): try:
raise ValueError('Given value is not a string.') return ('false', 'true').index(string.lower())
acceptable_true_values = ['true'] except Exception:
acceptable_false_values = ['false']
string = string.lower()
if string in acceptable_true_values:
return True
if string in acceptable_false_values:
return False
else:
raise ValueError('%s is neither True nor False.' % string) raise ValueError('%s is neither True nor False.' % string)
...@@ -138,3 +132,15 @@ def ipv6FromBin(ip, suffix=''): ...@@ -138,3 +132,15 @@ def ipv6FromBin(ip, suffix=''):
def lenNetmaskIpv6(netmask): def lenNetmaskIpv6(netmask):
return len(binFromIpv6(netmask).rstrip('0')) return len(binFromIpv6(netmask).rstrip('0'))
# Used for Python 2-3 compatibility
if str is bytes:
bytes2str = str2bytes = lambda s: s
def unicode2str(s):
return s.encode('utf-8')
else:
def bytes2str(s):
return s.decode()
def str2bytes(s):
return s.encode()
def unicode2str(s):
return s
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment