Commit 40773e15 authored by Cédric de Saint Martin's avatar Cédric de Saint Martin

Merge branch 'slapgrid-log'

parents 8000dab9 f445ab19
...@@ -106,21 +106,17 @@ def dispatch(command, is_node_command): ...@@ -106,21 +106,17 @@ def dispatch(command, is_node_command):
call(register) call(register)
elif command == 'software': elif command == 'software':
call(software, config=GLOBAL_SLAPOS_CONFIGURATION, call(software, config=GLOBAL_SLAPOS_CONFIGURATION,
option=['--pidfile /opt/slapos/slapgrid-sr.pid', option=['--pidfile /opt/slapos/slapgrid-sr.pid'])
'--logfile /opt/slapos/slapgrid-sr.log'])
elif command == 'instance': elif command == 'instance':
call(instance, config=GLOBAL_SLAPOS_CONFIGURATION, call(instance, config=GLOBAL_SLAPOS_CONFIGURATION,
option=['--pidfile /opt/slapos/slapgrid-cp.pid', option=['--pidfile /opt/slapos/slapgrid-cp.pid'])
'--logfile /opt/slapos/slapgrid-cp.log'])
elif command == 'report': elif command == 'report':
call(report, config=GLOBAL_SLAPOS_CONFIGURATION, call(report, config=GLOBAL_SLAPOS_CONFIGURATION,
option=['--pidfile /opt/slapos/slapgrid-ur.pid', option=['--pidfile /opt/slapos/slapgrid-ur.pid'])
'--logfile /opt/slapos/slapgrid-ur.log'])
elif command == 'bang': elif command == 'bang':
call(bang, config=True) call(bang, config=True)
elif command == 'format': elif command == 'format':
call(format, config=GLOBAL_SLAPOS_CONFIGURATION, call(format, config=GLOBAL_SLAPOS_CONFIGURATION])
option=['--log_file /opt/slapos/slapformat.log'])
elif command in ['start', 'stop', 'status', 'tail']: elif command in ['start', 'stop', 'status', 'tail']:
supervisord() supervisord()
supervisorctl() supervisorctl()
......
...@@ -379,7 +379,8 @@ class Partition(object): ...@@ -379,7 +379,8 @@ class Partition(object):
env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw) env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to bootstrap buildout in %r.' % (self.instance_path) message = 'Failed to bootstrap buildout in %r.' % (self.instance_path)
raise BuildoutFailedError(message) self.logger.error(message)
raise BuildoutFailedError('%s:\n%s\n' % (message, process_handler.output))
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout') buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
if not os.path.exists(buildout_binary): if not os.path.exists(buildout_binary):
...@@ -479,7 +480,8 @@ class Partition(object): ...@@ -479,7 +480,8 @@ class Partition(object):
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to destroy Computer Partition in %r.' % \ message = 'Failed to destroy Computer Partition in %r.' % \
self.instance_path self.instance_path
raise subprocess.CalledProcessError(message) self.logger.error(message)
raise subprocess.CalledProcessError(message, process_handler.output)
# Manually cleans what remains # Manually cleans what remains
try: try:
for f in [self.key_file, self.cert_file]: for f in [self.key_file, self.cert_file]:
......
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
import argparse import argparse
import ConfigParser import ConfigParser
from exception import BuildoutFailedError
from hashlib import md5 from hashlib import md5
from lxml import etree from lxml import etree
import logging import logging
...@@ -562,10 +563,25 @@ class Slapgrid(object): ...@@ -562,10 +563,25 @@ class Slapgrid(object):
logger.info('Destroying %r...' % software_release_uri) logger.info('Destroying %r...' % software_release_uri)
software.destroy() software.destroy()
logger.info('Destroyed %r.' % software_release_uri) logger.info('Destroyed %r.' % software_release_uri)
# Send log before exiting
except (SystemExit, KeyboardInterrupt): except (SystemExit, KeyboardInterrupt):
exception = traceback.format_exc() exception = traceback.format_exc()
software_release.error(exception) software_release.error(exception)
raise raise
# Buildout failed: send log but don't print it to output (already done)
except BuildoutFailedError, exception:
clean_run = False
try:
software_release.error(exception)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
exception = traceback.format_exc()
logger.error('Problem during reporting error, continuing:\n' +
exception)
# For everything else: log it, send it, continue.
except Exception: except Exception:
exception = traceback.format_exc() exception = traceback.format_exc()
logger.error(exception) logger.error(exception)
...@@ -800,10 +816,25 @@ class Slapgrid(object): ...@@ -800,10 +816,25 @@ class Slapgrid(object):
# Process the partition itself # Process the partition itself
self.processComputerPartition(computer_partition) self.processComputerPartition(computer_partition)
# Send log before exiting
except (SystemExit, KeyboardInterrupt): except (SystemExit, KeyboardInterrupt):
exception = traceback.format_exc() exception = traceback.format_exc()
computer_partition.error(exception) computer_partition.error(exception)
raise raise
# Buildout failed: send log but don't print it to output (already done)
except BuildoutFailedError, exception:
clean_run = False
try:
computer_partition.error(exception)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
exception = traceback.format_exc()
logger.error('Problem during reporting error, continuing:\n' +
exception)
# For everything else: log it, send it, continue.
except Exception as exception: except Exception as exception:
clean_run = False clean_run = False
logger.error(traceback.format_exc()) logger.error(traceback.format_exc())
......
...@@ -92,6 +92,8 @@ class AlreadyRunning(Exception): ...@@ -92,6 +92,8 @@ class AlreadyRunning(Exception):
class SlapPopen(subprocess.Popen): class SlapPopen(subprocess.Popen):
""" """
Almost normal subprocess with greedish features and logging. Almost normal subprocess with greedish features and logging.
Each line is logged "live", and self.output is a string containing the whole
log.
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
kwargs.update(stdin=subprocess.PIPE) kwargs.update(stdin=subprocess.PIPE)
...@@ -101,15 +103,17 @@ class SlapPopen(subprocess.Popen): ...@@ -101,15 +103,17 @@ class SlapPopen(subprocess.Popen):
self.stdin = None self.stdin = None
logger = logging.getLogger('SlapProcessManager') logger = logging.getLogger('SlapProcessManager')
# XXX-Cedric: this algorithm looks overkill for simple logging.
self.output = ''
while True: while True:
line = self.stdout.readline() line = self.stdout.readline()
if line == '' and self.poll() != None: if line == '' and self.poll() != None:
break break
self.output = self.output + line
if line[-1:] == '\n': if line[-1:] == '\n':
line = line[:-1] line = line[:-1]
logger.info(line) logger.info(line)
def getSoftwareUrlHash(url): def getSoftwareUrlHash(url):
return md5(url).hexdigest() return md5(url).hexdigest()
...@@ -275,10 +279,10 @@ def bootstrapBuildout(path, buildout=None, ...@@ -275,10 +279,10 @@ def bootstrapBuildout(path, buildout=None,
process_handler = SlapPopen(invocation_list, process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid), preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=path, **kw) cwd=path, **kw)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r.\n' % (path) message = 'Failed to run buildout profile in directory %r' % (path)
raise BuildoutFailedError(message) logger.error(message)
raise BuildoutFailedError('%s:\n%s\n' % (message, process_handler.output))
except OSError as error: except OSError as error:
raise BuildoutFailedError(error) raise BuildoutFailedError(error)
finally: finally:
...@@ -318,8 +322,9 @@ def launchBuildout(path, buildout_binary, ...@@ -318,8 +322,9 @@ def launchBuildout(path, buildout_binary,
preexec_fn=lambda: dropPrivileges(uid, gid), cwd=path, preexec_fn=lambda: dropPrivileges(uid, gid), cwd=path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw) env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
if process_handler.returncode is None or process_handler.returncode != 0: if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r\n' % (path) message = 'Failed to run buildout profile in directory %r' % (path)
raise BuildoutFailedError(message) logger.error(message)
raise BuildoutFailedError('%s:\n%s\n' % (message, process_handler.output))
except OSError as error: except OSError as error:
raise BuildoutFailedError(error) raise BuildoutFailedError(error)
finally: finally:
......
...@@ -32,6 +32,7 @@ import os ...@@ -32,6 +32,7 @@ import os
import shutil import shutil
import signal import signal
import slapos.slap.slap import slapos.slap.slap
import slapos.grid.utils
from slapos.grid.watchdog import Watchdog, getWatchdogID from slapos.grid.watchdog import Watchdog, getWatchdogID
import socket import socket
import sys import sys
...@@ -111,13 +112,21 @@ class BasicMixin: ...@@ -111,13 +112,21 @@ class BasicMixin:
self.buildout = None self.buildout = None
self.grid = slapgrid.Slapgrid(self.software_root, self.instance_root, self.grid = slapgrid.Slapgrid(self.software_root, self.instance_root,
self.master_url, self.computer_id, self.supervisord_socket, self.master_url, self.computer_id, self.supervisord_socket,
self.supervisord_configuration_path, self.usage_report_periodicity, self.supervisord_configuration_path,
self.buildout, develop=develop) self.buildout, develop=develop)
# monkey patch buildout bootstrap
def dummy(*args, **kw):
pass
slapos.grid.utils.bootstrapBuildout = dummy
def launchSlapgrid(self,develop=False): def launchSlapgrid(self,develop=False):
self.setSlapgrid(develop=develop) self.setSlapgrid(develop=develop)
return self.grid.processComputerPartitionList() return self.grid.processComputerPartitionList()
def launchSlapgridSoftware(self,develop=False):
self.setSlapgrid(develop=develop)
return self.grid.processSoftwareReleaseList()
def tearDown(self): def tearDown(self):
# XXX: Hardcoded pid, as it is not configurable in slapos # XXX: Hardcoded pid, as it is not configurable in slapos
svc = os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid') svc = os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')
...@@ -246,6 +255,9 @@ class ComputerForTest: ...@@ -246,6 +255,9 @@ class ComputerForTest:
for instance in self.instance_list: for instance in self.instance_list:
slap_computer._computer_partition_list.append( slap_computer._computer_partition_list.append(
instance.getInstance(computer_id)) instance.getInstance(computer_id))
for software in self.software_list:
slap_computer._software_release_list.append(
software.getSoftware(computer_id))
return slap_computer return slap_computer
def setServerResponse(self): def setServerResponse(self):
...@@ -290,8 +302,22 @@ class ComputerForTest: ...@@ -290,8 +302,22 @@ class ComputerForTest:
if 'dropPrivileges' not in line]) if 'dropPrivileges' not in line])
instance.error = True instance.error = True
return (200, {}, '') return (200, {}, '')
else:
return (404, {}, '') elif method == 'POST' and 'url' in parsed_qs:
# XXX hardcoded to first sofwtare release!
software = self.software_list[0]
software.sequence.append(parsed_url.path)
if parsed_url.path == 'buildingSoftwareRelease':
return (200, {}, '')
if parsed_url.path == 'softwareReleaseError':
software.error_log = '\n'.join([line for line \
in parsed_qs['error_log'][0].splitlines()
if 'dropPrivileges' not in line])
software.error = True
return (200, {}, '')
else:
return (500, {}, '')
return server_response return server_response
...@@ -361,19 +387,29 @@ class SoftwareForTest: ...@@ -361,19 +387,29 @@ class SoftwareForTest:
self.software_hash = \ self.software_hash = \
slapos.grid.utils.getSoftwareUrlHash(self.name) slapos.grid.utils.getSoftwareUrlHash(self.name)
self.srdir = os.path.join(self.software_root, self.software_hash) self.srdir = os.path.join(self.software_root, self.software_hash)
self.requested_state = 'available'
os.mkdir(self.srdir) os.mkdir(self.srdir)
self.setTemplateCfg() self.setTemplateCfg()
self.srbindir = os.path.join(self.srdir, 'bin') self.srbindir = os.path.join(self.srdir, 'bin')
os.mkdir(self.srbindir) os.mkdir(self.srbindir)
self.setBuildout() self.setBuildout()
def setTemplateCfg (self,template = """[buildout]"""): def getSoftware (self, computer_id):
"""
Will return current requested state of software
"""
software = slapos.slap.SoftwareRelease(self.name, computer_id)
software._requested_state = self.requested_state
return software
def setTemplateCfg (self,template="""[buildout]"""):
""" """
Set template.cfg Set template.cfg
""" """
open(os.path.join(self.srdir, 'template.cfg'), 'w').write(template) open(os.path.join(self.srdir, 'template.cfg'), 'w').write(template)
def setBuildout (self,buildout = """#!/bin/sh def setBuildout (self, buildout="""#!/bin/sh
touch worked"""): touch worked"""):
""" """
Set a buildout exec in bin Set a buildout exec in bin
...@@ -1038,6 +1074,24 @@ return 42""") ...@@ -1038,6 +1074,24 @@ return 42""")
self.assertEqual(instance1.sequence, self.assertEqual(instance1.sequence,
['availableComputerPartition', 'stoppedComputerPartition']) ['availableComputerPartition', 'stoppedComputerPartition'])
def test_one_partition_buildout_fail_is_correctly_logged(self):
"""
1. We set up an instance using a corrupted buildout
2. It will fail, make sure that whole log is sent to master
"""
computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
instance = computer.instance_list[0]
line1 = "Nerdy kitten: Can I has a process crash?"
line2 = "Cedric: Sure, here it is."
instance.software.setBuildout("""#!/bin/sh
echo %s; echo %s; exit 42""" % (line1, line2))
self.launchSlapgrid()
self.assertEqual(instance.sequence, ['softwareInstanceError'])
# We don't care of actual formatting, we just want to have full log
self.assertTrue(line1 in instance.error_log)
self.assertTrue(line2 in instance.error_log)
self.assertTrue("Failed to run buildout" in instance.error_log)
class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
""" """
...@@ -1179,6 +1233,30 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1179,6 +1233,30 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
[instance.software.software_hash]) [instance.software.software_hash])
self.assertEqual(computer.sequence, ['getFullComputerInformation']) self.assertEqual(computer.sequence, ['getFullComputerInformation'])
class TestSlapgridSoftwareRelease(MasterMixin, unittest.TestCase):
def test_one_software_buildout_fail_is_correctly_logged(self):
"""
1. We set up a software using a corrupted buildout
2. It will fail, make sure that whole log is sent to master
"""
computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
software = computer.software_list[0]
line1 = "Nerdy kitten: Can I has a process crash?"
line2 = "Cedric: Sure, here it is."
software.setBuildout("""#!/bin/sh
echo %s; echo %s; exit 42""" % (line1, line2))
self.launchSlapgridSoftware()
self.assertEqual(software.sequence,
['buildingSoftwareRelease', 'softwareReleaseError'])
# We don't care of actual formatting, we just want to have full log
self.assertTrue(line1 in software.error_log)
self.assertTrue(line2 in software.error_log)
self.assertTrue("Failed to run buildout" in software.error_log)
class SlapgridInitialization(unittest.TestCase): class SlapgridInitialization(unittest.TestCase):
""" """
"Abstract" class setting setup and teardown for TestSlapgridArgumentTuple "Abstract" class setting setup and teardown for TestSlapgridArgumentTuple
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment