Merge branch 'slapgrid-log'

slapos/entry.py

@@ -106,21 +106,17 @@ def dispatch(command, is_node_command):
       call(register)
     elif command == 'software':
       call(software, config=GLOBAL_SLAPOS_CONFIGURATION,
-           option=['--pidfile /opt/slapos/slapgrid-sr.pid',
-                   '--logfile /opt/slapos/slapgrid-sr.log'])
+           option=['--pidfile /opt/slapos/slapgrid-sr.pid'])
     elif command == 'instance':
       call(instance, config=GLOBAL_SLAPOS_CONFIGURATION,
-           option=['--pidfile /opt/slapos/slapgrid-cp.pid',
-                   '--logfile /opt/slapos/slapgrid-cp.log'])
+           option=['--pidfile /opt/slapos/slapgrid-cp.pid'])
     elif command == 'report':
       call(report, config=GLOBAL_SLAPOS_CONFIGURATION,
-           option=['--pidfile /opt/slapos/slapgrid-ur.pid',
-                   '--logfile /opt/slapos/slapgrid-ur.log'])
+           option=['--pidfile /opt/slapos/slapgrid-ur.pid'])
     elif command == 'bang':
       call(bang, config=True)
     elif command == 'format':
-      call(format, config=GLOBAL_SLAPOS_CONFIGURATION,
-           option=['--log_file /opt/slapos/slapformat.log'])
+      call(format, config=GLOBAL_SLAPOS_CONFIGURATION])
     elif command in ['start', 'stop', 'status', 'tail']:
       supervisord()
       supervisorctl()

slapos/grid/SlapObject.py

@@ -379,7 +379,8 @@ class Partition(object):
         env=utils.getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
       if process_handler.returncode is None or process_handler.returncode != 0:
         message = 'Failed to bootstrap buildout in %r.' % (self.instance_path)
-        raise BuildoutFailedError(message)
+        self.logger.error(message)
+        raise BuildoutFailedError('%s:\n%s\n' % (message, process_handler.output))
       buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
 
     if not os.path.exists(buildout_binary):
@@ -479,7 +480,8 @@ class Partition(object):
       if process_handler.returncode is None or process_handler.returncode != 0:
         message = 'Failed to destroy Computer Partition in %r.' % \
             self.instance_path
-        raise subprocess.CalledProcessError(message)
+        self.logger.error(message)
+        raise subprocess.CalledProcessError(message, process_handler.output)
     # Manually cleans what remains
     try:
       for f in [self.key_file, self.cert_file]:

slapos/grid/slapgrid.py

@@ -29,6 +29,7 @@
 
 import argparse
 import ConfigParser
+from exception import BuildoutFailedError
 from hashlib import md5
 from lxml import etree
 import logging
@@ -562,10 +563,25 @@ class Slapgrid(object):
             logger.info('Destroying %r...' % software_release_uri)
             software.destroy()
             logger.info('Destroyed %r.' % software_release_uri)
+      # Send log before exiting
       except (SystemExit, KeyboardInterrupt):
         exception = traceback.format_exc()
         software_release.error(exception)
         raise
+
+      # Buildout failed: send log but don't print it to output (already done)
+      except BuildoutFailedError, exception:
+        clean_run = False
+        try:
+          software_release.error(exception)
+        except (SystemExit, KeyboardInterrupt):
+          raise
+        except Exception:
+          exception = traceback.format_exc()
+          logger.error('Problem during reporting error, continuing:\n' +
+            exception)
+
+      # For everything else: log it, send it, continue.
       except Exception:
         exception = traceback.format_exc()
         logger.error(exception)
@@ -800,10 +816,25 @@ class Slapgrid(object):
         # Process the partition itself
         self.processComputerPartition(computer_partition)
 
+      # Send log before exiting
       except (SystemExit, KeyboardInterrupt):
         exception = traceback.format_exc()
         computer_partition.error(exception)
         raise
+
+      # Buildout failed: send log but don't print it to output (already done)
+      except BuildoutFailedError, exception:
+        clean_run = False
+        try:
+          computer_partition.error(exception)
+        except (SystemExit, KeyboardInterrupt):
+          raise
+        except Exception:
+          exception = traceback.format_exc()
+          logger.error('Problem during reporting error, continuing:\n' +
+            exception)
+
+      # For everything else: log it, send it, continue.
       except Exception as exception:
         clean_run = False
         logger.error(traceback.format_exc())

slapos/grid/utils.py

@@ -92,6 +92,8 @@ class AlreadyRunning(Exception):
 class SlapPopen(subprocess.Popen):
   """
   Almost normal subprocess with greedish features and logging.
+  Each line is logged "live", and self.output is a string containing the whole
+  log.
   """
   def __init__(self, *args, **kwargs):
     kwargs.update(stdin=subprocess.PIPE)
@@ -101,15 +103,17 @@ class SlapPopen(subprocess.Popen):
     self.stdin = None
 
     logger = logging.getLogger('SlapProcessManager')
+    # XXX-Cedric: this algorithm looks overkill for simple logging.
+    self.output = ''
     while True:
       line = self.stdout.readline()
       if line == '' and self.poll() != None:
         break
+      self.output = self.output + line
       if line[-1:] == '\n':
         line = line[:-1]
       logger.info(line)
 
-
 def getSoftwareUrlHash(url):
   return md5(url).hexdigest()
 
@@ -275,10 +279,10 @@ def bootstrapBuildout(path, buildout=None,
     process_handler = SlapPopen(invocation_list,
             preexec_fn=lambda: dropPrivileges(uid, gid),
             cwd=path, **kw)
-
     if process_handler.returncode is None or process_handler.returncode != 0:
-      message = 'Failed to run buildout profile in directory %r.\n' % (path)
-      raise BuildoutFailedError(message)
+      message = 'Failed to run buildout profile in directory %r' % (path)
+      logger.error(message)
+      raise BuildoutFailedError('%s:\n%s\n' % (message, process_handler.output))
   except OSError as error:
     raise BuildoutFailedError(error)
   finally:
@@ -318,8 +322,9 @@ def launchBuildout(path, buildout_binary,
             preexec_fn=lambda: dropPrivileges(uid, gid), cwd=path,
             env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
     if process_handler.returncode is None or process_handler.returncode != 0:
-      message = 'Failed to run buildout profile in directory %r\n' % (path)
-      raise BuildoutFailedError(message)
+      message = 'Failed to run buildout profile in directory %r' % (path)
+      logger.error(message)
+      raise BuildoutFailedError('%s:\n%s\n' % (message, process_handler.output))
   except OSError as error:
     raise BuildoutFailedError(error)
   finally:

slapos/tests/slapgrid.py

@@ -32,6 +32,7 @@ import os
 import shutil
 import signal
 import slapos.slap.slap
+import slapos.grid.utils
 from slapos.grid.watchdog import Watchdog, getWatchdogID
 import socket
 import sys
@@ -111,13 +112,21 @@ class BasicMixin:
     self.buildout = None
     self.grid = slapgrid.Slapgrid(self.software_root, self.instance_root,
       self.master_url, self.computer_id, self.supervisord_socket,
-      self.supervisord_configuration_path, self.usage_report_periodicity,
+      self.supervisord_configuration_path,
       self.buildout, develop=develop)
+    # monkey patch buildout bootstrap
+    def dummy(*args, **kw):
+      pass
+    slapos.grid.utils.bootstrapBuildout = dummy
 
   def launchSlapgrid(self,develop=False):
     self.setSlapgrid(develop=develop)
     return self.grid.processComputerPartitionList()
 
+  def launchSlapgridSoftware(self,develop=False):
+    self.setSlapgrid(develop=develop)
+    return self.grid.processSoftwareReleaseList()
+
   def tearDown(self):
     # XXX: Hardcoded pid, as it is not configurable in slapos
     svc = os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')
@@ -246,6 +255,9 @@ class ComputerForTest:
     for instance in self.instance_list:
       slap_computer._computer_partition_list.append(
         instance.getInstance(computer_id))
+    for software in self.software_list:
+      slap_computer._software_release_list.append(
+        software.getSoftware(computer_id))
     return slap_computer
 
   def setServerResponse(self):
@@ -290,8 +302,22 @@ class ComputerForTest:
                                  if 'dropPrivileges' not in line])
           instance.error = True
           return (200, {}, '')
-        else:
-          return (404, {}, '')
+
+      elif method == 'POST' and 'url' in parsed_qs:
+        # XXX hardcoded to first sofwtare release!
+        software = self.software_list[0]
+        software.sequence.append(parsed_url.path)
+        if parsed_url.path == 'buildingSoftwareRelease':
+          return (200, {}, '')
+        if parsed_url.path == 'softwareReleaseError':
+          software.error_log = '\n'.join([line for line \
+                                   in parsed_qs['error_log'][0].splitlines()
+                                 if 'dropPrivileges' not in line])
+          software.error = True
+          return (200, {}, '')
+
+      else:
+        return (500, {}, '')
     return server_response
 
 
@@ -361,19 +387,29 @@ class SoftwareForTest:
     self.software_hash = \
         slapos.grid.utils.getSoftwareUrlHash(self.name)
     self.srdir = os.path.join(self.software_root, self.software_hash)
+    self.requested_state = 'available'
     os.mkdir(self.srdir)
     self.setTemplateCfg()
     self.srbindir = os.path.join(self.srdir, 'bin')
     os.mkdir(self.srbindir)
     self.setBuildout()
 
-  def setTemplateCfg (self,template = """[buildout]"""):
+  def getSoftware (self, computer_id):
+    """
+    Will return current requested state of software
+    """
+    software = slapos.slap.SoftwareRelease(self.name, computer_id)
+    software._requested_state = self.requested_state
+    return software
+
+
+  def setTemplateCfg (self,template="""[buildout]"""):
     """
     Set template.cfg
     """
     open(os.path.join(self.srdir, 'template.cfg'), 'w').write(template)
 
-  def setBuildout (self,buildout = """#!/bin/sh
+  def setBuildout (self, buildout="""#!/bin/sh
 touch worked"""):
     """
     Set a buildout exec in bin
@@ -1038,6 +1074,24 @@ return 42""")
     self.assertEqual(instance1.sequence,
                      ['availableComputerPartition', 'stoppedComputerPartition'])
 
+  def test_one_partition_buildout_fail_is_correctly_logged(self):
+    """
+    1. We set up an instance using a corrupted buildout
+    2. It will fail, make sure that whole log is sent to master
+    """
+    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
+    instance = computer.instance_list[0]
+
+    line1 = "Nerdy kitten: Can I has a process crash?"
+    line2 = "Cedric: Sure, here it is."
+    instance.software.setBuildout("""#!/bin/sh
+echo %s; echo %s; exit 42""" % (line1, line2))
+    self.launchSlapgrid()
+    self.assertEqual(instance.sequence, ['softwareInstanceError'])
+    # We don't care of actual formatting, we just want to have full log
+    self.assertTrue(line1 in instance.error_log)
+    self.assertTrue(line2 in instance.error_log)
+    self.assertTrue("Failed to run buildout" in instance.error_log)
 
 class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
   """
@@ -1179,6 +1233,30 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
                                [instance.software.software_hash])
     self.assertEqual(computer.sequence, ['getFullComputerInformation'])
 
+
+
+
+class TestSlapgridSoftwareRelease(MasterMixin, unittest.TestCase):
+  def test_one_software_buildout_fail_is_correctly_logged(self):
+    """
+    1. We set up a software using a corrupted buildout
+    2. It will fail, make sure that whole log is sent to master
+    """
+    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
+    software = computer.software_list[0]
+
+    line1 = "Nerdy kitten: Can I has a process crash?"
+    line2 = "Cedric: Sure, here it is."
+    software.setBuildout("""#!/bin/sh
+echo %s; echo %s; exit 42""" % (line1, line2))
+    self.launchSlapgridSoftware()
+    self.assertEqual(software.sequence,
+                     ['buildingSoftwareRelease', 'softwareReleaseError'])
+    # We don't care of actual formatting, we just want to have full log
+    self.assertTrue(line1 in software.error_log)
+    self.assertTrue(line2 in software.error_log)
+    self.assertTrue("Failed to run buildout" in software.error_log)
+
 class SlapgridInitialization(unittest.TestCase):
   """
   "Abstract" class setting setup and teardown for TestSlapgridArgumentTuple