slapgrid: Refactor iterating over dir entries

slapos/grid/promise/__init__.py

@@ -44,7 +44,7 @@ import hashlib
 from datetime import datetime
 from multiprocessing import Process, Queue as MQueue
 from six.moves import queue, reload_module
-from slapos.util import str2bytes, mkdir_p, chownDirectory
+from slapos.util import str2bytes, mkdir_p, chownDirectory, listifdir
 from slapos.grid.utils import dropPrivileges, killProcessTree
 from slapos.grid.promise import interface
 from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
@@ -731,65 +731,63 @@ class PromiseLauncher(object):
     error = 0
     success = 0
     promise_name_list = []
-    if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder):
-      for promise_name in os.listdir(self.promise_folder):
-        for suffix in ['.pyc', '.pyo']:
-          if promise_name.endswith(suffix):
-            promise_path = os.path.join(self.promise_folder, promise_name)
-            if not os.path.exists(promise_path[:-1]):
-              try:
-                os.unlink(promise_path)
-              except Exception as e:
-                self.logger.warning('Failed to remove %r because of %s', promise_path, e)
-              else:
-                self.logger.debug('Removed stale %r', promise_path)
-
-        if promise_name.startswith('__init__') or \
-            not promise_name.endswith('.py'):
-          continue
-
-        promise_name_list.append(promise_name)
-        if self.run_only_promise_list is not None and not \
-            promise_name in self.run_only_promise_list:
-          continue
-
+    for promise_name in listifdir(self.promise_folder):
+      if promise_name.endswith(('.pyc', '.pyo')):
         promise_path = os.path.join(self.promise_folder, promise_name)
-        config = {
-          'path': promise_path,
-          'name': promise_name
-        }
-        config.update(base_config)
-        promise_result = self._launchPromise(promise_name, promise_path, config)
-        if promise_result:
-          change_date = promise_result.date.strftime('%Y-%m-%dT%H:%M:%S+0000')
-          if promise_result.hasFailed():
-            promise_status = 'FAILED'
+        if not os.path.exists(promise_path[:-1]):
+          try:
+            os.unlink(promise_path)
+          except Exception as e:
+            self.logger.warning('Failed to remove %r because of %s', promise_path, e)
+          else:
+            self.logger.debug('Removed stale %r', promise_path)
+
+      if promise_name.startswith('__init__') or \
+          not promise_name.endswith('.py'):
+        continue
+
+      promise_name_list.append(promise_name)
+      if self.run_only_promise_list is not None and not \
+          promise_name in self.run_only_promise_list:
+        continue
+
+      promise_path = os.path.join(self.promise_folder, promise_name)
+      config = {
+        'path': promise_path,
+        'name': promise_name
+      }
+      config.update(base_config)
+      promise_result = self._launchPromise(promise_name, promise_path, config)
+      if promise_result:
+        change_date = promise_result.date.strftime('%Y-%m-%dT%H:%M:%S+0000')
+        if promise_result.hasFailed():
+          promise_status = 'FAILED'
+          error += 1
+        else:
+          promise_status = "OK"
+          success += 1
+        if promise_name in previous_state_dict:
+          status, previous_change_date, _ = previous_state_dict[promise_name]
+          if promise_status == status:
+            change_date = previous_change_date
+
+        message = promise_result.message if promise_result.message else ""
+        new_state_dict[promise_name] = [
+          promise_status,
+          change_date,
+          hashlib.md5(str2bytes(message)).hexdigest()]
+
+        if promise_result.hasFailed() and not failed_promise_name:
+          failed_promise_name = promise_name
+          failed_promise_output = promise_result.message
+      else:
+        # The promise was skip, so for statistic point of view we preserve
+        # the previous result
+        if promise_name in new_state_dict:
+          if new_state_dict[promise_name][0] == "FAILED":
             error += 1
           else:
-            promise_status = "OK"
             success += 1
-          if promise_name in previous_state_dict:
-            status, previous_change_date, _ = previous_state_dict[promise_name]
-            if promise_status == status:
-              change_date = previous_change_date
-
-          message = promise_result.message if promise_result.message else ""
-          new_state_dict[promise_name] = [
-            promise_status,
-            change_date,
-            hashlib.md5(str2bytes(message)).hexdigest()]
-      
-          if promise_result.hasFailed() and not failed_promise_name:
-            failed_promise_name = promise_name
-            failed_promise_output = promise_result.message
-        else:
-          # The promise was skip, so for statistic point of view we preserve
-          # the previous result
-          if promise_name in new_state_dict:
-            if new_state_dict[promise_name][0] == "FAILED":
-              error += 1
-            else:
-              success += 1
 
     if not self.run_only_promise_list and len(promise_name_list) > 0:
       # cleanup stale json files
@@ -816,10 +814,9 @@ class PromiseLauncher(object):
         if key not in promise_name_list:
           new_state_dict.pop(key, None)
 
-    if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \
-        and os.path.isdir(self.legacy_promise_folder):
+    if not self.run_only_promise_list:
       # run legacy promise styles
-      for promise_name in os.listdir(self.legacy_promise_folder):
+      for promise_name in listifdir(self.legacy_promise_folder):
         promise_path = os.path.join(self.legacy_promise_folder, promise_name)
         if not os.path.isfile(promise_path) or \
             not os.access(promise_path, os.X_OK):

slapos/grid/slapgrid.py

@@ -55,7 +55,7 @@ from slapos import manager as slapmanager
 from slapos.slap.slap import NotFoundError
 from slapos.slap.slap import ServerError
 from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
-from slapos.util import mkdir_p, chownDirectory, string_to_boolean
+from slapos.util import mkdir_p, chownDirectory, string_to_boolean, listifdir
 from slapos.grid.exception import BuildoutFailedError
 from slapos.grid.SlapObject import Software, Partition
 from slapos.grid.svcbackend import (launchSupervisord,
@@ -1596,10 +1596,7 @@ stderr_logfile_backups=1
         instance_path = os.path.join(self.instance_root,
             computer_partition.getId())
         report_path = os.path.join(instance_path, 'etc', 'report')
-        if os.path.isdir(report_path):
-          script_list_to_run = os.listdir(report_path)
-        else:
-          script_list_to_run = []
+        script_list_to_run = listifdir(report_path)
 
         # We now generate the pseudorandom name for the xml file
         # and we add it in the invocation_list

slapos/util.py

@@ -95,6 +95,18 @@ def mkdir_p(path, mode=0o700):
             raise
 
 
+def listifdir(path):
+  """
+  Like listdir, but returns an empty tuple if the path is not a directory.
+  """
+  try:
+    return os.listdir(path)
+  except OSError as e:
+    if e.errno != errno.ENOENT:
+      raise
+    return ()
+
+
 def chownDirectory(path, uid, gid):
   if os.getuid() != 0:
     # we are probably inside of a webrunner