cleanup pyc and pyo files when updating git repository

slapos/recipe/build/tests.py

@@ -23,6 +23,7 @@ class GitCloneNonInformativeTests(unittest.TestCase):
 
   def setUp(self):
     self.dir = os.path.realpath(tempfile.mkdtemp())
+    self.parts_directory_path = os.path.join(self.dir, 'test_parts')
 
   def tearDown(self):
     shutil.rmtree(self.dir)
@@ -38,26 +39,26 @@ class GitCloneNonInformativeTests(unittest.TestCase):
     os.chmod(path, mode)
     return path
 
-  def make_recipe(self, buildout, name, options):
+  def makeGitCloneRecipe(self, options):
     from slapos.recipe.gitclone import Recipe
-    parts_directory_path = os.path.join(self.dir, 'test_parts')
-
     bo = {
         'buildout': {
-            'parts-directory': parts_directory_path,
+            'parts-directory': self.parts_directory_path,
             'directory': self.dir,
          }
     }
-    bo.update(buildout)
-    return Recipe(bo, name, options)
+    default_options = {
+      'repository': GIT_REPOSITORY,
+      'forbid-download-cache': 'true',
+      }
+    default_options.update(**options)
+    return Recipe(bo, 'test', default_options)
 
   def test_using_download_cache_if_git_fails(self):
     from slapos.recipe.gitclone import GIT_CLONE_ERROR_MESSAGE, \
         GIT_CLONE_CACHE_ERROR_MESSAGE
-    recipe = self.make_recipe({}, 'test', {
-        'repository': BAD_GIT_REPOSITORY,
-        'forbid-download-cache': 'false',
-    })
+    recipe = self.makeGitCloneRecipe({"forbid-download-cache": "false",
+                                      "repository": BAD_GIT_REPOSITORY})
     os.chdir(self.dir)
     try:
       recipe.install()
@@ -69,10 +70,8 @@ class GitCloneNonInformativeTests(unittest.TestCase):
   def test_not_using_download_cache_if_forbidden(self):
     from slapos.recipe.gitclone import GIT_CLONE_ERROR_MESSAGE, \
         GIT_CLONE_ERROR_MESSAGE
-    recipe = self.make_recipe({}, 'test', {
-        'repository': BAD_GIT_REPOSITORY,
-        'forbid-download-cache': 'true',
-    })
+    recipe = self.makeGitCloneRecipe({"forbid-download-cache": "true",
+                                      "repository": BAD_GIT_REPOSITORY})
     os.chdir(self.dir)
     try:
       recipe.install()
@@ -81,6 +80,20 @@ class GitCloneNonInformativeTests(unittest.TestCase):
     except zc.buildout.UserError, e:
       self.assertEquals(e.message, GIT_CLONE_ERROR_MESSAGE)
 
+  def test_cleanup_of_pyc_files(self):
+    recipe = self.makeGitCloneRecipe({})
+    recipe.install()
+    git_repository_path = os.path.join(self.parts_directory_path, "test")
+    self.assertTrue(os.path.exists(git_repository_path))
+    bad_file_path = os.path.join(git_repository_path, "foo.pyc")
+    bade_file = open(bad_file_path, 'w')
+    bade_file.close()
+    self.assertTrue(os.path.exists(bad_file_path))
+    # install again and make sure pyc file is removed
+    recipe.update()
+    self.assertTrue(os.path.exists(git_repository_path))
+    self.assertFalse(os.path.exists(bad_file_path), "pyc file not removed")
+
 def test_suite():
   suite = unittest.TestSuite((
       doctest.DocFileSuite(

slapos/recipe/gitclone.py

@@ -197,6 +197,17 @@ class Recipe(object):
 
     return [self.location]
 
+  def deletePycFiles(self, path):
+    """Delete *.pyc files so that deleted/moved files can not be imported"""
+    for path, dir_list, file_list in os.walk(path):
+      for file in file_list:
+        if file[-4:] in ('.pyc', '.pyo'):
+          # allow several processes clean the same folder at the same time
+          try:
+            os.remove(os.path.join(path, file))
+          except OSError, e:
+            if e.errno != errno.ENOENT:
+              raise
 
   def update(self):
     """
@@ -204,6 +215,10 @@ class Recipe(object):
     If user doesn't develop, reset to remote revision (or branch if revision is
     not specified).
     """
+    # first cleanup pyc files
+    self.deletePycFiles(self.location)
+
+    # then update
     check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
 
     # If develop parameter is set, don't reset/update.