gitclone: add support for submodules, enabled by default

An unrelated change is about develop mode: the behaviour was simplified to do nothing if a working copy already exists.

TODO: handle removal of submodules

/reviewed-on nexedi/slapos.recipe.build!7

README.rst

@@ -121,7 +121,7 @@ try to redownload resource with wrong md5sum.
  slapos.recipe.build:gitclone
 ==============================
 
-Checkout a git repository.
+Checkout a git repository and its submodules by default.
 Supports slapos.libnetworkcache if present, and if boolean 'use-cache' option
 is true.
 
@@ -236,6 +236,7 @@ Then let's run the buildout::
   Running uninstall recipe.
   Installing git-clone.
   Cloning into '/sample-buildout/parts/git-clone'...
+  HEAD is now at 2566127 ...
 
 Let's take a look at the buildout parts directory now::
 
@@ -249,15 +250,12 @@ And let's see that current revision is "2566127"::
   >>> print(subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], universal_newlines=True))
   2566127
 
-When updating, it will do a "git fetch; git reset revision"::
+When updating, it shouldn't do anything as revision is mentioned::
 
   >>> cd(sample_buildout)
   >>> print(system(buildout))
   Updating git-clone.
   ...
-  HEAD is now at 2566127 ... 
-  ...
-  <BLANKLINE>
 
 Empty revision/branch
 ~~~~~~~~~~~~~~~~~~~~~
@@ -327,7 +325,7 @@ and branch parameter is ignored::
 Setup a "develop" repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If you need to setup a repository that will be manually alterated over time for
+If you need to setup a repository that will be manually altered over time for
 development purposes, you need to make sure buildout will NOT alter it and NOT
 erase your local modifications by specifying the "develop" flag::
 
@@ -366,7 +364,6 @@ repository::
   >>> cd(sample_buildout)
   >>> print(system(buildout))
   Updating git-clone.
-  Fetching origin
   ...
   <BLANKLINE>
 
@@ -386,17 +383,7 @@ Then, when update occurs, nothing is done::
   >>> cd(sample_buildout)
   >>> print(system(buildout))
   Updating git-clone.
-  Fetching origin
-  Fetching broken
-  Unable to update:
-  Traceback (most recent call last):
-  ...
-  ...CalledProcessError: Command '['git', 'fetch', '--all']' returned non-zero exit status 1
-  <BLANKLINE>
   ...
-  fatal: unable to access 'http://git.erp5.org/repos/nowhere/': The requested URL returned error: 500
-  error: Could not fetch broken
-  <BLANKLINE> 
 
   >>> cd(sample_buildout, 'parts', 'git-clone')
   >>> print(system('cat local_change'))
@@ -450,6 +437,21 @@ boolean option::
   repository = https://example.net/example.git/
   ignore-ssl-certificate = true
 
+Ignore cloning submodules
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, cloning the repository will clone its submodules also. You can force
+git to ignore cloning submodules by defining `ignore-cloning-submodules` boolean
+option to 'true'::
+
+  [buildout]
+  parts = git-clone
+
+  [git-clone]
+  recipe = slapos.recipe.build:gitclone
+  repository = https://lab.nexedi.com/tiwariayush/test_erp5
+  ignore-cloning-submodules = true
+
 Other options
 ~~~~~~~~~~~~~
 

slapos/recipe/gitclone.py

@@ -34,7 +34,7 @@ import time
 import traceback
 
 from zc.buildout import UserError
-from subprocess import call, check_call, CalledProcessError
+from subprocess import check_call, CalledProcessError
 import subprocess
 
 try:
@@ -145,7 +145,8 @@ class Recipe(object):
       self.git_command = 'git'
     self.sparse = options.get('sparse-checkout', '').strip()
     # Set boolean values
-    for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate'):
+    for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate',
+                'ignore-cloning-submodules'):
       setattr(self, key.replace('-', '_'), options.get(key, '').lower() in TRUE_VALUES)
     if self.shared:
       self.use_cache = False
@@ -171,7 +172,6 @@ class Recipe(object):
       command.append(revision)
     check_call(command, cwd=self.location)
 
-
   def install(self):
     """
     Do a git clone.
@@ -213,6 +213,13 @@ class Recipe(object):
 
     if config and self.use_cache:
       raise NotImplementedError
+
+    if not self.ignore_cloning_submodules:
+      # `--recurse-submodules` to the git clone command will automatically
+      # initialize and update each submodule in the repository.
+      config.append('submodule.recurse=true')
+      git_clone_command.append('--recurse-submodules')
+
     for config in config:
       git_clone_command += '--config', config
 
@@ -230,6 +237,11 @@ class Recipe(object):
       if self.use_cache:
         upload_network_cached(os.path.join(self.location, '.git'),
                               self.repository, self.revision, self.networkcache)
+      if not self.ignore_cloning_submodules:
+        # Update submodule repository to the commit which is being pointed to
+        # in main repo
+        check_call([self.git_command, 'submodule', 'update', '--init',
+                    '--recursive'], cwd=self.location)
     except CalledProcessError:
       print("Unable to download from git repository."
             " Trying from network cache...")
@@ -260,32 +272,35 @@ class Recipe(object):
   def update(self):
     """
     Do a git fetch.
-    If user doesn't develop, reset to remote revision (or branch if revision is
-    not specified).
+    If user doesn't develop, reset to upstream of the branch.
+    With support for submodules, we also update the submodule repository if
+    there has been update to the pointer of the submodules in the parent repo.
+    This however puts the updated submodules in a DETACHED state.
     """
+    # If develop or revision parameter, no need to update
+    if self.develop or self.revision:
+      return
     try:
       # first cleanup pyc files
       self.deletePycFiles(self.location)
 
-      # then update,
-      # but, to save time, only if we don't have the revision already
-      revision_already_fetched = \
-            self.revision and   \
-            call([self.git_command, 'rev-parse', '--verify', self.revision],
-                    cwd=self.location) == 0
-      if not revision_already_fetched:
-        check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
+      # Fetch and reset to the upstream
+      check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
+      self.gitReset('@{upstream}')
+
+      if not self.ignore_cloning_submodules:
+        # Update the submodule to the commit which parent repo points to if
+        # there has been revision is present for the parent repo.
+        # According to man-page of submodule, update also clones missing
+        # submodules and updating the working tree of the submodules. This is
+        # why we do update here only after we are ok with revision of parent
+        # repo being checked out to the desired one.
+        # It will also init a submodule if required
+        # NOTE: This will put the submodule repo in a `Detached` state.
+        check_call([self.git_command, 'submodule', 'update', '--init', '-f',
+                    '--recursive'], cwd=self.location)
 
-      # If develop parameter is set, don't reset/update.
-      # Otherwise, reset --hard
-      if not self.develop:
-        if self.revision:
-          self.gitReset(self.revision)
-        else:
-          self.gitReset('@{upstream}')
     except:
-      if not self.develop:
-        raise
       # Buildout will remove the installed location and mark the part as not
       # installed if an error occurs during update. If we are developping this
       # repository we do not want this to happen.