init: add --submodules to sync manifest submodules

repo sync can sync submodules via the --fetch-submodules option.
However, if the manifest repo has submodules, those will not be synced.
Having submodules in the manifest repo -- while not commonly done -- can
be useful for inheriting a manifest from another project using <include>
and layering changes on top of it.  In this way, you can avoid having to
deal with merge conflicts between your own manifests and the other
project's manifests (for example, if you're managing an Android fork).

Add a --submodule option to init that automatically syncs the submodules
in the manifest repo whenever the manifest repo changes.

Change-Id: I45d34f04517774c1462d7f233f482d1d81a332a8
Signed-off-by: Martin Kelly <mkelly@xevo.com>

manifest_xml.py

@@ -393,6 +393,10 @@ class XmlManifest(object):
   def IsArchive(self):
     return self.manifestProject.config.GetBoolean('repo.archive')
 
+  @property
+  def HasSubmodules(self):
+    return self.manifestProject.config.GetBoolean('repo.submodules')
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}

project.py

@@ -1198,7 +1198,8 @@ class Project(object):
                        no_tags=False,
                        archive=False,
                        optimized_fetch=False,
-                       prune=False):
+                       prune=False,
+                       submodules=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -1275,7 +1276,8 @@ class Project(object):
     if (need_to_fetch and
         not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                               current_branch_only=current_branch_only,
-                              no_tags=no_tags, prune=prune, depth=depth)):
+                              no_tags=no_tags, prune=prune, depth=depth,
+                              submodules=submodules)):
       return False
 
     if self.worktree:
@@ -1331,11 +1333,11 @@ class Project(object):
       raise ManifestInvalidRevisionError('revision %s in %s not found' %
                                          (self.revisionExpr, self.name))
 
-  def Sync_LocalHalf(self, syncbuf, force_sync=False):
+  def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
-    self._InitWorkTree(force_sync=force_sync)
+    self._InitWorkTree(force_sync=force_sync, submodules=submodules)
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
@@ -1344,6 +1346,9 @@ class Project(object):
       self._FastForward(revid)
       self._CopyAndLinkFiles()
 
+    def _dosubmodules():
+      self._SyncSubmodules(quiet=True)
+
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
@@ -1377,6 +1382,8 @@ class Project(object):
 
       try:
         self._Checkout(revid, quiet=True)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1401,6 +1408,8 @@ class Project(object):
                    branch.name)
       try:
         self._Checkout(revid, quiet=True)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1426,6 +1435,8 @@ class Project(object):
         # strict subset.  We can fast-forward safely.
         #
         syncbuf.later1(self, _doff)
+        if submodules:
+          syncbuf.later1(self, _dosubmodules)
         return
 
     # Examine the local commits not in the remote.  Find the
@@ -1477,19 +1488,28 @@ class Project(object):
     branch.Save()
 
     if cnt_mine > 0 and self.rebase:
+      def _docopyandlink():
+        self._CopyAndLinkFiles()
+
       def _dorebase():
         self._Rebase(upstream='%s^1' % last_mine, onto=revid)
-        self._CopyAndLinkFiles()
       syncbuf.later2(self, _dorebase)
+      if submodules:
+        syncbuf.later2(self, _dosubmodules)
+      syncbuf.later2(self, _docopyandlink)
     elif local_changes:
       try:
         self._ResetHard(revid)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
         self._CopyAndLinkFiles()
       except GitError as e:
         syncbuf.fail(self, e)
         return
     else:
       syncbuf.later1(self, _doff)
+      if submodules:
+        syncbuf.later1(self, _dosubmodules)
 
   def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
@@ -1892,7 +1912,8 @@ class Project(object):
                    alt_dir=None,
                    no_tags=False,
                    prune=False,
-                   depth=None):
+                   depth=None,
+                   submodules=False):
 
     is_sha1 = False
     tag_name = None
@@ -2004,6 +2025,9 @@ class Project(object):
     if prune:
       cmd.append('--prune')
 
+    if submodules:
+      cmd.append('--recurse-submodules=on-demand')
+
     spec = []
     if not current_branch_only:
       # Fetch whole repo
@@ -2224,6 +2248,13 @@ class Project(object):
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
+  def _SyncSubmodules(self, quiet=True):
+    cmd = ['submodule', 'update', '--init', '--recursive']
+    if quiet:
+      cmd.append('-q')
+    if GitCommand(self, cmd).Wait() != 0:
+      raise GitError('%s submodule update --init --recursive %s ' % self.name)
+
   def _Rebase(self, upstream, onto=None):
     cmd = ['rebase']
     if onto is not None:
@@ -2464,7 +2495,7 @@ class Project(object):
         else:
           raise
 
-  def _InitWorkTree(self, force_sync=False):
+  def _InitWorkTree(self, force_sync=False, submodules=False):
     dotgit = os.path.join(self.worktree, '.git')
     init_dotgit = not os.path.exists(dotgit)
     try:
@@ -2479,7 +2510,7 @@ class Project(object):
         if force_sync:
           try:
             shutil.rmtree(dotgit)
-            return self._InitWorkTree(force_sync=False)
+            return self._InitWorkTree(force_sync=False, submodules=submodules)
           except:
             raise e
         raise e
@@ -2493,6 +2524,8 @@ class Project(object):
         if GitCommand(self, cmd).Wait() != 0:
           raise GitError("cannot initialize work tree")
 
+        if submodules:
+          self._SyncSubmodules(quiet=True)
         self._CopyAndLinkFiles()
     except Exception:
       if init_dotgit:

repo

@@ -192,6 +192,9 @@ group.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
                       'each project. See git archive.')
+group.add_option('--submodules',
+                 dest='submodules', action='store_true',
+                 help='sync any submodules associated with the manifest repo')
 group.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '

subcmds/init.py

@@ -111,6 +111,9 @@ to update the working directory files.
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
                       'each project. See git archive.')
+    g.add_option('--submodules',
+                 dest='submodules', action='store_true',
+                 help='sync any submodules associated with the manifest repo')
     g.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '
@@ -236,10 +239,13 @@ to update the working directory files.
               'in another location.', file=sys.stderr)
         sys.exit(1)
 
+    if opt.submodules:
+      m.config.SetString('repo.submodules', 'true')
+
     if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet,
         clone_bundle=not opt.no_clone_bundle,
         current_branch_only=opt.current_branch_only,
-        no_tags=opt.no_tags):
+        no_tags=opt.no_tags, submodules=opt.submodules):
       r = m.GetRemote(m.remote.name)
       print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
@@ -253,7 +259,7 @@ to update the working directory files.
       m.MetaBranchSwitch()
 
     syncbuf = SyncBuffer(m.config)
-    m.Sync_LocalHalf(syncbuf)
+    m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
     syncbuf.Finish()
 
     if is_new or m.CurrentBranch is None:

subcmds/sync.py

@@ -723,11 +723,12 @@ later is required to fix a server side protocol bug.
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
                           no_tags=opt.no_tags,
-                          optimized_fetch=opt.optimized_fetch)
+                          optimized_fetch=opt.optimized_fetch,
+                          submodules=self.manifest.HasSubmodules)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
-      mp.Sync_LocalHalf(syncbuf)
+      mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
       if not syncbuf.Finish():
         sys.exit(1)
       self._ReloadManifest(manifest_name)