Introduce manifest format using git submodules

If a manifest top level directory contains '.gitmodules' we now
assume this is a git module format manifest and switch to using
that code, rather than the legacy XML based manifest.

At the same time, we move the bare repository for a project from
$TOP/.repo/projects/$REPO_PATH.git to be $REPO_NAME.git instead.
This makes it easier for us to later support a repo init from an
existing work tree, as we can more accurately predict the path of
the project's repository in the workspace.  It also means that the
$TOP/.repo/projects/ directory is layed out like a mirror would be.

Signed-off-by: Shawn O. Pearce <sop@google.com>

docs/manifest_submodule.txt

@@ -0,0 +1,130 @@
+repo Manifest Format (submodule)
+================================
+
+A repo manifest describes the structure of a repo client; that is
+the directories that are visible and where they should be obtained
+from with git.
+
+The basic structure of a manifest is a bare Git repository holding
+a 'gitmodules' file in the top level directory, and one or more
+gitlink references pointing at commits from the referenced projects.
+This is the same structure as used by 'git submodule'.
+
+Manifests are inherently version controlled, since they are kept
+within a Git repository.  Updates to manifests are automatically
+obtained by clients during `repo sync`.
+
+.gitmodules
+===========
+
+The '.gitmodules' file, located in the top-level directory of the
+client's working tree (or manifest repository), is a text file with
+a syntax matching the requirements of 'git config'.
+
+This file contains one subsection per project (also called a
+submodule by git), and the subsection value is a unique name to
+describe the project. Each submodule section must contain the
+following required keys:
+
+  * path
+  * url
+
+submodule.<name>.path
+---------------------
+
+Defines the path, relative to the top-level directory of the client's
+working tree, where the project is expected to be checked out. The
+path name must not end with a '/'. All paths must be unique within
+the .gitmodules file.
+
+At the specified path within the manifest repository a gitlink
+tree entry (an entry with file mode 160000) must exist referencing
+a commit SHA-1 from the project.  This tree entry specifies the
+exact version of the project that `repo sync` will synchronize the
+client's working tree to.
+
+submodule.<name>.url
+--------------------
+
+Defines a URL from where the project repository can be cloned.
+By default `repo sync` will clone from this URL whenever a user
+needs to access this project.
+
+submodule.<name>.revision
+-------------------------
+
+Name of the branch in the project repository that Gerrit Code Review
+should automatically refresh the project's gitlink entry from.
+
+If set, during submit of a change within the referenced project,
+Gerrit Code Review will automatically update the manifest
+repository's corresponding gitlink to the new commit SHA-1 of
+this branch.
+
+Valid values are a short branch name (e.g. 'master'), a full ref
+name (e.g. 'refs/heads/master'), or '.' to request using the same
+branch name as the manifest branch itself.  Since '.' automatically
+uses the manifest branch, '.' is the recommended value.
+
+If this key is not set, Gerrit Code Review will NOT automatically
+update the gitlink. An unset key requires the manifest maintainer
+to manually update the gitlink when it is necessary to reference
+a different revision of the project.
+
+submodule.<name>.update
+-----------------------
+
+This key is not supported by repo.  If set, it will be ignored.
+
+.review
+=======
+
+The optional '.review' file, located in the top-level directory of
+the client's working tree (or manifest repository), is a text file
+with a syntax matching the requirements of 'git config'.
+
+This file describes how `repo upload` should interact with the
+project's preferred code review system.
+
+review.url
+----------
+
+URL of the default Gerrit Code Review server.  If a project does
+not have a specific URL in the '.review' file, this default URL
+will be used instead.
+
+review.<name>.url
+-----------------
+
+Project specific URL of the Gerrit Code Review server, for the
+submodule whose project name is <name>.
+
+Example
+=======
+
+  $ cat .gitmodules
+  [submodule "app/Clock"]
+    path = clock
+    url = git://vcs.example.com/ClockWidget.git
+    revision = .
+  [submodule "app/Browser"]
+    path = net/browser
+    url = git://netgroup.example.com/network/web/Browser.git
+    revision = .
+
+  $ cat .review
+  [review]
+    url = vcs-gerrit.example.com
+  [review "app/Browser"]
+    url = netgroup.example.com
+
+In the above example, the app/Clock project will send its code
+reviews to the default server, vcs-gerrit.example.com, while
+app/Browser will send its code reviews to netgroup.example.com.
+
+See Also
+========
+
+ * http://www.kernel.org/pub/software/scm/git/docs/gitmodules.html
+ * http://www.kernel.org/pub/software/scm/git/docs/git-config.html
+ * http://code.google.com/p/gerrit/

manifest_loader.py

@@ -13,11 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from manifest_submodule import SubmoduleManifest
 from manifest_xml import XmlManifest
 
 def ParseManifest(repodir, type=None):
   if type:
     return type(repodir)
+  if SubmoduleManifest.Is(repodir):
+    return SubmoduleManifest(repodir)
   return XmlManifest(repodir)
 
 _manifest = None

manifest_submodule.py

@@ -0,0 +1,474 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+import shutil
+
+from error import GitError
+from error import ManifestParseError
+from git_command import GitCommand
+from git_config import GitConfig
+from git_config import IsId
+from manifest import Manifest
+from progress import Progress
+from project import RemoteSpec
+from project import Project
+from project import MetaProject
+from project import R_HEADS
+from project import HEAD
+from project import _lwrite
+
+import manifest_xml
+
+GITLINK = '160000'
+
+def _rmdir(dir, top):
+  while dir != top:
+    try:
+      os.rmdir(dir)
+    except OSError:
+      break
+    dir = os.path.dirname(dir)
+
+def _rmref(gitdir, ref):
+  os.remove(os.path.join(gitdir, ref))
+  log = os.path.join(gitdir, 'logs', ref)
+  if os.path.exists(log):
+    os.remove(log)
+    _rmdir(os.path.dirname(log), gitdir)
+
+def _has_gitmodules(d):
+  return os.path.exists(os.path.join(d, '.gitmodules'))
+
+class SubmoduleManifest(Manifest):
+  """manifest from .gitmodules file"""
+
+  @classmethod
+  def Is(cls, repodir):
+    return _has_gitmodules(os.path.dirname(repodir)) \
+        or _has_gitmodules(os.path.join(repodir, 'manifest')) \
+        or _has_gitmodules(os.path.join(repodir, 'manifests'))
+
+  @classmethod
+  def IsBare(cls, p):
+    try:
+      p.bare_git.cat_file('-e', '%s:.gitmodules' % p.GetRevisionId())
+    except GitError:
+      return False
+    return True
+
+  def __init__(self, repodir):
+    Manifest.__init__(self, repodir)
+
+    gitdir = os.path.join(repodir, 'manifest.git')
+    config = GitConfig.ForRepository(gitdir = gitdir)
+
+    if config.GetBoolean('repo.mirror'):
+      worktree = os.path.join(repodir, 'manifest')
+      relpath = None
+    else:
+      worktree = self.topdir
+      relpath  = '.'
+
+    self.manifestProject = MetaProject(self, '__manifest__',
+      gitdir   = gitdir,
+      worktree = worktree,
+      relpath  = relpath)
+    self._modules = GitConfig(os.path.join(worktree, '.gitmodules'),
+                              pickleFile = os.path.join(
+                                repodir, '.repopickle_gitmodules'
+                              ))
+    self._review = GitConfig(os.path.join(worktree, '.review'),
+                             pickleFile = os.path.join(
+                               repodir, '.repopickle_review'
+                             ))
+    self._Unload()
+
+  @property
+  def projects(self):
+    self._Load()
+    return self._projects
+
+  def InitBranch(self):
+    m = self.manifestProject
+    if m.CurrentBranch is None:
+      b = m.revisionExpr
+      if b.startswith(R_HEADS):
+        b = b[len(R_HEADS):]
+      return m.StartBranch(b)
+    return True
+
+  def SetMRefs(self, project):
+    if project.revisionId is None:
+      # Special project, e.g. the manifest or repo executable.
+      #
+      return
+
+    ref = 'refs/remotes/m'
+    cur = project.bare_ref.get(ref)
+    exp = project.revisionId
+    if cur != exp:
+      msg = 'manifest set to %s' % exp
+      project.bare_git.UpdateRef(ref, exp, message = msg, detach = True)
+
+    ref = 'refs/remotes/m-revision'
+    cur = project.bare_ref.symref(ref)
+    exp = project.revisionExpr
+    if exp is None:
+      if cur:
+        _rmref(project.gitdir, ref)
+    elif cur != exp:
+      remote = project.GetRemote(project.remote.name)
+      dst = remote.ToLocal(exp)
+      msg = 'manifest set to %s (%s)' % (exp, dst)
+      project.bare_git.symbolic_ref('-m', msg, ref, dst)
+
+  def Upgrade_Local(self, old):
+    if isinstance(old, manifest_xml.XmlManifest):
+      self.FromXml_Local_1(old, checkout=True)
+      self.FromXml_Local_2(old)
+    else:
+      raise ManifestParseError, 'cannot upgrade manifest'
+
+  def FromXml_Local_1(self, old, checkout):
+    os.rename(old.manifestProject.gitdir,
+              os.path.join(old.repodir, 'manifest.git'))
+
+    oldmp = old.manifestProject
+    oldBranch = oldmp.CurrentBranch
+    b = oldmp.GetBranch(oldBranch).merge
+    if not b:
+      raise ManifestParseError, 'cannot upgrade manifest'
+    if b.startswith(R_HEADS):
+      b = b[len(R_HEADS):]
+
+    newmp = self.manifestProject
+    self._CleanOldMRefs(newmp)
+    if oldBranch != b:
+      newmp.bare_git.branch('-m', oldBranch, b)
+      newmp.config.ClearCache()
+
+    old_remote = newmp.GetBranch(b).remote.name
+    act_remote = self._GuessRemoteName(old)
+    if old_remote != act_remote:
+      newmp.bare_git.remote('rename', old_remote, act_remote)
+      newmp.config.ClearCache()
+    newmp.remote.name = act_remote
+    print >>sys.stderr, "Assuming remote named '%s'" % act_remote
+
+    if checkout:
+      for p in old.projects.values():
+        for c in p.copyfiles:
+          if os.path.exists(c.abs_dest):
+            os.remove(c.abs_dest)
+      newmp._InitWorkTree()
+    else:
+      newmp._LinkWorkTree()
+
+    _lwrite(os.path.join(newmp.worktree,'.git',HEAD),
+            'ref: refs/heads/%s\n' % b)
+
+  def _GuessRemoteName(self, old):
+    used = {}
+    for p in old.projects.values():
+      n = p.remote.name
+      used[n] = used.get(n, 0) + 1
+
+    remote_name = 'origin'
+    remote_used = 0
+    for n in used.keys():
+      if remote_used < used[n]:
+        remote_used = used[n]
+        remote_name = n
+    return remote_name
+
+  def FromXml_Local_2(self, old):
+    shutil.rmtree(old.manifestProject.worktree)
+    os.remove(old._manifestFile)
+
+    my_remote = self._Remote().name
+    new_base = os.path.join(self.repodir, 'projects')
+    old_base = os.path.join(self.repodir, 'projects.old')
+    os.rename(new_base, old_base)
+    os.makedirs(new_base)
+
+    info = []
+    pm = Progress('Converting projects', len(self.projects))
+    for p in self.projects.values():
+      pm.update()
+
+      old_p = old.projects.get(p.name)
+      old_gitdir = os.path.join(old_base, '%s.git' % p.relpath)
+      if not os.path.isdir(old_gitdir):
+        continue
+
+      parent = os.path.dirname(p.gitdir)
+      if not os.path.isdir(parent):
+        os.makedirs(parent)
+      os.rename(old_gitdir, p.gitdir)
+      _rmdir(os.path.dirname(old_gitdir), self.repodir)
+
+      if not os.path.isdir(p.worktree):
+        os.makedirs(p.worktree)
+
+      if os.path.isdir(os.path.join(p.worktree, '.git')):
+        p._LinkWorkTree(relink=True)
+
+      self._CleanOldMRefs(p)
+      if old_p and old_p.remote.name != my_remote:
+        info.append("%s/: renamed remote '%s' to '%s'" \
+                    % (p.relpath, old_p.remote.name, my_remote))
+        p.bare_git.remote('rename', old_p.remote.name, my_remote)
+        p.config.ClearCache()
+
+      self.SetMRefs(p)
+    pm.end()
+    for i in info:
+      print >>sys.stderr, i
+
+  def _CleanOldMRefs(self, p):
+    all_refs = p._allrefs
+    for ref in all_refs.keys():
+      if ref.startswith(manifest_xml.R_M):
+        if p.bare_ref.symref(ref) != '':
+          _rmref(p.gitdir, ref)
+        else:
+          p.bare_git.DeleteRef(ref, all_refs[ref])
+
+  def FromXml_Definition(self, old):
+    """Convert another manifest representation to this one.
+    """
+    mp = self.manifestProject
+    gm = self._modules
+    gr = self._review
+
+    fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab')
+    fd.write('/.repo\n')
+    fd.close()
+
+    sort_projects = list(old.projects.keys())
+    sort_projects.sort()
+
+    b = mp.GetBranch(mp.CurrentBranch).merge
+    if b.startswith(R_HEADS):
+      b = b[len(R_HEADS):]
+
+    info = []
+    pm = Progress('Converting manifest', len(sort_projects))
+    for p in sort_projects:
+      pm.update()
+      p = old.projects[p]
+
+      gm.SetString('submodule.%s.path' % p.name, p.relpath)
+      gm.SetString('submodule.%s.url' % p.name, p.remote.url)
+
+      if gr.GetString('review.url') is None:
+        gr.SetString('review.url', p.remote.review)
+      elif gr.GetString('review.url') != p.remote.review:
+        gr.SetString('review.%s.url' % p.name, p.remote.review)
+
+      r = p.revisionExpr
+      if r and not IsId(r):
+        if r.startswith(R_HEADS):
+          r = r[len(R_HEADS):]
+        if r == b:
+          r = '.'
+        gm.SetString('submodule.%s.revision' % p.name, r)
+
+      for c in p.copyfiles:
+        info.append('Moved %s out of %s' % (c.src, p.relpath))
+        c._Copy()
+        p.work_git.rm(c.src)
+        mp.work_git.add(c.dest)
+
+      self.SetRevisionId(p.relpath, p.GetRevisionId())
+    mp.work_git.add('.gitignore', '.gitmodules', '.review')
+    pm.end()
+    for i in info:
+      print >>sys.stderr, i
+
+  def _Unload(self):
+    self._loaded = False
+    self._projects = {}
+    self._revisionIds = None
+    self.branch = None
+
+  def _Load(self):
+    if not self._loaded:
+      f = os.path.join(self.repodir, manifest_xml.LOCAL_MANIFEST_NAME)
+      if os.path.exists(f):
+        print >>sys.stderr, 'warning: ignoring %s' % f
+
+      m = self.manifestProject
+      b = m.CurrentBranch
+      if not b:
+        raise ManifestParseError, 'manifest cannot be on detached HEAD'
+      b = m.GetBranch(b).merge
+      if b.startswith(R_HEADS):
+        b = b[len(R_HEADS):]
+      self.branch = b
+      m.remote.name = self._Remote().name
+
+      self._ParseModules()
+
+      if self.IsMirror:
+        self._AddMetaProjectMirror(self.repoProject)
+        self._AddMetaProjectMirror(self.manifestProject)
+
+      self._loaded = True
+
+  def _ParseModules(self):
+    byPath = dict()
+    for name in self._modules.GetSubSections('submodule'):
+      p = self._ParseProject(name)
+      if self._projects.get(p.name):
+        raise ManifestParseError, 'duplicate project "%s"' % p.name
+      if byPath.get(p.relpath):
+        raise ManifestParseError, 'duplicate path "%s"' % p.relpath
+      self._projects[p.name] = p
+      byPath[p.relpath] = p
+
+    for relpath in self._allRevisionIds.keys():
+      if relpath not in byPath:
+        raise ManifestParseError, \
+          'project "%s" not in .gitmodules' \
+          % relpath
+
+  def _Remote(self):
+    m = self.manifestProject
+    b = m.GetBranch(m.CurrentBranch)
+    return b.remote
+
+  def _ResolveUrl(self, url):
+    if url.startswith('./') or url.startswith('../'):
+      base = self._Remote().url
+      try:
+        base = base[:base.rindex('/')+1]
+      except ValueError:
+        base = base[:base.rindex(':')+1]
+      if url.startswith('./'):
+        url = url[2:]
+      while '/' in base and url.startswith('../'):
+        base = base[:base.rindex('/')+1]
+        url = url[3:]
+      return base + url
+    return url
+
+  def _GetRevisionId(self, path):
+    return self._allRevisionIds.get(path)
+
+  @property
+  def _allRevisionIds(self):
+    if self._revisionIds is None:
+      a = dict()
+      p = GitCommand(self.manifestProject,
+                     ['ls-files','-z','--stage'],
+                     capture_stdout = True)
+      for line in p.process.stdout.read().split('\0')[:-1]:
+        l_info, l_path = line.split('\t', 2)
+        l_mode, l_id, l_stage = l_info.split(' ', 2)
+        if l_mode == GITLINK and l_stage == '0':
+          a[l_path] = l_id
+      p.Wait()
+      self._revisionIds = a
+    return self._revisionIds
+
+  def SetRevisionId(self, path, id):
+    self.manifestProject.work_git.update_index(
+      '--add','--cacheinfo', GITLINK, id, path)
+
+  def _ParseProject(self, name):
+    gm = self._modules
+    gr = self._review
+
+    path = gm.GetString('submodule.%s.path' % name)
+    if not path:
+      path = name
+
+    revId = self._GetRevisionId(path)
+    if not revId:
+      raise ManifestParseError(
+        'submodule "%s" has no revision at "%s"' \
+        % (name, path))
+
+    url = gm.GetString('submodule.%s.url' % name)
+    if not url:
+      url = name
+    url = self._ResolveUrl(url)
+
+    review = gr.GetString('review.%s.url' % name)
+    if not review:
+      review = gr.GetString('review.url')
+    if not review:
+      review = self._Remote().review
+
+    remote = RemoteSpec(self._Remote().name, url, review)
+    revExpr = gm.GetString('submodule.%s.revision' % name)
+    if revExpr == '.':
+      revExpr = self.branch
+
+    if self.IsMirror:
+      relpath = None
+      worktree = None
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+    else:
+      worktree = os.path.join(self.topdir, path)
+      gitdir = os.path.join(self.repodir, 'projects/%s.git' % name)
+
+    return Project(manifest = self,
+                   name = name,
+                   remote = remote,
+                   gitdir = gitdir,
+                   worktree = worktree,
+                   relpath = path,
+                   revisionExpr = revExpr,
+                   revisionId = revId)
+
+  def _AddMetaProjectMirror(self, m):
+    m_url = m.GetRemote(m.remote.name).url
+    if m_url.endswith('/.git'):
+      raise ManifestParseError, 'refusing to mirror %s' % m_url
+
+    name = self._GuessMetaName(m_url)
+    if name.endswith('.git'):
+      name = name[:-4]
+
+    if name not in self._projects:
+      m.PreSync()
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+      project = Project(manifest = self,
+                        name = name,
+                        remote = RemoteSpec(self._Remote().name, m_url),
+                        gitdir = gitdir,
+                        worktree = None,
+                        relpath = None,
+                        revisionExpr = m.revisionExpr,
+                        revisionId = None)
+      self._projects[project.name] = project
+
+  def _GuessMetaName(self, m_url):
+    parts = m_url.split('/')
+    name = parts[-1]
+    parts = parts[0:-1]
+    s = len(parts) - 1
+    while s > 0:
+      l = '/'.join(parts[0:s]) + '/'
+      r = '/'.join(parts[s:]) + '/'
+      for p in self._projects.values():
+        if p.name.startswith(r) and p.remote.url.startswith(l):
+          return r + name
+      s -= 1
+    return m_url[m_url.rindex('/') + 1:]

subcmds/init.py

@@ -21,6 +21,7 @@ from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from project import SyncBuffer
 from git_command import git_require, MIN_GIT_VERSION
+from manifest_submodule import SubmoduleManifest
 from manifest_xml import XmlManifest
 from subcmds.sync import _ReloadManifest
 
@@ -144,6 +145,14 @@ to update the working directory files.
       print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
       sys.exit(1)
 
+    if is_new and SubmoduleManifest.IsBare(m):
+      new = self.GetManifest(reparse=True, type=SubmoduleManifest)
+      if m.gitdir != new.manifestProject.gitdir:
+        os.rename(m.gitdir, new.manifestProject.gitdir)
+        new = self.GetManifest(reparse=True, type=SubmoduleManifest)
+      m = new.manifestProject
+      self._ApplyOptions(opt, is_new)
+
     if not is_new:
       # Force the manifest to load if it exists, the old graph
       # may be needed inside of _ReloadManifest().

subcmds/manifest.py

@@ -22,7 +22,7 @@ from manifest_xml import XmlManifest
 def _doc(name):
   r = os.path.dirname(__file__)
   r = os.path.dirname(r)
-  fd = open(os.path.join(r, 'docs', 'manifest_xml.txt'))
+  fd = open(os.path.join(r, 'docs', name))
   try:
     return fd.read()
   finally:
@@ -48,6 +48,8 @@ in a Git repository for use during future 'repo init' invocations.
     help = ''
     if isinstance(self.manifest, XmlManifest):
       help += self._xmlHelp + '\n' + _doc('manifest_xml.txt')
+    if isinstance(self.manifest, SubmoduleManifest):
+      help += _doc('manifest_submodule.txt')
     return help
 
   def _Options(self, p):