add experimental git worktree support

This provides initial support for using git worktrees internally
instead of our own ad-hoc symlink tree.  It's been lightly tested
which is why it's not currently exposed via --help.

When people opt-in to worktrees in an existing repo client checkout,
no projects are migrated.  Instead, only new projects will use the
worktree method.  This allows for limited testing/opting in without
having to completely blow things away or get a second checkout.

Bug: https://crbug.com/gerrit/11486
Change-Id: Ic3ff891b30940a6ba497b406b2a387e0a8517ed8
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/254075
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>

docs/internal-fs-layout.md

@@ -102,6 +102,11 @@ support, see the [manifest-format.md] file.
     respective servers ...
 *   `subprojects/`: Like `projects/`, but for git submodules.
 *   `subproject-objects/`: Like `project-objects/`, but for git submodules.
+*   `worktrees/`: Bare checkouts of every project synced by the manifest.  The
+    filesystem layout matches the `<project name=...` setting in the manifest
+    (i.e. the path on the remote server).
+
+    This is used when git worktrees are enabled.
 
 ### Global settings
 
@@ -121,6 +126,7 @@ User controlled settings are initialized when running `repo init`.
 | repo.partialclone | `--partial-clone`         | Create [partial git clones] |
 | repo.reference    | `--reference`             | Reference repo client checkout |
 | repo.submodules   | `--submodules`            | Sync git submodules |
+| repo.worktree     | `--worktree`              | Use `git worktree` for checkouts |
 | user.email        | `--config-name`           | User's e-mail address; Copied into `.git/config` when checking out a new project |
 | user.name         | `--config-name`           | User's name; Copied into `.git/config` when checking out a new project |
 

manifest_xml.py

@@ -146,9 +146,17 @@ class XmlManifest(object):
                                    gitdir=os.path.join(repodir, 'repo/.git'),
                                    worktree=os.path.join(repodir, 'repo'))
 
-    self.manifestProject = MetaProject(self, 'manifests',
-                                       gitdir=os.path.join(repodir, 'manifests.git'),
-                                       worktree=os.path.join(repodir, 'manifests'))
+    mp = MetaProject(self, 'manifests',
+                     gitdir=os.path.join(repodir, 'manifests.git'),
+                     worktree=os.path.join(repodir, 'manifests'))
+    self.manifestProject = mp
+
+    # This is a bit hacky, but we're in a chicken & egg situation: all the
+    # normal repo settings live in the manifestProject which we just setup
+    # above, so we couldn't easily query before that.  We assume Project()
+    # init doesn't care if this changes afterwards.
+    if mp.config.GetBoolean('repo.worktree'):
+      mp.use_git_worktrees = True
 
     self._Unload()
 
@@ -427,6 +435,10 @@ class XmlManifest(object):
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
 
+  @property
+  def UseGitWorktrees(self):
+    return self.manifestProject.config.GetBoolean('repo.worktree')
+
   @property
   def IsArchive(self):
     return self.manifestProject.config.GetBoolean('repo.archive')
@@ -873,8 +885,10 @@ class XmlManifest(object):
     groups = self._ParseGroups(groups)
 
     if parent is None:
-      relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
+      relpath, worktree, gitdir, objdir, use_git_worktrees = \
+          self.GetProjectPaths(name, path)
     else:
+      use_git_worktrees = False
       relpath, worktree, gitdir, objdir = \
           self.GetSubprojectPaths(parent, name, path)
 
@@ -903,6 +917,7 @@ class XmlManifest(object):
                       upstream=upstream,
                       parent=parent,
                       dest_branch=dest_branch,
+                      use_git_worktrees=use_git_worktrees,
                       **extra_proj_attrs)
 
     for n in node.childNodes:
@@ -918,6 +933,7 @@ class XmlManifest(object):
     return project
 
   def GetProjectPaths(self, name, path):
+    use_git_worktrees = False
     relpath = path
     if self.IsMirror:
       worktree = None
@@ -926,8 +942,15 @@ class XmlManifest(object):
     else:
       worktree = os.path.join(self.topdir, path).replace('\\', '/')
       gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
-      objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
-    return relpath, worktree, gitdir, objdir
+      # We allow people to mix git worktrees & non-git worktrees for now.
+      # This allows for in situ migration of repo clients.
+      if os.path.exists(gitdir) or not self.UseGitWorktrees:
+        objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+      else:
+        use_git_worktrees = True
+        gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
+        objdir = gitdir
+    return relpath, worktree, gitdir, objdir, use_git_worktrees
 
   def GetProjectsWithName(self, name):
     return self._projects.get(name, [])

project.py

@@ -866,6 +866,7 @@ class Project(object):
                clone_depth=None,
                upstream=None,
                parent=None,
+               use_git_worktrees=False,
                is_derived=False,
                dest_branch=None,
                optimized_fetch=False,
@@ -889,6 +890,7 @@ class Project(object):
       sync_tags: The `sync-tags` attribute of manifest.xml's project element.
       upstream: The `upstream` attribute of manifest.xml's project element.
       parent: The parent Project object.
+      use_git_worktrees: Whether to use `git worktree` for this project.
       is_derived: False if the project was explicitly defined in the manifest;
                   True if the project is a discovered submodule.
       dest_branch: The branch to which to push changes for review by default.
@@ -923,6 +925,10 @@ class Project(object):
     self.clone_depth = clone_depth
     self.upstream = upstream
     self.parent = parent
+    # NB: Do not use this setting in __init__ to change behavior so that the
+    # manifest.git checkout can inspect & change it after instantiating.  See
+    # the XmlManifest init code for more info.
+    self.use_git_worktrees = use_git_worktrees
     self.is_derived = is_derived
     self.optimized_fetch = optimized_fetch
     self.subprojects = []
@@ -1872,15 +1878,19 @@ class Project(object):
       except KeyError:
         head = None
     if revid and head and revid == head:
-      ref = os.path.join(self.gitdir, R_HEADS + name)
-      try:
-        os.makedirs(os.path.dirname(ref))
-      except OSError:
-        pass
-      _lwrite(ref, '%s\n' % revid)
-      _lwrite(self.GetHeadPath(), 'ref: %s%s\n' % (R_HEADS, name))
-      branch.Save()
-      return True
+      if self.use_git_worktrees:
+        self.work_git.update_ref(HEAD, revid)
+        branch.Save()
+      else:
+        ref = os.path.join(self.gitdir, R_HEADS + name)
+        try:
+          os.makedirs(os.path.dirname(ref))
+        except OSError:
+          pass
+        _lwrite(ref, '%s\n' % revid)
+        _lwrite(self.GetHeadPath(), 'ref: %s%s\n' % (R_HEADS, name))
+        branch.Save()
+        return True
 
     if GitCommand(self,
                   ['checkout', '-b', branch.name, revid],
@@ -2617,6 +2627,11 @@ class Project(object):
         os.makedirs(self.objdir)
         self.bare_objdir.init()
 
+        # Enable per-worktree config file support if possible.  This is more a
+        # nice-to-have feature for users rather than a hard requirement.
+        if self.use_git_worktrees and git_require((2, 19, 0)):
+          self.config.SetString('extensions.worktreeConfig', 'true')
+
       # If we have a separate directory to hold refs, initialize it as well.
       if self.objdir != self.gitdir:
         if init_git_dir:
@@ -2651,13 +2666,15 @@ class Project(object):
             mirror_git = os.path.join(ref_dir, self.name + '.git')
           repo_git = os.path.join(ref_dir, '.repo', 'projects',
                                   self.relpath + '.git')
+          worktrees_git = os.path.join(ref_dir, '.repo', 'worktrees',
+                                       self.name + '.git')
 
           if os.path.exists(mirror_git):
             ref_dir = mirror_git
-
           elif os.path.exists(repo_git):
             ref_dir = repo_git
-
+          elif os.path.exists(worktrees_git):
+            ref_dir = worktrees_git
           else:
             ref_dir = None
 
@@ -2765,6 +2782,10 @@ class Project(object):
         self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
   def _CheckDirReference(self, srcdir, destdir, share_refs):
+    # Git worktrees don't use symlinks to share at all.
+    if self.use_git_worktrees:
+      return
+
     symlink_files = self.shareable_files[:]
     symlink_dirs = self.shareable_dirs[:]
     if share_refs:
@@ -2864,11 +2885,38 @@ class Project(object):
         else:
           raise
 
+  def _InitGitWorktree(self):
+    """Init the project using git worktrees."""
+    self.bare_git.worktree('prune')
+    self.bare_git.worktree('add', '-ff', '--checkout', '--detach', '--lock',
+                           self.worktree, self.GetRevisionId())
+
+    # Rewrite the internal state files to use relative paths between the
+    # checkouts & worktrees.
+    dotgit = os.path.join(self.worktree, '.git')
+    with open(dotgit, 'r') as fp:
+      # Figure out the checkout->worktree path.
+      setting = fp.read()
+      assert setting.startswith('gitdir:')
+      git_worktree_path = setting.split(':', 1)[1].strip()
+    # Use relative path from checkout->worktree.
+    with open(dotgit, 'w') as fp:
+      print('gitdir:', os.path.relpath(git_worktree_path, self.worktree),
+            file=fp)
+    # Use relative path from worktree->checkout.
+    with open(os.path.join(git_worktree_path, 'gitdir'), 'w') as fp:
+      print(os.path.relpath(dotgit, git_worktree_path), file=fp)
+
   def _InitWorkTree(self, force_sync=False, submodules=False):
     realdotgit = os.path.join(self.worktree, '.git')
     tmpdotgit = realdotgit + '.tmp'
     init_dotgit = not os.path.exists(realdotgit)
     if init_dotgit:
+      if self.use_git_worktrees:
+        self._InitGitWorktree()
+        self._CopyAndLinkFiles()
+        return
+
       dotgit = tmpdotgit
       platform_utils.rmtree(tmpdotgit, ignore_errors=True)
       os.makedirs(tmpdotgit)

repo

@@ -302,6 +302,8 @@ def GetParser(gitc_init=False):
   group.add_option('--clone-filter', action='store', default='blob:none',
                    help='filter for use with --partial-clone '
                         '[default: %default]')
+  group.add_option('--worktree', action='store_true',
+                   help=optparse.SUPPRESS_HELP)
   group.add_option('--archive', action='store_true',
                    help='checkout an archive instead of a git repository for '
                         'each project. See git archive.')

subcmds/init.py

@@ -15,6 +15,8 @@
 # limitations under the License.
 
 from __future__ import print_function
+
+import optparse
 import os
 import platform
 import re
@@ -128,6 +130,10 @@ to update the working directory files.
     g.add_option('--clone-filter', action='store', default='blob:none',
                  dest='clone_filter',
                  help='filter for use with --partial-clone [default: %default]')
+    # TODO(vapier): Expose option with real help text once this has been in the
+    # wild for a while w/out significant bug reports.  Goal is by ~Sep 2020.
+    g.add_option('--worktree', action='store_true',
+                 help=optparse.SUPPRESS_HELP)
     g.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
@@ -246,6 +252,20 @@ to update the working directory files.
     if opt.dissociate:
       m.config.SetString('repo.dissociate', 'true')
 
+    if opt.worktree:
+      if opt.mirror:
+        print('fatal: --mirror and --worktree are incompatible',
+              file=sys.stderr)
+        sys.exit(1)
+      if opt.submodules:
+        print('fatal: --submodules and --worktree are incompatible',
+              file=sys.stderr)
+        sys.exit(1)
+      m.config.SetString('repo.worktree', 'true')
+      if is_new:
+        m.use_git_worktrees = True
+      print('warning: --worktree is experimental!', file=sys.stderr)
+
     if opt.archive:
       if is_new:
         m.config.SetString('repo.archive', 'true')
@@ -459,6 +479,10 @@ to update the working directory files.
             % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
             file=sys.stderr)
 
+    if opt.worktree:
+      # Older versions of git supported worktree, but had dangerous gc bugs.
+      git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
+
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 

subcmds/sync.py

@@ -15,6 +15,8 @@
 # limitations under the License.
 
 from __future__ import print_function
+
+import errno
 import json
 import netrc
 from optparse import SUPPRESS_HELP
@@ -569,7 +571,8 @@ later is required to fix a server side protocol bug.
     gc_gitdirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
-      if len(project.manifest.GetProjectsWithName(project.name)) > 1:
+      if (not project.use_git_worktrees and
+          len(project.manifest.GetProjectsWithName(project.name)) > 1):
         print('%s: Shared project %s found, disabling pruning.' %
               (project.relpath, project.name))
         if git_require((2, 7, 0)):
@@ -637,13 +640,22 @@ later is required to fix a server side protocol bug.
     # Delete the .git directory first, so we're less likely to have a partially
     # working git repository around. There shouldn't be any git projects here,
     # so rmtree works.
+    dotgit = os.path.join(path, '.git')
+    # Try to remove plain files first in case of git worktrees.  If this fails
+    # for any reason, we'll fall back to rmtree, and that'll display errors if
+    # it can't remove things either.
+    try:
+      platform_utils.remove(dotgit)
+    except OSError:
+      pass
     try:
-      platform_utils.rmtree(os.path.join(path, '.git'))
+      platform_utils.rmtree(dotgit)
     except OSError as e:
-      print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
-      print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
-      print('       remove manually, then run sync again', file=sys.stderr)
-      return 1
+      if e.errno != errno.ENOENT:
+        print('error: %s: %s' % (dotgit, str(e)), file=sys.stderr)
+        print('error: %s: Failed to delete obsolete path; remove manually, then '
+              'run sync again' % (path,), file=sys.stderr)
+        return 1
 
     # Delete everything under the worktree, except for directories that contain
     # another git project