GITC: Add repo start support.

Add repo start support for GITC checkouts. If the user is in
the GITC FS view, they can now run repo start to check out
the sources and create a new working branch.

When "repo start" is called on a GITC project, the revision
tag is set to an empty string and saved in a new tag:
old-revision. This tells the GITC filesystem to display the
local copy of the sources when being viewed. The local copy
is created by pulling the project sources and the new branch
is created based off the original project revision.

Updated main.py to setup each command's gitc_manifest when
appropriate.

Updated repo sync's logic to sync opened projects and
updating the GITC manifest file for the rest.

Change-Id: I7e4809d1c4fc43c69b26f2f1bebe45aab0cae628

command.py

@@ -106,13 +106,13 @@ class Command(object):
   def _UpdatePathToProjectMap(self, project):
     self._by_path[project.worktree] = project
 
-  def _GetProjectByPath(self, path):
+  def _GetProjectByPath(self, manifest, path):
     project = None
     if os.path.exists(path):
       oldpath = None
       while path \
         and path != oldpath \
-        and path != self.manifest.topdir:
+        and path != manifest.topdir:
         try:
           project = self._by_path[path]
           break
@@ -126,13 +126,16 @@ class Command(object):
         pass
     return project
 
-  def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False):
+  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+                  submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects_list = self.manifest.projects
+    if not manifest:
+      manifest = self.manifest
+    all_projects_list = manifest.projects
     result = []
 
-    mp = self.manifest.manifestProject
+    mp = manifest.manifestProject
 
     if not groups:
         groups = mp.config.GetString('manifest.groups')
@@ -155,11 +158,11 @@ class Command(object):
       self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        projects = self.manifest.GetProjectsWithName(arg)
+        projects = manifest.GetProjectsWithName(arg)
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(path)
+          project = self._GetProjectByPath(manifest, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -170,7 +173,7 @@ class Command(object):
               self._UpdatePathToProjectMap(subproject)
               search_again = True
             if search_again:
-              project = self._GetProjectByPath(path) or project
+              project = self._GetProjectByPath(manifest, path) or project
 
           if project:
             projects = [project]

gitc_utils.py

@@ -16,6 +16,7 @@
 from __future__ import print_function
 import os
 import sys
+import time
 
 import git_command
 import git_config
@@ -26,6 +27,18 @@ GITC_MANIFEST_DIR = '/usr/local/google/gitc/'
 GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
 NUM_BATCH_RETRIEVE_REVISIONID = 300
 
+def parse_clientdir(gitc_fs_path):
+  """Parse a path in the GITC FS and return its client name.
+
+  @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+  @returns: The GITC client name
+  """
+  if (gitc_fs_path == GITC_FS_ROOT_DIR or
+      not gitc_fs_path.startswith(GITC_FS_ROOT_DIR)):
+    return None
+  return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
+
 def _set_project_revisions(projects):
   """Sets the revisionExpr for a list of projects.
 
@@ -50,19 +63,37 @@ def _set_project_revisions(projects):
       sys.exit(1)
     proj.revisionExpr = gitcmd.stdout.split('\t')[0]
 
-def generate_gitc_manifest(client_dir, manifest):
+def generate_gitc_manifest(client_dir, manifest, projects=None):
   """Generate a manifest for shafsd to use for this GITC client.
 
   @param client_dir: GITC client directory to install the .manifest file in.
   @param manifest: XmlManifest object representing the repo manifest.
+  @param projects: List of projects we want to update, this must be a sublist
+                   of manifest.projects to work properly. If not provided,
+                   manifest.projects is used.
   """
   print('Generating GITC Manifest by fetching revision SHAs for each '
         'project.')
+  if projects is None:
+    projects = manifest.projects
   index = 0
-  while index < len(manifest.projects):
+  while index < len(projects):
     _set_project_revisions(
-        manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+        projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
     index += NUM_BATCH_RETRIEVE_REVISIONID
   # Save the manifest.
+  save_manifest(manifest, client_dir=client_dir)
+
+def save_manifest(manifest, client_dir=None):
+  """Save the manifest file in the client_dir.
+
+  @param client_dir: Client directory to save the manifest in.
+  @param manifest: Manifest object to save.
+  """
+  if not client_dir:
+    client_dir = manifest.gitc_client_dir
   with open(os.path.join(client_dir, '.manifest'), 'w') as f:
     manifest.Save(f)
+  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+  # Give the GITC filesystem time to register the manifest changes.
+  time.sleep(3)

main.py

@@ -51,7 +51,8 @@ from error import ManifestParseError
 from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
-from manifest_xml import XmlManifest
+import gitc_utils
+from manifest_xml import GitcManifest, XmlManifest
 from pager import RunPager
 from wrapper import WrapperPath, Wrapper
 
@@ -129,6 +130,12 @@ class _Repo(object):
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
+    cmd.gitc_manifest = None
+    gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+    if gitc_client_name:
+      cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
+      cmd.manifest.isGitcClient = True
+
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:

manifest_xml.py

@@ -29,6 +29,7 @@ else:
   urllib = imp.new_module('urllib')
   urllib.parse = urlparse
 
+import gitc_utils
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
@@ -112,6 +113,7 @@ class XmlManifest(object):
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
     self.localManifestWarning = False
+    self.isGitcClient = False
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -306,6 +308,8 @@ class XmlManifest(object):
       if p.clone_depth:
         e.setAttribute('clone-depth', str(p.clone_depth))
 
+      self._output_manifest_project_extras(p, e)
+
       if p.subprojects:
         subprojects = set(subp.name for subp in p.subprojects)
         output_projects(p, e, list(sorted(subprojects)))
@@ -323,6 +327,10 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  def _output_manifest_project_extras(self, p, e):
+    """Manifests can modify e if they support extra project attributes."""
+    pass
+
   @property
   def paths(self):
     self._Load()
@@ -712,7 +720,7 @@ class XmlManifest(object):
   def _UnjoinName(self, parent_name, name):
     return os.path.relpath(name, parent_name)
 
-  def _ParseProject(self, node, parent = None):
+  def _ParseProject(self, node, parent = None, **extra_proj_attrs):
     """
     reads a <project> element from the manifest file
     """
@@ -807,7 +815,8 @@ class XmlManifest(object):
                       clone_depth = clone_depth,
                       upstream = upstream,
                       parent = parent,
-                      dest_branch = dest_branch)
+                      dest_branch = dest_branch,
+                      **extra_proj_attrs)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
@@ -938,3 +947,26 @@ class XmlManifest(object):
       diff['added'].append(toProjects[proj])
 
     return diff
+
+
+class GitcManifest(XmlManifest):
+
+  def __init__(self, repodir, gitc_client_name):
+    """Initialize the GitcManifest object."""
+    super(GitcManifest, self).__init__(repodir)
+    self.isGitcClient = True
+    self.gitc_client_name = gitc_client_name
+    self.gitc_client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+                                        gitc_client_name)
+    self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
+
+  def _ParseProject(self, node, parent = None):
+    """Override _ParseProject and add support for GITC specific attributes."""
+    return super(GitcManifest, self)._ParseProject(
+        node, parent=parent, old_revision=node.getAttribute('old-revision'))
+
+  def _output_manifest_project_extras(self, p, e):
+    """Output GITC Specific Project attributes"""
+    if p.old_revision:
+        e.setAttribute('old-revision', str(p.old_revision))
+

project.py

@@ -572,7 +572,8 @@ class Project(object):
                parent=None,
                is_derived=False,
                dest_branch=None,
-               optimized_fetch=False):
+               optimized_fetch=False,
+               old_revision=None):
     """Init a Project object.
 
     Args:
@@ -596,6 +597,7 @@ class Project(object):
       dest_branch: The branch to which to push changes for review by default.
       optimized_fetch: If True, when a project is set to a sha1 revision, only
                        fetch from the remote if the sha1 is not present locally.
+      old_revision: saved git commit id for open GITC projects.
     """
     self.manifest = manifest
     self.name = name
@@ -643,6 +645,7 @@ class Project(object):
     self.bare_ref = GitRefs(gitdir)
     self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
     self.dest_branch = dest_branch
+    self.old_revision = old_revision
 
     # This will be filled in if a project is later identified to be the
     # project containing repo hooks.
@@ -1195,6 +1198,8 @@ class Project(object):
     self._InitHooks()
 
   def _CopyAndLinkFiles(self):
+    if self.manifest.isGitcClient:
+      return
     for copyfile in self.copyfiles:
       copyfile._Copy()
     for linkfile in self.linkfiles:
@@ -1425,9 +1430,11 @@ class Project(object):
 
 ## Branch Management ##
 
-  def StartBranch(self, name):
+  def StartBranch(self, name, branch_merge=''):
     """Create a new branch off the manifest's revision.
     """
+    if not branch_merge:
+      branch_merge = self.revisionExpr
     head = self.work_git.GetHead()
     if head == (R_HEADS + name):
       return True
@@ -1441,9 +1448,9 @@ class Project(object):
 
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revisionExpr
-    if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
-      branch.merge = R_HEADS + self.revisionExpr
+    branch.merge = branch_merge
+    if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
+      branch.merge = R_HEADS + branch_merge
     revid = self.GetRevisionId(all_refs)
 
     if head.startswith(R_HEADS):
@@ -1451,7 +1458,6 @@ class Project(object):
         head = all_refs[head]
       except KeyError:
         head = None
-
     if revid and head and revid == head:
       ref = os.path.join(self.gitdir, R_HEADS + name)
       try:

subcmds/start.py

@@ -14,11 +14,15 @@
 # limitations under the License.
 
 from __future__ import print_function
+import os
 import sys
+
 from command import Command
 from git_config import IsId
 from git_command import git
+import gitc_utils
 from progress import Progress
+from project import SyncBuffer
 
 class Start(Command):
   common = True
@@ -53,20 +57,50 @@ revision specified in the manifest.
         print("error: at least one project must be specified", file=sys.stderr)
         sys.exit(1)
 
-    all_projects = self.GetProjects(projects)
+    proj_name_to_gitc_proj_dict = {}
+    if self.gitc_manifest:
+      all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+                                      missing_ok=True)
+      for project in all_projects:
+        if project.old_revision:
+          project.already_synced = True
+        else:
+          project.already_synced = False
+          project.old_revision = project.revisionExpr
+        proj_name_to_gitc_proj_dict[project.name] = project
+        project.revisionExpr = None
+      # Save the GITC manifest.
+      gitc_utils.save_manifest(self.gitc_manifest)
 
+    all_projects = self.GetProjects(projects,
+                                    missing_ok=bool(self.gitc_manifest))
     pm = Progress('Starting %s' % nb, len(all_projects))
     for project in all_projects:
       pm.update()
+      if self.gitc_manifest:
+        gitc_project = proj_name_to_gitc_proj_dict[project.name]
+        # Sync projects that have already been opened.
+        if not gitc_project.already_synced:
+          proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+                                       project.relpath)
+          project.worktree = proj_localdir
+          if not os.path.exists(proj_localdir):
+            os.makedirs(proj_localdir)
+          project.Sync_NetworkHalf()
+          sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+          project.Sync_LocalHalf(sync_buf)
+          project.revisionExpr = gitc_project.old_revision
+
       # If the current revision is a specific SHA1 then we can't push back
       # to it; so substitute with dest_branch if defined, or with manifest
       # default revision instead.
+      branch_merge = ''
       if IsId(project.revisionExpr):
         if project.dest_branch:
-          project.revisionExpr = project.dest_branch
+          branch_merge = project.dest_branch
         else:
-          project.revisionExpr = self.manifest.default.revisionExpr
-      if not project.StartBranch(nb):
+          branch_merge = self.manifest.default.revisionExpr
+      if not project.StartBranch(nb, branch_merge=branch_merge):
         err.append(project)
     pm.end()
 

subcmds/sync.py

@@ -549,15 +549,6 @@ later is required to fix a server side protocol bug.
              cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1]))
       sys.exit(1)
 
-    self._gitc_sync = False
-    if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR):
-      self._gitc_sync = True
-      self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split(
-          '/')[0]
-      self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
-                                      self._client_name)
-      print('Updating GITC client: %s' % self._client_name)
-
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
@@ -677,12 +668,6 @@ later is required to fix a server side protocol bug.
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
-    if self._gitc_sync:
-      gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest)
-      print('GITC client successfully synced.')
-      return
-
-
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
@@ -697,6 +682,35 @@ later is required to fix a server side protocol bug.
       self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
+
+    # TODO (sbasi) - Add support for manifest changes, aka projects
+    # have been added or deleted from the manifest.
+    if self.gitc_manifest:
+      gitc_manifest_projects = self.GetProjects(args,
+                                                manifest=self.gitc_manifest,
+                                                missing_ok=True)
+      gitc_projects = []
+      opened_projects = []
+      for project in gitc_manifest_projects:
+        if not project.old_revision:
+          gitc_projects.append(project)
+        else:
+          opened_projects.append(project)
+
+      if gitc_projects and not opt.local_only:
+        print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
+        gitc_utils.generate_gitc_manifest(self.gitc_manifest.gitc_client_dir,
+                                          self.gitc_manifest,
+                                          gitc_projects)
+        print('GITC client successfully synced.')
+
+      # The opened projects need to be synced as normal, therefore we
+      # generate a new args list to represent the opened projects.
+      args = []
+      for proj in opened_projects:
+        args.append(os.path.relpath(proj.worktree, cwd))
+      if not args:
+        return
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)