sync: kill git fetch process before SSH control master process

If the SSH control master process is killed while an active git
fetch is using its network socket, the underlying SSH client may
not realize the connection was broken.  This can lead to both the
client and the server waiting indefinitely for network messages
which will never be sent.

Work around the problem by keeping track of any processes that use
the tunnels we establish.  If we are about to kill any of the SSH
control masters that we started, ensure the clients using them are
successfully killed first.

Change-Id: Ida6c124dcb0c6a26bf7dd69cba2fbdc2ecd5b2fc
Signed-off-by: Shawn O. Pearce <sop@google.com>

git_command.py

@@ -17,6 +17,7 @@ import os
 import sys
 import subprocess
 import tempfile
+from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
 
@@ -29,6 +30,7 @@ LAST_CWD = None
 
 _ssh_proxy_path = None
 _ssh_sock_path = None
+_ssh_clients = []
 
 def ssh_sock(create=True):
   global _ssh_sock_path
@@ -51,6 +53,24 @@ def _ssh_proxy():
       'git_ssh')
   return _ssh_proxy_path
 
+def _add_ssh_client(p):
+  _ssh_clients.append(p)
+
+def _remove_ssh_client(p):
+  try:
+    _ssh_clients.remove(p)
+  except ValueError:
+    pass
+
+def terminate_ssh_clients():
+  global _ssh_clients
+  for p in _ssh_clients:
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
+  _ssh_clients = []
 
 class _GitCall(object):
   def version(self):
@@ -188,6 +208,9 @@ class GitCommand(object):
     except Exception, e:
       raise GitError('%s: %s' % (command[1], e))
 
+    if ssh_proxy:
+      _add_ssh_client(p)
+
     self.process = p
     self.stdin = p.stdin
 
@@ -210,4 +233,8 @@ class GitCommand(object):
     else:
       p.stderr = None
 
-    return self.process.wait()
+    try:
+      rc = p.wait()
+    finally:
+      _remove_ssh_client(p)
+    return rc

git_config.py

@@ -23,7 +23,10 @@ from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
-from git_command import GitCommand, ssh_sock
+
+from git_command import GitCommand
+from git_command import ssh_sock
+from git_command import terminate_ssh_clients
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
@@ -391,6 +394,8 @@ def _open_ssh(host, port):
   return True
 
 def close_ssh():
+  terminate_ssh_clients()
+
   for key,p in _ssh_cache.iteritems():
     try:
       os.kill(p.pid, SIGTERM)