project.py 81 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674
  1. # Copyright (C) 2008 The Android Open Source Project
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import print_function
  15. import traceback
  16. import errno
  17. import filecmp
  18. import os
  19. import random
  20. import re
  21. import shutil
  22. import stat
  23. import subprocess
  24. import sys
  25. import tarfile
  26. import tempfile
  27. import time
  28. from color import Coloring
  29. from git_command import GitCommand, git_require
  30. from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
  31. from error import GitError, HookError, UploadError
  32. from error import ManifestInvalidRevisionError
  33. from error import NoManifestException
  34. from trace import IsTrace, Trace
  35. from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
  36. from pyversion import is_python3
  37. if not is_python3():
  38. # pylint:disable=W0622
  39. input = raw_input
  40. # pylint:enable=W0622
  41. def _lwrite(path, content):
  42. lock = '%s.lock' % path
  43. fd = open(lock, 'wb')
  44. try:
  45. fd.write(content)
  46. finally:
  47. fd.close()
  48. try:
  49. os.rename(lock, path)
  50. except OSError:
  51. os.remove(lock)
  52. raise
  53. def _error(fmt, *args):
  54. msg = fmt % args
  55. print('error: %s' % msg, file=sys.stderr)
  56. def not_rev(r):
  57. return '^' + r
  58. def sq(r):
  59. return "'" + r.replace("'", "'\''") + "'"
  60. _project_hook_list = None
  61. def _ProjectHooks():
  62. """List the hooks present in the 'hooks' directory.
  63. These hooks are project hooks and are copied to the '.git/hooks' directory
  64. of all subprojects.
  65. This function caches the list of hooks (based on the contents of the
  66. 'repo/hooks' directory) on the first call.
  67. Returns:
  68. A list of absolute paths to all of the files in the hooks directory.
  69. """
  70. global _project_hook_list
  71. if _project_hook_list is None:
  72. d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
  73. d = os.path.join(d , 'hooks')
  74. _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
  75. return _project_hook_list
  76. class DownloadedChange(object):
  77. _commit_cache = None
  78. def __init__(self, project, base, change_id, ps_id, commit):
  79. self.project = project
  80. self.base = base
  81. self.change_id = change_id
  82. self.ps_id = ps_id
  83. self.commit = commit
  84. @property
  85. def commits(self):
  86. if self._commit_cache is None:
  87. self._commit_cache = self.project.bare_git.rev_list(
  88. '--abbrev=8',
  89. '--abbrev-commit',
  90. '--pretty=oneline',
  91. '--reverse',
  92. '--date-order',
  93. not_rev(self.base),
  94. self.commit,
  95. '--')
  96. return self._commit_cache
  97. class ReviewableBranch(object):
  98. _commit_cache = None
  99. def __init__(self, project, branch, base):
  100. self.project = project
  101. self.branch = branch
  102. self.base = base
  103. @property
  104. def name(self):
  105. return self.branch.name
  106. @property
  107. def commits(self):
  108. if self._commit_cache is None:
  109. self._commit_cache = self.project.bare_git.rev_list(
  110. '--abbrev=8',
  111. '--abbrev-commit',
  112. '--pretty=oneline',
  113. '--reverse',
  114. '--date-order',
  115. not_rev(self.base),
  116. R_HEADS + self.name,
  117. '--')
  118. return self._commit_cache
  119. @property
  120. def unabbrev_commits(self):
  121. r = dict()
  122. for commit in self.project.bare_git.rev_list(
  123. not_rev(self.base),
  124. R_HEADS + self.name,
  125. '--'):
  126. r[commit[0:8]] = commit
  127. return r
  128. @property
  129. def date(self):
  130. return self.project.bare_git.log(
  131. '--pretty=format:%cd',
  132. '-n', '1',
  133. R_HEADS + self.name,
  134. '--')
  135. def UploadForReview(self, people, auto_topic=False, draft=False, dest_branch=None):
  136. self.project.UploadForReview(self.name,
  137. people,
  138. auto_topic=auto_topic,
  139. draft=draft,
  140. dest_branch=dest_branch)
  141. def GetPublishedRefs(self):
  142. refs = {}
  143. output = self.project.bare_git.ls_remote(
  144. self.branch.remote.SshReviewUrl(self.project.UserEmail),
  145. 'refs/changes/*')
  146. for line in output.split('\n'):
  147. try:
  148. (sha, ref) = line.split()
  149. refs[sha] = ref
  150. except ValueError:
  151. pass
  152. return refs
  153. class StatusColoring(Coloring):
  154. def __init__(self, config):
  155. Coloring.__init__(self, config, 'status')
  156. self.project = self.printer('header', attr = 'bold')
  157. self.branch = self.printer('header', attr = 'bold')
  158. self.nobranch = self.printer('nobranch', fg = 'red')
  159. self.important = self.printer('important', fg = 'red')
  160. self.added = self.printer('added', fg = 'green')
  161. self.changed = self.printer('changed', fg = 'red')
  162. self.untracked = self.printer('untracked', fg = 'red')
  163. class DiffColoring(Coloring):
  164. def __init__(self, config):
  165. Coloring.__init__(self, config, 'diff')
  166. self.project = self.printer('header', attr = 'bold')
  167. class _Annotation:
  168. def __init__(self, name, value, keep):
  169. self.name = name
  170. self.value = value
  171. self.keep = keep
  172. class _CopyFile:
  173. def __init__(self, src, dest, abssrc, absdest):
  174. self.src = src
  175. self.dest = dest
  176. self.abs_src = abssrc
  177. self.abs_dest = absdest
  178. def _Copy(self):
  179. src = self.abs_src
  180. dest = self.abs_dest
  181. # copy file if it does not exist or is out of date
  182. if not os.path.exists(dest) or not filecmp.cmp(src, dest):
  183. try:
  184. # remove existing file first, since it might be read-only
  185. if os.path.exists(dest):
  186. os.remove(dest)
  187. else:
  188. dest_dir = os.path.dirname(dest)
  189. if not os.path.isdir(dest_dir):
  190. os.makedirs(dest_dir)
  191. shutil.copy(src, dest)
  192. # make the file read-only
  193. mode = os.stat(dest)[stat.ST_MODE]
  194. mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
  195. os.chmod(dest, mode)
  196. except IOError:
  197. _error('Cannot copy file %s to %s', src, dest)
  198. class _LinkFile:
  199. def __init__(self, src, dest, abssrc, absdest):
  200. self.src = src
  201. self.dest = dest
  202. self.abs_src = abssrc
  203. self.abs_dest = absdest
  204. def _Link(self):
  205. src = self.abs_src
  206. dest = self.abs_dest
  207. # link file if it does not exist or is out of date
  208. if not os.path.islink(dest) or os.readlink(dest) != src:
  209. try:
  210. # remove existing file first, since it might be read-only
  211. if os.path.exists(dest):
  212. os.remove(dest)
  213. else:
  214. dest_dir = os.path.dirname(dest)
  215. if not os.path.isdir(dest_dir):
  216. os.makedirs(dest_dir)
  217. os.symlink(src, dest)
  218. except IOError:
  219. _error('Cannot link file %s to %s', src, dest)
  220. class RemoteSpec(object):
  221. def __init__(self,
  222. name,
  223. url = None,
  224. review = None,
  225. revision = None):
  226. self.name = name
  227. self.url = url
  228. self.review = review
  229. self.revision = revision
  230. class RepoHook(object):
  231. """A RepoHook contains information about a script to run as a hook.
  232. Hooks are used to run a python script before running an upload (for instance,
  233. to run presubmit checks). Eventually, we may have hooks for other actions.
  234. This shouldn't be confused with files in the 'repo/hooks' directory. Those
  235. files are copied into each '.git/hooks' folder for each project. Repo-level
  236. hooks are associated instead with repo actions.
  237. Hooks are always python. When a hook is run, we will load the hook into the
  238. interpreter and execute its main() function.
  239. """
  240. def __init__(self,
  241. hook_type,
  242. hooks_project,
  243. topdir,
  244. abort_if_user_denies=False):
  245. """RepoHook constructor.
  246. Params:
  247. hook_type: A string representing the type of hook. This is also used
  248. to figure out the name of the file containing the hook. For
  249. example: 'pre-upload'.
  250. hooks_project: The project containing the repo hooks. If you have a
  251. manifest, this is manifest.repo_hooks_project. OK if this is None,
  252. which will make the hook a no-op.
  253. topdir: Repo's top directory (the one containing the .repo directory).
  254. Scripts will run with CWD as this directory. If you have a manifest,
  255. this is manifest.topdir
  256. abort_if_user_denies: If True, we'll throw a HookError() if the user
  257. doesn't allow us to run the hook.
  258. """
  259. self._hook_type = hook_type
  260. self._hooks_project = hooks_project
  261. self._topdir = topdir
  262. self._abort_if_user_denies = abort_if_user_denies
  263. # Store the full path to the script for convenience.
  264. if self._hooks_project:
  265. self._script_fullpath = os.path.join(self._hooks_project.worktree,
  266. self._hook_type + '.py')
  267. else:
  268. self._script_fullpath = None
  269. def _GetHash(self):
  270. """Return a hash of the contents of the hooks directory.
  271. We'll just use git to do this. This hash has the property that if anything
  272. changes in the directory we will return a different has.
  273. SECURITY CONSIDERATION:
  274. This hash only represents the contents of files in the hook directory, not
  275. any other files imported or called by hooks. Changes to imported files
  276. can change the script behavior without affecting the hash.
  277. Returns:
  278. A string representing the hash. This will always be ASCII so that it can
  279. be printed to the user easily.
  280. """
  281. assert self._hooks_project, "Must have hooks to calculate their hash."
  282. # We will use the work_git object rather than just calling GetRevisionId().
  283. # That gives us a hash of the latest checked in version of the files that
  284. # the user will actually be executing. Specifically, GetRevisionId()
  285. # doesn't appear to change even if a user checks out a different version
  286. # of the hooks repo (via git checkout) nor if a user commits their own revs.
  287. #
  288. # NOTE: Local (non-committed) changes will not be factored into this hash.
  289. # I think this is OK, since we're really only worried about warning the user
  290. # about upstream changes.
  291. return self._hooks_project.work_git.rev_parse('HEAD')
  292. def _GetMustVerb(self):
  293. """Return 'must' if the hook is required; 'should' if not."""
  294. if self._abort_if_user_denies:
  295. return 'must'
  296. else:
  297. return 'should'
  298. def _CheckForHookApproval(self):
  299. """Check to see whether this hook has been approved.
  300. We'll look at the hash of all of the hooks. If this matches the hash that
  301. the user last approved, we're done. If it doesn't, we'll ask the user
  302. about approval.
  303. Note that we ask permission for each individual hook even though we use
  304. the hash of all hooks when detecting changes. We'd like the user to be
  305. able to approve / deny each hook individually. We only use the hash of all
  306. hooks because there is no other easy way to detect changes to local imports.
  307. Returns:
  308. True if this hook is approved to run; False otherwise.
  309. Raises:
  310. HookError: Raised if the user doesn't approve and abort_if_user_denies
  311. was passed to the consturctor.
  312. """
  313. hooks_config = self._hooks_project.config
  314. git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
  315. # Get the last hash that the user approved for this hook; may be None.
  316. old_hash = hooks_config.GetString(git_approval_key)
  317. # Get the current hash so we can tell if scripts changed since approval.
  318. new_hash = self._GetHash()
  319. if old_hash is not None:
  320. # User previously approved hook and asked not to be prompted again.
  321. if new_hash == old_hash:
  322. # Approval matched. We're done.
  323. return True
  324. else:
  325. # Give the user a reason why we're prompting, since they last told
  326. # us to "never ask again".
  327. prompt = 'WARNING: Scripts have changed since %s was allowed.\n\n' % (
  328. self._hook_type)
  329. else:
  330. prompt = ''
  331. # Prompt the user if we're not on a tty; on a tty we'll assume "no".
  332. if sys.stdout.isatty():
  333. prompt += ('Repo %s run the script:\n'
  334. ' %s\n'
  335. '\n'
  336. 'Do you want to allow this script to run '
  337. '(yes/yes-never-ask-again/NO)? ') % (
  338. self._GetMustVerb(), self._script_fullpath)
  339. response = input(prompt).lower()
  340. print()
  341. # User is doing a one-time approval.
  342. if response in ('y', 'yes'):
  343. return True
  344. elif response == 'yes-never-ask-again':
  345. hooks_config.SetString(git_approval_key, new_hash)
  346. return True
  347. # For anything else, we'll assume no approval.
  348. if self._abort_if_user_denies:
  349. raise HookError('You must allow the %s hook or use --no-verify.' %
  350. self._hook_type)
  351. return False
  352. def _ExecuteHook(self, **kwargs):
  353. """Actually execute the given hook.
  354. This will run the hook's 'main' function in our python interpreter.
  355. Args:
  356. kwargs: Keyword arguments to pass to the hook. These are often specific
  357. to the hook type. For instance, pre-upload hooks will contain
  358. a project_list.
  359. """
  360. # Keep sys.path and CWD stashed away so that we can always restore them
  361. # upon function exit.
  362. orig_path = os.getcwd()
  363. orig_syspath = sys.path
  364. try:
  365. # Always run hooks with CWD as topdir.
  366. os.chdir(self._topdir)
  367. # Put the hook dir as the first item of sys.path so hooks can do
  368. # relative imports. We want to replace the repo dir as [0] so
  369. # hooks can't import repo files.
  370. sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
  371. # Exec, storing global context in the context dict. We catch exceptions
  372. # and convert to a HookError w/ just the failing traceback.
  373. context = {}
  374. try:
  375. execfile(self._script_fullpath, context)
  376. except Exception:
  377. raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
  378. traceback.format_exc(), self._hook_type))
  379. # Running the script should have defined a main() function.
  380. if 'main' not in context:
  381. raise HookError('Missing main() in: "%s"' % self._script_fullpath)
  382. # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
  383. # We don't actually want hooks to define their main with this argument--
  384. # it's there to remind them that their hook should always take **kwargs.
  385. # For instance, a pre-upload hook should be defined like:
  386. # def main(project_list, **kwargs):
  387. #
  388. # This allows us to later expand the API without breaking old hooks.
  389. kwargs = kwargs.copy()
  390. kwargs['hook_should_take_kwargs'] = True
  391. # Call the main function in the hook. If the hook should cause the
  392. # build to fail, it will raise an Exception. We'll catch that convert
  393. # to a HookError w/ just the failing traceback.
  394. try:
  395. context['main'](**kwargs)
  396. except Exception:
  397. raise HookError('%s\nFailed to run main() for %s hook; see traceback '
  398. 'above.' % (
  399. traceback.format_exc(), self._hook_type))
  400. finally:
  401. # Restore sys.path and CWD.
  402. sys.path = orig_syspath
  403. os.chdir(orig_path)
  404. def Run(self, user_allows_all_hooks, **kwargs):
  405. """Run the hook.
  406. If the hook doesn't exist (because there is no hooks project or because
  407. this particular hook is not enabled), this is a no-op.
  408. Args:
  409. user_allows_all_hooks: If True, we will never prompt about running the
  410. hook--we'll just assume it's OK to run it.
  411. kwargs: Keyword arguments to pass to the hook. These are often specific
  412. to the hook type. For instance, pre-upload hooks will contain
  413. a project_list.
  414. Raises:
  415. HookError: If there was a problem finding the hook or the user declined
  416. to run a required hook (from _CheckForHookApproval).
  417. """
  418. # No-op if there is no hooks project or if hook is disabled.
  419. if ((not self._hooks_project) or
  420. (self._hook_type not in self._hooks_project.enabled_repo_hooks)):
  421. return
  422. # Bail with a nice error if we can't find the hook.
  423. if not os.path.isfile(self._script_fullpath):
  424. raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
  425. # Make sure the user is OK with running the hook.
  426. if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
  427. return
  428. # Run the hook with the same version of python we're using.
  429. self._ExecuteHook(**kwargs)
  430. class Project(object):
  431. def __init__(self,
  432. manifest,
  433. name,
  434. remote,
  435. gitdir,
  436. objdir,
  437. worktree,
  438. relpath,
  439. revisionExpr,
  440. revisionId,
  441. rebase = True,
  442. groups = None,
  443. sync_c = False,
  444. sync_s = False,
  445. clone_depth = None,
  446. upstream = None,
  447. parent = None,
  448. is_derived = False,
  449. dest_branch = None):
  450. """Init a Project object.
  451. Args:
  452. manifest: The XmlManifest object.
  453. name: The `name` attribute of manifest.xml's project element.
  454. remote: RemoteSpec object specifying its remote's properties.
  455. gitdir: Absolute path of git directory.
  456. objdir: Absolute path of directory to store git objects.
  457. worktree: Absolute path of git working tree.
  458. relpath: Relative path of git working tree to repo's top directory.
  459. revisionExpr: The `revision` attribute of manifest.xml's project element.
  460. revisionId: git commit id for checking out.
  461. rebase: The `rebase` attribute of manifest.xml's project element.
  462. groups: The `groups` attribute of manifest.xml's project element.
  463. sync_c: The `sync-c` attribute of manifest.xml's project element.
  464. sync_s: The `sync-s` attribute of manifest.xml's project element.
  465. upstream: The `upstream` attribute of manifest.xml's project element.
  466. parent: The parent Project object.
  467. is_derived: False if the project was explicitly defined in the manifest;
  468. True if the project is a discovered submodule.
  469. dest_branch: The branch to which to push changes for review by default.
  470. """
  471. self.manifest = manifest
  472. self.name = name
  473. self.remote = remote
  474. self.gitdir = gitdir.replace('\\', '/')
  475. self.objdir = objdir.replace('\\', '/')
  476. if worktree:
  477. self.worktree = worktree.replace('\\', '/')
  478. else:
  479. self.worktree = None
  480. self.relpath = relpath
  481. self.revisionExpr = revisionExpr
  482. if revisionId is None \
  483. and revisionExpr \
  484. and IsId(revisionExpr):
  485. self.revisionId = revisionExpr
  486. else:
  487. self.revisionId = revisionId
  488. self.rebase = rebase
  489. self.groups = groups
  490. self.sync_c = sync_c
  491. self.sync_s = sync_s
  492. self.clone_depth = clone_depth
  493. self.upstream = upstream
  494. self.parent = parent
  495. self.is_derived = is_derived
  496. self.subprojects = []
  497. self.snapshots = {}
  498. self.copyfiles = []
  499. self.linkfiles = []
  500. self.annotations = []
  501. self.config = GitConfig.ForRepository(
  502. gitdir = self.gitdir,
  503. defaults = self.manifest.globalConfig)
  504. if self.worktree:
  505. self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
  506. else:
  507. self.work_git = None
  508. self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
  509. self.bare_ref = GitRefs(gitdir)
  510. self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
  511. self.dest_branch = dest_branch
  512. # This will be filled in if a project is later identified to be the
  513. # project containing repo hooks.
  514. self.enabled_repo_hooks = []
  515. @property
  516. def Derived(self):
  517. return self.is_derived
  518. @property
  519. def Exists(self):
  520. return os.path.isdir(self.gitdir)
  521. @property
  522. def CurrentBranch(self):
  523. """Obtain the name of the currently checked out branch.
  524. The branch name omits the 'refs/heads/' prefix.
  525. None is returned if the project is on a detached HEAD.
  526. """
  527. b = self.work_git.GetHead()
  528. if b.startswith(R_HEADS):
  529. return b[len(R_HEADS):]
  530. return None
  531. def IsRebaseInProgress(self):
  532. w = self.worktree
  533. g = os.path.join(w, '.git')
  534. return os.path.exists(os.path.join(g, 'rebase-apply')) \
  535. or os.path.exists(os.path.join(g, 'rebase-merge')) \
  536. or os.path.exists(os.path.join(w, '.dotest'))
  537. def IsDirty(self, consider_untracked=True):
  538. """Is the working directory modified in some way?
  539. """
  540. self.work_git.update_index('-q',
  541. '--unmerged',
  542. '--ignore-missing',
  543. '--refresh')
  544. if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
  545. return True
  546. if self.work_git.DiffZ('diff-files'):
  547. return True
  548. if consider_untracked and self.work_git.LsOthers():
  549. return True
  550. return False
  551. _userident_name = None
  552. _userident_email = None
  553. @property
  554. def UserName(self):
  555. """Obtain the user's personal name.
  556. """
  557. if self._userident_name is None:
  558. self._LoadUserIdentity()
  559. return self._userident_name
  560. @property
  561. def UserEmail(self):
  562. """Obtain the user's email address. This is very likely
  563. to be their Gerrit login.
  564. """
  565. if self._userident_email is None:
  566. self._LoadUserIdentity()
  567. return self._userident_email
  568. def _LoadUserIdentity(self):
  569. u = self.bare_git.var('GIT_COMMITTER_IDENT')
  570. m = re.compile("^(.*) <([^>]*)> ").match(u)
  571. if m:
  572. self._userident_name = m.group(1)
  573. self._userident_email = m.group(2)
  574. else:
  575. self._userident_name = ''
  576. self._userident_email = ''
  577. def GetRemote(self, name):
  578. """Get the configuration for a single remote.
  579. """
  580. return self.config.GetRemote(name)
  581. def GetBranch(self, name):
  582. """Get the configuration for a single branch.
  583. """
  584. return self.config.GetBranch(name)
  585. def GetBranches(self):
  586. """Get all existing local branches.
  587. """
  588. current = self.CurrentBranch
  589. all_refs = self._allrefs
  590. heads = {}
  591. for name, ref_id in all_refs.items():
  592. if name.startswith(R_HEADS):
  593. name = name[len(R_HEADS):]
  594. b = self.GetBranch(name)
  595. b.current = name == current
  596. b.published = None
  597. b.revision = ref_id
  598. heads[name] = b
  599. for name, ref_id in all_refs.items():
  600. if name.startswith(R_PUB):
  601. name = name[len(R_PUB):]
  602. b = heads.get(name)
  603. if b:
  604. b.published = ref_id
  605. return heads
  606. def MatchesGroups(self, manifest_groups):
  607. """Returns true if the manifest groups specified at init should cause
  608. this project to be synced.
  609. Prefixing a manifest group with "-" inverts the meaning of a group.
  610. All projects are implicitly labelled with "all".
  611. labels are resolved in order. In the example case of
  612. project_groups: "all,group1,group2"
  613. manifest_groups: "-group1,group2"
  614. the project will be matched.
  615. The special manifest group "default" will match any project that
  616. does not have the special project group "notdefault"
  617. """
  618. expanded_manifest_groups = manifest_groups or ['default']
  619. expanded_project_groups = ['all'] + (self.groups or [])
  620. if not 'notdefault' in expanded_project_groups:
  621. expanded_project_groups += ['default']
  622. matched = False
  623. for group in expanded_manifest_groups:
  624. if group.startswith('-') and group[1:] in expanded_project_groups:
  625. matched = False
  626. elif group in expanded_project_groups:
  627. matched = True
  628. return matched
  629. ## Status Display ##
  630. def HasChanges(self):
  631. """Returns true if there are uncommitted changes.
  632. """
  633. self.work_git.update_index('-q',
  634. '--unmerged',
  635. '--ignore-missing',
  636. '--refresh')
  637. if self.IsRebaseInProgress():
  638. return True
  639. if self.work_git.DiffZ('diff-index', '--cached', HEAD):
  640. return True
  641. if self.work_git.DiffZ('diff-files'):
  642. return True
  643. if self.work_git.LsOthers():
  644. return True
  645. return False
  646. def PrintWorkTreeStatus(self, output_redir=None):
  647. """Prints the status of the repository to stdout.
  648. Args:
  649. output: If specified, redirect the output to this object.
  650. """
  651. if not os.path.isdir(self.worktree):
  652. if output_redir == None:
  653. output_redir = sys.stdout
  654. print(file=output_redir)
  655. print('project %s/' % self.relpath, file=output_redir)
  656. print(' missing (run "repo sync")', file=output_redir)
  657. return
  658. self.work_git.update_index('-q',
  659. '--unmerged',
  660. '--ignore-missing',
  661. '--refresh')
  662. rb = self.IsRebaseInProgress()
  663. di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
  664. df = self.work_git.DiffZ('diff-files')
  665. do = self.work_git.LsOthers()
  666. if not rb and not di and not df and not do and not self.CurrentBranch:
  667. return 'CLEAN'
  668. out = StatusColoring(self.config)
  669. if not output_redir == None:
  670. out.redirect(output_redir)
  671. out.project('project %-40s', self.relpath + '/')
  672. branch = self.CurrentBranch
  673. if branch is None:
  674. out.nobranch('(*** NO BRANCH ***)')
  675. else:
  676. out.branch('branch %s', branch)
  677. out.nl()
  678. if rb:
  679. out.important('prior sync failed; rebase still in progress')
  680. out.nl()
  681. paths = list()
  682. paths.extend(di.keys())
  683. paths.extend(df.keys())
  684. paths.extend(do)
  685. for p in sorted(set(paths)):
  686. try:
  687. i = di[p]
  688. except KeyError:
  689. i = None
  690. try:
  691. f = df[p]
  692. except KeyError:
  693. f = None
  694. if i:
  695. i_status = i.status.upper()
  696. else:
  697. i_status = '-'
  698. if f:
  699. f_status = f.status.lower()
  700. else:
  701. f_status = '-'
  702. if i and i.src_path:
  703. line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
  704. i.src_path, p, i.level)
  705. else:
  706. line = ' %s%s\t%s' % (i_status, f_status, p)
  707. if i and not f:
  708. out.added('%s', line)
  709. elif (i and f) or (not i and f):
  710. out.changed('%s', line)
  711. elif not i and not f:
  712. out.untracked('%s', line)
  713. else:
  714. out.write('%s', line)
  715. out.nl()
  716. return 'DIRTY'
  717. def PrintWorkTreeDiff(self, absolute_paths=False):
  718. """Prints the status of the repository to stdout.
  719. """
  720. out = DiffColoring(self.config)
  721. cmd = ['diff']
  722. if out.is_on:
  723. cmd.append('--color')
  724. cmd.append(HEAD)
  725. if absolute_paths:
  726. cmd.append('--src-prefix=a/%s/' % self.relpath)
  727. cmd.append('--dst-prefix=b/%s/' % self.relpath)
  728. cmd.append('--')
  729. p = GitCommand(self,
  730. cmd,
  731. capture_stdout = True,
  732. capture_stderr = True)
  733. has_diff = False
  734. for line in p.process.stdout:
  735. if not has_diff:
  736. out.nl()
  737. out.project('project %s/' % self.relpath)
  738. out.nl()
  739. has_diff = True
  740. print(line[:-1])
  741. p.Wait()
  742. ## Publish / Upload ##
  743. def WasPublished(self, branch, all_refs=None):
  744. """Was the branch published (uploaded) for code review?
  745. If so, returns the SHA-1 hash of the last published
  746. state for the branch.
  747. """
  748. key = R_PUB + branch
  749. if all_refs is None:
  750. try:
  751. return self.bare_git.rev_parse(key)
  752. except GitError:
  753. return None
  754. else:
  755. try:
  756. return all_refs[key]
  757. except KeyError:
  758. return None
  759. def CleanPublishedCache(self, all_refs=None):
  760. """Prunes any stale published refs.
  761. """
  762. if all_refs is None:
  763. all_refs = self._allrefs
  764. heads = set()
  765. canrm = {}
  766. for name, ref_id in all_refs.items():
  767. if name.startswith(R_HEADS):
  768. heads.add(name)
  769. elif name.startswith(R_PUB):
  770. canrm[name] = ref_id
  771. for name, ref_id in canrm.items():
  772. n = name[len(R_PUB):]
  773. if R_HEADS + n not in heads:
  774. self.bare_git.DeleteRef(name, ref_id)
  775. def GetUploadableBranches(self, selected_branch=None):
  776. """List any branches which can be uploaded for review.
  777. """
  778. heads = {}
  779. pubed = {}
  780. for name, ref_id in self._allrefs.items():
  781. if name.startswith(R_HEADS):
  782. heads[name[len(R_HEADS):]] = ref_id
  783. elif name.startswith(R_PUB):
  784. pubed[name[len(R_PUB):]] = ref_id
  785. ready = []
  786. for branch, ref_id in heads.items():
  787. if branch in pubed and pubed[branch] == ref_id:
  788. continue
  789. if selected_branch and branch != selected_branch:
  790. continue
  791. rb = self.GetUploadableBranch(branch)
  792. if rb:
  793. ready.append(rb)
  794. return ready
  795. def GetUploadableBranch(self, branch_name):
  796. """Get a single uploadable branch, or None.
  797. """
  798. branch = self.GetBranch(branch_name)
  799. base = branch.LocalMerge
  800. if branch.LocalMerge:
  801. rb = ReviewableBranch(self, branch, base)
  802. if rb.commits:
  803. return rb
  804. return None
  805. def UploadForReview(self, branch=None,
  806. people=([],[]),
  807. auto_topic=False,
  808. draft=False,
  809. dest_branch=None):
  810. """Uploads the named branch for code review.
  811. """
  812. if branch is None:
  813. branch = self.CurrentBranch
  814. if branch is None:
  815. raise GitError('not currently on a branch')
  816. branch = self.GetBranch(branch)
  817. if not branch.LocalMerge:
  818. raise GitError('branch %s does not track a remote' % branch.name)
  819. if not branch.remote.review:
  820. raise GitError('remote %s has no review url' % branch.remote.name)
  821. if dest_branch is None:
  822. dest_branch = self.dest_branch
  823. if dest_branch is None:
  824. dest_branch = branch.merge
  825. if not dest_branch.startswith(R_HEADS):
  826. dest_branch = R_HEADS + dest_branch
  827. if not branch.remote.projectname:
  828. branch.remote.projectname = self.name
  829. branch.remote.Save()
  830. url = branch.remote.ReviewUrl(self.UserEmail)
  831. if url is None:
  832. raise UploadError('review not configured')
  833. cmd = ['push']
  834. if url.startswith('ssh://'):
  835. rp = ['gerrit receive-pack']
  836. for e in people[0]:
  837. rp.append('--reviewer=%s' % sq(e))
  838. for e in people[1]:
  839. rp.append('--cc=%s' % sq(e))
  840. cmd.append('--receive-pack=%s' % " ".join(rp))
  841. cmd.append(url)
  842. if dest_branch.startswith(R_HEADS):
  843. dest_branch = dest_branch[len(R_HEADS):]
  844. upload_type = 'for'
  845. if draft:
  846. upload_type = 'drafts'
  847. ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
  848. dest_branch)
  849. if auto_topic:
  850. ref_spec = ref_spec + '/' + branch.name
  851. if not url.startswith('ssh://'):
  852. rp = ['r=%s' % p for p in people[0]] + \
  853. ['cc=%s' % p for p in people[1]]
  854. if rp:
  855. ref_spec = ref_spec + '%' + ','.join(rp)
  856. cmd.append(ref_spec)
  857. if GitCommand(self, cmd, bare = True).Wait() != 0:
  858. raise UploadError('Upload failed')
  859. msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
  860. self.bare_git.UpdateRef(R_PUB + branch.name,
  861. R_HEADS + branch.name,
  862. message = msg)
  863. ## Sync ##
  864. def _ExtractArchive(self, tarpath, path=None):
  865. """Extract the given tar on its current location
  866. Args:
  867. - tarpath: The path to the actual tar file
  868. """
  869. try:
  870. with tarfile.open(tarpath, 'r') as tar:
  871. tar.extractall(path=path)
  872. return True
  873. except (IOError, tarfile.TarError) as e:
  874. print("error: Cannot extract archive %s: "
  875. "%s" % (tarpath, str(e)), file=sys.stderr)
  876. return False
  877. def Sync_NetworkHalf(self,
  878. quiet=False,
  879. is_new=None,
  880. current_branch_only=False,
  881. clone_bundle=True,
  882. no_tags=False,
  883. archive=False):
  884. """Perform only the network IO portion of the sync process.
  885. Local working directory/branch state is not affected.
  886. """
  887. if archive and not isinstance(self, MetaProject):
  888. if self.remote.url.startswith(('http://', 'https://')):
  889. print("error: %s: Cannot fetch archives from http/https "
  890. "remotes." % self.name, file=sys.stderr)
  891. return False
  892. name = self.relpath.replace('\\', '/')
  893. name = name.replace('/', '_')
  894. tarpath = '%s.tar' % name
  895. topdir = self.manifest.topdir
  896. try:
  897. self._FetchArchive(tarpath, cwd=topdir)
  898. except GitError as e:
  899. print('error: %s' % str(e), file=sys.stderr)
  900. return False
  901. # From now on, we only need absolute tarpath
  902. tarpath = os.path.join(topdir, tarpath)
  903. if not self._ExtractArchive(tarpath, path=topdir):
  904. return False
  905. try:
  906. os.remove(tarpath)
  907. except OSError as e:
  908. print("warn: Cannot remove archive %s: "
  909. "%s" % (tarpath, str(e)), file=sys.stderr)
  910. self._CopyAndLinkFiles()
  911. return True
  912. if is_new is None:
  913. is_new = not self.Exists
  914. if is_new:
  915. self._InitGitDir()
  916. else:
  917. self._UpdateHooks()
  918. self._InitRemote()
  919. if is_new:
  920. alt = os.path.join(self.gitdir, 'objects/info/alternates')
  921. try:
  922. fd = open(alt, 'rb')
  923. try:
  924. alt_dir = fd.readline().rstrip()
  925. finally:
  926. fd.close()
  927. except IOError:
  928. alt_dir = None
  929. else:
  930. alt_dir = None
  931. if clone_bundle \
  932. and alt_dir is None \
  933. and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
  934. is_new = False
  935. if not current_branch_only:
  936. if self.sync_c:
  937. current_branch_only = True
  938. elif not self.manifest._loaded:
  939. # Manifest cannot check defaults until it syncs.
  940. current_branch_only = False
  941. elif self.manifest.default.sync_c:
  942. current_branch_only = True
  943. has_sha1 = ID_RE.match(self.revisionExpr) and self._CheckForSha1()
  944. if (not has_sha1 #Need to fetch since we don't already have this revision
  945. and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
  946. current_branch_only=current_branch_only,
  947. no_tags=no_tags)):
  948. return False
  949. if self.worktree:
  950. self._InitMRef()
  951. else:
  952. self._InitMirrorHead()
  953. try:
  954. os.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
  955. except OSError:
  956. pass
  957. return True
  958. def PostRepoUpgrade(self):
  959. self._InitHooks()
  960. def _CopyAndLinkFiles(self):
  961. for copyfile in self.copyfiles:
  962. copyfile._Copy()
  963. for linkfile in self.linkfiles:
  964. linkfile._Link()
  965. def GetCommitRevisionId(self):
  966. """Get revisionId of a commit.
  967. Use this method instead of GetRevisionId to get the id of the commit rather
  968. than the id of the current git object (for example, a tag)
  969. """
  970. if not self.revisionExpr.startswith(R_TAGS):
  971. return self.GetRevisionId(self._allrefs)
  972. try:
  973. return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
  974. except GitError:
  975. raise ManifestInvalidRevisionError(
  976. 'revision %s in %s not found' % (self.revisionExpr,
  977. self.name))
  978. def GetRevisionId(self, all_refs=None):
  979. if self.revisionId:
  980. return self.revisionId
  981. rem = self.GetRemote(self.remote.name)
  982. rev = rem.ToLocal(self.revisionExpr)
  983. if all_refs is not None and rev in all_refs:
  984. return all_refs[rev]
  985. try:
  986. return self.bare_git.rev_parse('--verify', '%s^0' % rev)
  987. except GitError:
  988. raise ManifestInvalidRevisionError(
  989. 'revision %s in %s not found' % (self.revisionExpr,
  990. self.name))
  991. def Sync_LocalHalf(self, syncbuf):
  992. """Perform only the local IO portion of the sync process.
  993. Network access is not required.
  994. """
  995. self._InitWorkTree()
  996. all_refs = self.bare_ref.all
  997. self.CleanPublishedCache(all_refs)
  998. revid = self.GetRevisionId(all_refs)
  999. def _doff():
  1000. self._FastForward(revid)
  1001. self._CopyAndLinkFiles()
  1002. head = self.work_git.GetHead()
  1003. if head.startswith(R_HEADS):
  1004. branch = head[len(R_HEADS):]
  1005. try:
  1006. head = all_refs[head]
  1007. except KeyError:
  1008. head = None
  1009. else:
  1010. branch = None
  1011. if branch is None or syncbuf.detach_head:
  1012. # Currently on a detached HEAD. The user is assumed to
  1013. # not have any local modifications worth worrying about.
  1014. #
  1015. if self.IsRebaseInProgress():
  1016. syncbuf.fail(self, _PriorSyncFailedError())
  1017. return
  1018. if head == revid:
  1019. # No changes; don't do anything further.
  1020. # Except if the head needs to be detached
  1021. #
  1022. if not syncbuf.detach_head:
  1023. return
  1024. else:
  1025. lost = self._revlist(not_rev(revid), HEAD)
  1026. if lost:
  1027. syncbuf.info(self, "discarding %d commits", len(lost))
  1028. try:
  1029. self._Checkout(revid, quiet=True)
  1030. except GitError as e:
  1031. syncbuf.fail(self, e)
  1032. return
  1033. self._CopyAndLinkFiles()
  1034. return
  1035. if head == revid:
  1036. # No changes; don't do anything further.
  1037. #
  1038. return
  1039. branch = self.GetBranch(branch)
  1040. if not branch.LocalMerge:
  1041. # The current branch has no tracking configuration.
  1042. # Jump off it to a detached HEAD.
  1043. #
  1044. syncbuf.info(self,
  1045. "leaving %s; does not track upstream",
  1046. branch.name)
  1047. try:
  1048. self._Checkout(revid, quiet=True)
  1049. except GitError as e:
  1050. syncbuf.fail(self, e)
  1051. return
  1052. self._CopyAndLinkFiles()
  1053. return
  1054. upstream_gain = self._revlist(not_rev(HEAD), revid)
  1055. pub = self.WasPublished(branch.name, all_refs)
  1056. if pub:
  1057. not_merged = self._revlist(not_rev(revid), pub)
  1058. if not_merged:
  1059. if upstream_gain:
  1060. # The user has published this branch and some of those
  1061. # commits are not yet merged upstream. We do not want
  1062. # to rewrite the published commits so we punt.
  1063. #
  1064. syncbuf.fail(self,
  1065. "branch %s is published (but not merged) and is now %d commits behind"
  1066. % (branch.name, len(upstream_gain)))
  1067. return
  1068. elif pub == head:
  1069. # All published commits are merged, and thus we are a
  1070. # strict subset. We can fast-forward safely.
  1071. #
  1072. syncbuf.later1(self, _doff)
  1073. return
  1074. # Examine the local commits not in the remote. Find the
  1075. # last one attributed to this user, if any.
  1076. #
  1077. local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
  1078. last_mine = None
  1079. cnt_mine = 0
  1080. for commit in local_changes:
  1081. commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
  1082. if committer_email == self.UserEmail:
  1083. last_mine = commit_id
  1084. cnt_mine += 1
  1085. if not upstream_gain and cnt_mine == len(local_changes):
  1086. return
  1087. if self.IsDirty(consider_untracked=False):
  1088. syncbuf.fail(self, _DirtyError())
  1089. return
  1090. # If the upstream switched on us, warn the user.
  1091. #
  1092. if branch.merge != self.revisionExpr:
  1093. if branch.merge and self.revisionExpr:
  1094. syncbuf.info(self,
  1095. 'manifest switched %s...%s',
  1096. branch.merge,
  1097. self.revisionExpr)
  1098. elif branch.merge:
  1099. syncbuf.info(self,
  1100. 'manifest no longer tracks %s',
  1101. branch.merge)
  1102. if cnt_mine < len(local_changes):
  1103. # Upstream rebased. Not everything in HEAD
  1104. # was created by this user.
  1105. #
  1106. syncbuf.info(self,
  1107. "discarding %d commits removed from upstream",
  1108. len(local_changes) - cnt_mine)
  1109. branch.remote = self.GetRemote(self.remote.name)
  1110. if not ID_RE.match(self.revisionExpr):
  1111. # in case of manifest sync the revisionExpr might be a SHA1
  1112. branch.merge = self.revisionExpr
  1113. branch.Save()
  1114. if cnt_mine > 0 and self.rebase:
  1115. def _dorebase():
  1116. self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
  1117. self._CopyAndLinkFiles()
  1118. syncbuf.later2(self, _dorebase)
  1119. elif local_changes:
  1120. try:
  1121. self._ResetHard(revid)
  1122. self._CopyAndLinkFiles()
  1123. except GitError as e:
  1124. syncbuf.fail(self, e)
  1125. return
  1126. else:
  1127. syncbuf.later1(self, _doff)
  1128. def AddCopyFile(self, src, dest, absdest):
  1129. # dest should already be an absolute path, but src is project relative
  1130. # make src an absolute path
  1131. abssrc = os.path.join(self.worktree, src)
  1132. self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
  1133. def AddLinkFile(self, src, dest, absdest):
  1134. # dest should already be an absolute path, but src is project relative
  1135. # make src an absolute path
  1136. abssrc = os.path.join(self.worktree, src)
  1137. self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
  1138. def AddAnnotation(self, name, value, keep):
  1139. self.annotations.append(_Annotation(name, value, keep))
  1140. def DownloadPatchSet(self, change_id, patch_id):
  1141. """Download a single patch set of a single change to FETCH_HEAD.
  1142. """
  1143. remote = self.GetRemote(self.remote.name)
  1144. cmd = ['fetch', remote.name]
  1145. cmd.append('refs/changes/%2.2d/%d/%d' \
  1146. % (change_id % 100, change_id, patch_id))
  1147. if GitCommand(self, cmd, bare=True).Wait() != 0:
  1148. return None
  1149. return DownloadedChange(self,
  1150. self.GetRevisionId(),
  1151. change_id,
  1152. patch_id,
  1153. self.bare_git.rev_parse('FETCH_HEAD'))
  1154. ## Branch Management ##
  1155. def StartBranch(self, name):
  1156. """Create a new branch off the manifest's revision.
  1157. """
  1158. head = self.work_git.GetHead()
  1159. if head == (R_HEADS + name):
  1160. return True
  1161. all_refs = self.bare_ref.all
  1162. if (R_HEADS + name) in all_refs:
  1163. return GitCommand(self,
  1164. ['checkout', name, '--'],
  1165. capture_stdout = True,
  1166. capture_stderr = True).Wait() == 0
  1167. branch = self.GetBranch(name)
  1168. branch.remote = self.GetRemote(self.remote.name)
  1169. branch.merge = self.revisionExpr
  1170. revid = self.GetRevisionId(all_refs)
  1171. if head.startswith(R_HEADS):
  1172. try:
  1173. head = all_refs[head]
  1174. except KeyError:
  1175. head = None
  1176. if revid and head and revid == head:
  1177. ref = os.path.join(self.gitdir, R_HEADS + name)
  1178. try:
  1179. os.makedirs(os.path.dirname(ref))
  1180. except OSError:
  1181. pass
  1182. _lwrite(ref, '%s\n' % revid)
  1183. _lwrite(os.path.join(self.worktree, '.git', HEAD),
  1184. 'ref: %s%s\n' % (R_HEADS, name))
  1185. branch.Save()
  1186. return True
  1187. if GitCommand(self,
  1188. ['checkout', '-b', branch.name, revid],
  1189. capture_stdout = True,
  1190. capture_stderr = True).Wait() == 0:
  1191. branch.Save()
  1192. return True
  1193. return False
  1194. def CheckoutBranch(self, name):
  1195. """Checkout a local topic branch.
  1196. Args:
  1197. name: The name of the branch to checkout.
  1198. Returns:
  1199. True if the checkout succeeded; False if it didn't; None if the branch
  1200. didn't exist.
  1201. """
  1202. rev = R_HEADS + name
  1203. head = self.work_git.GetHead()
  1204. if head == rev:
  1205. # Already on the branch
  1206. #
  1207. return True
  1208. all_refs = self.bare_ref.all
  1209. try:
  1210. revid = all_refs[rev]
  1211. except KeyError:
  1212. # Branch does not exist in this project
  1213. #
  1214. return None
  1215. if head.startswith(R_HEADS):
  1216. try:
  1217. head = all_refs[head]
  1218. except KeyError:
  1219. head = None
  1220. if head == revid:
  1221. # Same revision; just update HEAD to point to the new
  1222. # target branch, but otherwise take no other action.
  1223. #
  1224. _lwrite(os.path.join(self.worktree, '.git', HEAD),
  1225. 'ref: %s%s\n' % (R_HEADS, name))
  1226. return True
  1227. return GitCommand(self,
  1228. ['checkout', name, '--'],
  1229. capture_stdout = True,
  1230. capture_stderr = True).Wait() == 0
  1231. def AbandonBranch(self, name):
  1232. """Destroy a local topic branch.
  1233. Args:
  1234. name: The name of the branch to abandon.
  1235. Returns:
  1236. True if the abandon succeeded; False if it didn't; None if the branch
  1237. didn't exist.
  1238. """
  1239. rev = R_HEADS + name
  1240. all_refs = self.bare_ref.all
  1241. if rev not in all_refs:
  1242. # Doesn't exist
  1243. return None
  1244. head = self.work_git.GetHead()
  1245. if head == rev:
  1246. # We can't destroy the branch while we are sitting
  1247. # on it. Switch to a detached HEAD.
  1248. #
  1249. head = all_refs[head]
  1250. revid = self.GetRevisionId(all_refs)
  1251. if head == revid:
  1252. _lwrite(os.path.join(self.worktree, '.git', HEAD),
  1253. '%s\n' % revid)
  1254. else:
  1255. self._Checkout(revid, quiet=True)
  1256. return GitCommand(self,
  1257. ['branch', '-D', name],
  1258. capture_stdout = True,
  1259. capture_stderr = True).Wait() == 0
  1260. def PruneHeads(self):
  1261. """Prune any topic branches already merged into upstream.
  1262. """
  1263. cb = self.CurrentBranch
  1264. kill = []
  1265. left = self._allrefs
  1266. for name in left.keys():
  1267. if name.startswith(R_HEADS):
  1268. name = name[len(R_HEADS):]
  1269. if cb is None or name != cb:
  1270. kill.append(name)
  1271. rev = self.GetRevisionId(left)
  1272. if cb is not None \
  1273. and not self._revlist(HEAD + '...' + rev) \
  1274. and not self.IsDirty(consider_untracked = False):
  1275. self.work_git.DetachHead(HEAD)
  1276. kill.append(cb)
  1277. if kill:
  1278. old = self.bare_git.GetHead()
  1279. if old is None:
  1280. old = 'refs/heads/please_never_use_this_as_a_branch_name'
  1281. try:
  1282. self.bare_git.DetachHead(rev)
  1283. b = ['branch', '-d']
  1284. b.extend(kill)
  1285. b = GitCommand(self, b, bare=True,
  1286. capture_stdout=True,
  1287. capture_stderr=True)
  1288. b.Wait()
  1289. finally:
  1290. self.bare_git.SetHead(old)
  1291. left = self._allrefs
  1292. for branch in kill:
  1293. if (R_HEADS + branch) not in left:
  1294. self.CleanPublishedCache()
  1295. break
  1296. if cb and cb not in kill:
  1297. kill.append(cb)
  1298. kill.sort()
  1299. kept = []
  1300. for branch in kill:
  1301. if (R_HEADS + branch) in left:
  1302. branch = self.GetBranch(branch)
  1303. base = branch.LocalMerge
  1304. if not base:
  1305. base = rev
  1306. kept.append(ReviewableBranch(self, branch, base))
  1307. return kept
  1308. ## Submodule Management ##
  1309. def GetRegisteredSubprojects(self):
  1310. result = []
  1311. def rec(subprojects):
  1312. if not subprojects:
  1313. return
  1314. result.extend(subprojects)
  1315. for p in subprojects:
  1316. rec(p.subprojects)
  1317. rec(self.subprojects)
  1318. return result
  1319. def _GetSubmodules(self):
  1320. # Unfortunately we cannot call `git submodule status --recursive` here
  1321. # because the working tree might not exist yet, and it cannot be used
  1322. # without a working tree in its current implementation.
  1323. def get_submodules(gitdir, rev):
  1324. # Parse .gitmodules for submodule sub_paths and sub_urls
  1325. sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
  1326. if not sub_paths:
  1327. return []
  1328. # Run `git ls-tree` to read SHAs of submodule object, which happen to be
  1329. # revision of submodule repository
  1330. sub_revs = git_ls_tree(gitdir, rev, sub_paths)
  1331. submodules = []
  1332. for sub_path, sub_url in zip(sub_paths, sub_urls):
  1333. try:
  1334. sub_rev = sub_revs[sub_path]
  1335. except KeyError:
  1336. # Ignore non-exist submodules
  1337. continue
  1338. submodules.append((sub_rev, sub_path, sub_url))
  1339. return submodules
  1340. re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
  1341. re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
  1342. def parse_gitmodules(gitdir, rev):
  1343. cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
  1344. try:
  1345. p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
  1346. bare = True, gitdir = gitdir)
  1347. except GitError:
  1348. return [], []
  1349. if p.Wait() != 0:
  1350. return [], []
  1351. gitmodules_lines = []
  1352. fd, temp_gitmodules_path = tempfile.mkstemp()
  1353. try:
  1354. os.write(fd, p.stdout)
  1355. os.close(fd)
  1356. cmd = ['config', '--file', temp_gitmodules_path, '--list']
  1357. p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
  1358. bare = True, gitdir = gitdir)
  1359. if p.Wait() != 0:
  1360. return [], []
  1361. gitmodules_lines = p.stdout.split('\n')
  1362. except GitError:
  1363. return [], []
  1364. finally:
  1365. os.remove(temp_gitmodules_path)
  1366. names = set()
  1367. paths = {}
  1368. urls = {}
  1369. for line in gitmodules_lines:
  1370. if not line:
  1371. continue
  1372. m = re_path.match(line)
  1373. if m:
  1374. names.add(m.group(1))
  1375. paths[m.group(1)] = m.group(2)
  1376. continue
  1377. m = re_url.match(line)
  1378. if m:
  1379. names.add(m.group(1))
  1380. urls[m.group(1)] = m.group(2)
  1381. continue
  1382. names = sorted(names)
  1383. return ([paths.get(name, '') for name in names],
  1384. [urls.get(name, '') for name in names])
  1385. def git_ls_tree(gitdir, rev, paths):
  1386. cmd = ['ls-tree', rev, '--']
  1387. cmd.extend(paths)
  1388. try:
  1389. p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
  1390. bare = True, gitdir = gitdir)
  1391. except GitError:
  1392. return []
  1393. if p.Wait() != 0:
  1394. return []
  1395. objects = {}
  1396. for line in p.stdout.split('\n'):
  1397. if not line.strip():
  1398. continue
  1399. object_rev, object_path = line.split()[2:4]
  1400. objects[object_path] = object_rev
  1401. return objects
  1402. try:
  1403. rev = self.GetRevisionId()
  1404. except GitError:
  1405. return []
  1406. return get_submodules(self.gitdir, rev)
  1407. def GetDerivedSubprojects(self):
  1408. result = []
  1409. if not self.Exists:
  1410. # If git repo does not exist yet, querying its submodules will
  1411. # mess up its states; so return here.
  1412. return result
  1413. for rev, path, url in self._GetSubmodules():
  1414. name = self.manifest.GetSubprojectName(self, path)
  1415. relpath, worktree, gitdir, objdir = \
  1416. self.manifest.GetSubprojectPaths(self, name, path)
  1417. project = self.manifest.paths.get(relpath)
  1418. if project:
  1419. result.extend(project.GetDerivedSubprojects())
  1420. continue
  1421. remote = RemoteSpec(self.remote.name,
  1422. url = url,
  1423. review = self.remote.review,
  1424. revision = self.remote.revision)
  1425. subproject = Project(manifest = self.manifest,
  1426. name = name,
  1427. remote = remote,
  1428. gitdir = gitdir,
  1429. objdir = objdir,
  1430. worktree = worktree,
  1431. relpath = relpath,
  1432. revisionExpr = self.revisionExpr,
  1433. revisionId = rev,
  1434. rebase = self.rebase,
  1435. groups = self.groups,
  1436. sync_c = self.sync_c,
  1437. sync_s = self.sync_s,
  1438. parent = self,
  1439. is_derived = True)
  1440. result.append(subproject)
  1441. result.extend(subproject.GetDerivedSubprojects())
  1442. return result
  1443. ## Direct Git Commands ##
  1444. def _CheckForSha1(self):
  1445. try:
  1446. # if revision (sha or tag) is not present then following function
  1447. # throws an error.
  1448. self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
  1449. return True
  1450. except GitError:
  1451. # There is no such persistent revision. We have to fetch it.
  1452. return False
  1453. def _FetchArchive(self, tarpath, cwd=None):
  1454. cmd = ['archive', '-v', '-o', tarpath]
  1455. cmd.append('--remote=%s' % self.remote.url)
  1456. cmd.append('--prefix=%s/' % self.relpath)
  1457. cmd.append(self.revisionExpr)
  1458. command = GitCommand(self, cmd, cwd=cwd,
  1459. capture_stdout=True,
  1460. capture_stderr=True)
  1461. if command.Wait() != 0:
  1462. raise GitError('git archive %s: %s' % (self.name, command.stderr))
  1463. def _RemoteFetch(self, name=None,
  1464. current_branch_only=False,
  1465. initial=False,
  1466. quiet=False,
  1467. alt_dir=None,
  1468. no_tags=False):
  1469. is_sha1 = False
  1470. tag_name = None
  1471. depth = None
  1472. # The depth should not be used when fetching to a mirror because
  1473. # it will result in a shallow repository that cannot be cloned or
  1474. # fetched from.
  1475. if not self.manifest.IsMirror:
  1476. if self.clone_depth:
  1477. depth = self.clone_depth
  1478. else:
  1479. depth = self.manifest.manifestProject.config.GetString('repo.depth')
  1480. if depth:
  1481. current_branch_only = True
  1482. if current_branch_only:
  1483. if ID_RE.match(self.revisionExpr) is not None:
  1484. is_sha1 = True
  1485. elif self.revisionExpr.startswith(R_TAGS):
  1486. # this is a tag and its sha1 value should never change
  1487. tag_name = self.revisionExpr[len(R_TAGS):]
  1488. if is_sha1 or tag_name is not None:
  1489. if self._CheckForSha1():
  1490. return True
  1491. if is_sha1 and (not self.upstream or ID_RE.match(self.upstream)):
  1492. current_branch_only = False
  1493. if not name:
  1494. name = self.remote.name
  1495. ssh_proxy = False
  1496. remote = self.GetRemote(name)
  1497. if remote.PreConnectFetch():
  1498. ssh_proxy = True
  1499. if initial:
  1500. if alt_dir and 'objects' == os.path.basename(alt_dir):
  1501. ref_dir = os.path.dirname(alt_dir)
  1502. packed_refs = os.path.join(self.gitdir, 'packed-refs')
  1503. remote = self.GetRemote(name)
  1504. all_refs = self.bare_ref.all
  1505. ids = set(all_refs.values())
  1506. tmp = set()
  1507. for r, ref_id in GitRefs(ref_dir).all.items():
  1508. if r not in all_refs:
  1509. if r.startswith(R_TAGS) or remote.WritesTo(r):
  1510. all_refs[r] = ref_id
  1511. ids.add(ref_id)
  1512. continue
  1513. if ref_id in ids:
  1514. continue
  1515. r = 'refs/_alt/%s' % ref_id
  1516. all_refs[r] = ref_id
  1517. ids.add(ref_id)
  1518. tmp.add(r)
  1519. tmp_packed = ''
  1520. old_packed = ''
  1521. for r in sorted(all_refs):
  1522. line = '%s %s\n' % (all_refs[r], r)
  1523. tmp_packed += line
  1524. if r not in tmp:
  1525. old_packed += line
  1526. _lwrite(packed_refs, tmp_packed)
  1527. else:
  1528. alt_dir = None
  1529. cmd = ['fetch']
  1530. # The --depth option only affects the initial fetch; after that we'll do
  1531. # full fetches of changes.
  1532. if depth and initial:
  1533. cmd.append('--depth=%s' % depth)
  1534. if quiet:
  1535. cmd.append('--quiet')
  1536. if not self.worktree:
  1537. cmd.append('--update-head-ok')
  1538. cmd.append(name)
  1539. # If using depth then we should not get all the tags since they may
  1540. # be outside of the depth.
  1541. if no_tags or depth:
  1542. cmd.append('--no-tags')
  1543. else:
  1544. cmd.append('--tags')
  1545. if not current_branch_only:
  1546. # Fetch whole repo
  1547. cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
  1548. elif tag_name is not None:
  1549. cmd.append('tag')
  1550. cmd.append(tag_name)
  1551. else:
  1552. branch = self.revisionExpr
  1553. if is_sha1:
  1554. branch = self.upstream
  1555. if branch.startswith(R_HEADS):
  1556. branch = branch[len(R_HEADS):]
  1557. cmd.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
  1558. ok = False
  1559. for _i in range(2):
  1560. ret = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait()
  1561. if ret == 0:
  1562. ok = True
  1563. break
  1564. elif current_branch_only and is_sha1 and ret == 128:
  1565. # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
  1566. # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
  1567. # abort the optimization attempt and do a full sync.
  1568. break
  1569. time.sleep(random.randint(30, 45))
  1570. if initial:
  1571. # Ensure that some refs exist. Otherwise, we probably aren't looking
  1572. # at a real git repository and may have a bad url.
  1573. if not self.bare_ref.all:
  1574. ok = False
  1575. if alt_dir:
  1576. if old_packed != '':
  1577. _lwrite(packed_refs, old_packed)
  1578. else:
  1579. os.remove(packed_refs)
  1580. self.bare_git.pack_refs('--all', '--prune')
  1581. if is_sha1 and current_branch_only and self.upstream:
  1582. # We just synced the upstream given branch; verify we
  1583. # got what we wanted, else trigger a second run of all
  1584. # refs.
  1585. if not self._CheckForSha1():
  1586. return self._RemoteFetch(name=name, current_branch_only=False,
  1587. initial=False, quiet=quiet, alt_dir=alt_dir)
  1588. return ok
  1589. def _ApplyCloneBundle(self, initial=False, quiet=False):
  1590. if initial and (self.manifest.manifestProject.config.GetString('repo.depth') or self.clone_depth):
  1591. return False
  1592. remote = self.GetRemote(self.remote.name)
  1593. bundle_url = remote.url + '/clone.bundle'
  1594. bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
  1595. if GetSchemeFromUrl(bundle_url) not in (
  1596. 'http', 'https', 'persistent-http', 'persistent-https'):
  1597. return False
  1598. bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
  1599. bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
  1600. exist_dst = os.path.exists(bundle_dst)
  1601. exist_tmp = os.path.exists(bundle_tmp)
  1602. if not initial and not exist_dst and not exist_tmp:
  1603. return False
  1604. if not exist_dst:
  1605. exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
  1606. if not exist_dst:
  1607. return False
  1608. cmd = ['fetch']
  1609. if quiet:
  1610. cmd.append('--quiet')
  1611. if not self.worktree:
  1612. cmd.append('--update-head-ok')
  1613. cmd.append(bundle_dst)
  1614. for f in remote.fetch:
  1615. cmd.append(str(f))
  1616. cmd.append('refs/tags/*:refs/tags/*')
  1617. ok = GitCommand(self, cmd, bare=True).Wait() == 0
  1618. if os.path.exists(bundle_dst):
  1619. os.remove(bundle_dst)
  1620. if os.path.exists(bundle_tmp):
  1621. os.remove(bundle_tmp)
  1622. return ok
  1623. def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
  1624. if os.path.exists(dstPath):
  1625. os.remove(dstPath)
  1626. cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
  1627. if quiet:
  1628. cmd += ['--silent']
  1629. if os.path.exists(tmpPath):
  1630. size = os.stat(tmpPath).st_size
  1631. if size >= 1024:
  1632. cmd += ['--continue-at', '%d' % (size,)]
  1633. else:
  1634. os.remove(tmpPath)
  1635. if 'http_proxy' in os.environ and 'darwin' == sys.platform:
  1636. cmd += ['--proxy', os.environ['http_proxy']]
  1637. cookiefile = self._GetBundleCookieFile(srcUrl)
  1638. if cookiefile:
  1639. cmd += ['--cookie', cookiefile]
  1640. if srcUrl.startswith('persistent-'):
  1641. srcUrl = srcUrl[len('persistent-'):]
  1642. cmd += [srcUrl]
  1643. if IsTrace():
  1644. Trace('%s', ' '.join(cmd))
  1645. try:
  1646. proc = subprocess.Popen(cmd)
  1647. except OSError:
  1648. return False
  1649. curlret = proc.wait()
  1650. if curlret == 22:
  1651. # From curl man page:
  1652. # 22: HTTP page not retrieved. The requested url was not found or
  1653. # returned another error with the HTTP error code being 400 or above.
  1654. # This return code only appears if -f, --fail is used.
  1655. if not quiet:
  1656. print("Server does not provide clone.bundle; ignoring.",
  1657. file=sys.stderr)
  1658. return False
  1659. if os.path.exists(tmpPath):
  1660. if curlret == 0 and self._IsValidBundle(tmpPath):
  1661. os.rename(tmpPath, dstPath)
  1662. return True
  1663. else:
  1664. os.remove(tmpPath)
  1665. return False
  1666. else:
  1667. return False
  1668. def _IsValidBundle(self, path):
  1669. try:
  1670. with open(path) as f:
  1671. if f.read(16) == '# v2 git bundle\n':
  1672. return True
  1673. else:
  1674. print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
  1675. return False
  1676. except OSError:
  1677. return False
  1678. def _GetBundleCookieFile(self, url):
  1679. if url.startswith('persistent-'):
  1680. try:
  1681. p = subprocess.Popen(
  1682. ['git-remote-persistent-https', '-print_config', url],
  1683. stdin=subprocess.PIPE, stdout=subprocess.PIPE,
  1684. stderr=subprocess.PIPE)
  1685. p.stdin.close() # Tell subprocess it's ok to close.
  1686. prefix = 'http.cookiefile='
  1687. cookiefile = None
  1688. for line in p.stdout:
  1689. line = line.strip()
  1690. if line.startswith(prefix):
  1691. cookiefile = line[len(prefix):]
  1692. break
  1693. if p.wait():
  1694. err_msg = p.stderr.read()
  1695. if ' -print_config' in err_msg:
  1696. pass # Persistent proxy doesn't support -print_config.
  1697. else:
  1698. print(err_msg, file=sys.stderr)
  1699. if cookiefile:
  1700. return cookiefile
  1701. except OSError as e:
  1702. if e.errno == errno.ENOENT:
  1703. pass # No persistent proxy.
  1704. raise
  1705. return GitConfig.ForUser().GetString('http.cookiefile')
  1706. def _Checkout(self, rev, quiet=False):
  1707. cmd = ['checkout']
  1708. if quiet:
  1709. cmd.append('-q')
  1710. cmd.append(rev)
  1711. cmd.append('--')
  1712. if GitCommand(self, cmd).Wait() != 0:
  1713. if self._allrefs:
  1714. raise GitError('%s checkout %s ' % (self.name, rev))
  1715. def _CherryPick(self, rev, quiet=False):
  1716. cmd = ['cherry-pick']
  1717. cmd.append(rev)
  1718. cmd.append('--')
  1719. if GitCommand(self, cmd).Wait() != 0:
  1720. if self._allrefs:
  1721. raise GitError('%s cherry-pick %s ' % (self.name, rev))
  1722. def _Revert(self, rev, quiet=False):
  1723. cmd = ['revert']
  1724. cmd.append('--no-edit')
  1725. cmd.append(rev)
  1726. cmd.append('--')
  1727. if GitCommand(self, cmd).Wait() != 0:
  1728. if self._allrefs:
  1729. raise GitError('%s revert %s ' % (self.name, rev))
  1730. def _ResetHard(self, rev, quiet=True):
  1731. cmd = ['reset', '--hard']
  1732. if quiet:
  1733. cmd.append('-q')
  1734. cmd.append(rev)
  1735. if GitCommand(self, cmd).Wait() != 0:
  1736. raise GitError('%s reset --hard %s ' % (self.name, rev))
  1737. def _Rebase(self, upstream, onto = None):
  1738. cmd = ['rebase']
  1739. if onto is not None:
  1740. cmd.extend(['--onto', onto])
  1741. cmd.append(upstream)
  1742. if GitCommand(self, cmd).Wait() != 0:
  1743. raise GitError('%s rebase %s ' % (self.name, upstream))
  1744. def _FastForward(self, head, ffonly=False):
  1745. cmd = ['merge', head]
  1746. if ffonly:
  1747. cmd.append("--ff-only")
  1748. if GitCommand(self, cmd).Wait() != 0:
  1749. raise GitError('%s merge %s ' % (self.name, head))
  1750. def _InitGitDir(self, mirror_git=None):
  1751. if not os.path.exists(self.gitdir):
  1752. # Initialize the bare repository, which contains all of the objects.
  1753. if not os.path.exists(self.objdir):
  1754. os.makedirs(self.objdir)
  1755. self.bare_objdir.init()
  1756. # If we have a separate directory to hold refs, initialize it as well.
  1757. if self.objdir != self.gitdir:
  1758. os.makedirs(self.gitdir)
  1759. self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
  1760. copy_all=True)
  1761. mp = self.manifest.manifestProject
  1762. ref_dir = mp.config.GetString('repo.reference') or ''
  1763. if ref_dir or mirror_git:
  1764. if not mirror_git:
  1765. mirror_git = os.path.join(ref_dir, self.name + '.git')
  1766. repo_git = os.path.join(ref_dir, '.repo', 'projects',
  1767. self.relpath + '.git')
  1768. if os.path.exists(mirror_git):
  1769. ref_dir = mirror_git
  1770. elif os.path.exists(repo_git):
  1771. ref_dir = repo_git
  1772. else:
  1773. ref_dir = None
  1774. if ref_dir:
  1775. _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
  1776. os.path.join(ref_dir, 'objects') + '\n')
  1777. self._UpdateHooks()
  1778. m = self.manifest.manifestProject.config
  1779. for key in ['user.name', 'user.email']:
  1780. if m.Has(key, include_defaults = False):
  1781. self.config.SetString(key, m.GetString(key))
  1782. if self.manifest.IsMirror:
  1783. self.config.SetString('core.bare', 'true')
  1784. else:
  1785. self.config.SetString('core.bare', None)
  1786. def _UpdateHooks(self):
  1787. if os.path.exists(self.gitdir):
  1788. # Always recreate hooks since they can have been changed
  1789. # since the latest update.
  1790. hooks = self._gitdir_path('hooks')
  1791. try:
  1792. to_rm = os.listdir(hooks)
  1793. except OSError:
  1794. to_rm = []
  1795. for old_hook in to_rm:
  1796. os.remove(os.path.join(hooks, old_hook))
  1797. self._InitHooks()
  1798. def _InitHooks(self):
  1799. hooks = os.path.realpath(self._gitdir_path('hooks'))
  1800. if not os.path.exists(hooks):
  1801. os.makedirs(hooks)
  1802. for stock_hook in _ProjectHooks():
  1803. name = os.path.basename(stock_hook)
  1804. if name in ('commit-msg',) and not self.remote.review \
  1805. and not self is self.manifest.manifestProject:
  1806. # Don't install a Gerrit Code Review hook if this
  1807. # project does not appear to use it for reviews.
  1808. #
  1809. # Since the manifest project is one of those, but also
  1810. # managed through gerrit, it's excluded
  1811. continue
  1812. dst = os.path.join(hooks, name)
  1813. if os.path.islink(dst):
  1814. continue
  1815. if os.path.exists(dst):
  1816. if filecmp.cmp(stock_hook, dst, shallow=False):
  1817. os.remove(dst)
  1818. else:
  1819. _error("%s: Not replacing %s hook", self.relpath, name)
  1820. continue
  1821. try:
  1822. os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
  1823. except OSError as e:
  1824. if e.errno == errno.EPERM:
  1825. raise GitError('filesystem must support symlinks')
  1826. else:
  1827. raise
  1828. def _InitRemote(self):
  1829. if self.remote.url:
  1830. remote = self.GetRemote(self.remote.name)
  1831. remote.url = self.remote.url
  1832. remote.review = self.remote.review
  1833. remote.projectname = self.name
  1834. if self.worktree:
  1835. remote.ResetFetch(mirror=False)
  1836. else:
  1837. remote.ResetFetch(mirror=True)
  1838. remote.Save()
  1839. def _InitMRef(self):
  1840. if self.manifest.branch:
  1841. self._InitAnyMRef(R_M + self.manifest.branch)
  1842. def _InitMirrorHead(self):
  1843. self._InitAnyMRef(HEAD)
  1844. def _InitAnyMRef(self, ref):
  1845. cur = self.bare_ref.symref(ref)
  1846. if self.revisionId:
  1847. if cur != '' or self.bare_ref.get(ref) != self.revisionId:
  1848. msg = 'manifest set to %s' % self.revisionId
  1849. dst = self.revisionId + '^0'
  1850. self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
  1851. else:
  1852. remote = self.GetRemote(self.remote.name)
  1853. dst = remote.ToLocal(self.revisionExpr)
  1854. if cur != dst:
  1855. msg = 'manifest set to %s' % self.revisionExpr
  1856. self.bare_git.symbolic_ref('-m', msg, ref, dst)
  1857. def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
  1858. """Update |dotgit| to reference |gitdir|, using symlinks where possible.
  1859. Args:
  1860. gitdir: The bare git repository. Must already be initialized.
  1861. dotgit: The repository you would like to initialize.
  1862. share_refs: If true, |dotgit| will store its refs under |gitdir|.
  1863. Only one work tree can store refs under a given |gitdir|.
  1864. copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
  1865. This saves you the effort of initializing |dotgit| yourself.
  1866. """
  1867. # These objects can be shared between several working trees.
  1868. symlink_files = ['description', 'info']
  1869. symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
  1870. if share_refs:
  1871. # These objects can only be used by a single working tree.
  1872. symlink_files += ['config', 'packed-refs', 'shallow']
  1873. symlink_dirs += ['logs', 'refs']
  1874. to_symlink = symlink_files + symlink_dirs
  1875. to_copy = []
  1876. if copy_all:
  1877. to_copy = os.listdir(gitdir)
  1878. for name in set(to_copy).union(to_symlink):
  1879. try:
  1880. src = os.path.realpath(os.path.join(gitdir, name))
  1881. dst = os.path.realpath(os.path.join(dotgit, name))
  1882. if os.path.lexists(dst) and not os.path.islink(dst):
  1883. raise GitError('cannot overwrite a local work tree')
  1884. # If the source dir doesn't exist, create an empty dir.
  1885. if name in symlink_dirs and not os.path.lexists(src):
  1886. os.makedirs(src)
  1887. if name in to_symlink:
  1888. os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
  1889. elif copy_all and not os.path.islink(dst):
  1890. if os.path.isdir(src):
  1891. shutil.copytree(src, dst)
  1892. elif os.path.isfile(src):
  1893. shutil.copy(src, dst)
  1894. except OSError as e:
  1895. if e.errno == errno.EPERM:
  1896. raise GitError('filesystem must support symlinks')
  1897. else:
  1898. raise
  1899. def _InitWorkTree(self):
  1900. dotgit = os.path.join(self.worktree, '.git')
  1901. if not os.path.exists(dotgit):
  1902. os.makedirs(dotgit)
  1903. self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
  1904. copy_all=False)
  1905. _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
  1906. cmd = ['read-tree', '--reset', '-u']
  1907. cmd.append('-v')
  1908. cmd.append(HEAD)
  1909. if GitCommand(self, cmd).Wait() != 0:
  1910. raise GitError("cannot initialize work tree")
  1911. self._CopyAndLinkFiles()
  1912. def _gitdir_path(self, path):
  1913. return os.path.realpath(os.path.join(self.gitdir, path))
  1914. def _revlist(self, *args, **kw):
  1915. a = []
  1916. a.extend(args)
  1917. a.append('--')
  1918. return self.work_git.rev_list(*a, **kw)
  1919. @property
  1920. def _allrefs(self):
  1921. return self.bare_ref.all
  1922. def _getLogs(self, rev1, rev2, oneline=False, color=True):
  1923. """Get logs between two revisions of this project."""
  1924. comp = '..'
  1925. if rev1:
  1926. revs = [rev1]
  1927. if rev2:
  1928. revs.extend([comp, rev2])
  1929. cmd = ['log', ''.join(revs)]
  1930. out = DiffColoring(self.config)
  1931. if out.is_on and color:
  1932. cmd.append('--color')
  1933. if oneline:
  1934. cmd.append('--oneline')
  1935. try:
  1936. log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
  1937. if log.Wait() == 0:
  1938. return log.stdout
  1939. except GitError:
  1940. # worktree may not exist if groups changed for example. In that case,
  1941. # try in gitdir instead.
  1942. if not os.path.exists(self.worktree):
  1943. return self.bare_git.log(*cmd[1:])
  1944. else:
  1945. raise
  1946. return None
  1947. def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True):
  1948. """Get the list of logs from this revision to given revisionId"""
  1949. logs = {}
  1950. selfId = self.GetRevisionId(self._allrefs)
  1951. toId = toProject.GetRevisionId(toProject._allrefs)
  1952. logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color)
  1953. logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color)
  1954. return logs
  1955. class _GitGetByExec(object):
  1956. def __init__(self, project, bare, gitdir):
  1957. self._project = project
  1958. self._bare = bare
  1959. self._gitdir = gitdir
  1960. def LsOthers(self):
  1961. p = GitCommand(self._project,
  1962. ['ls-files',
  1963. '-z',
  1964. '--others',
  1965. '--exclude-standard'],
  1966. bare = False,
  1967. gitdir=self._gitdir,
  1968. capture_stdout = True,
  1969. capture_stderr = True)
  1970. if p.Wait() == 0:
  1971. out = p.stdout
  1972. if out:
  1973. return out[:-1].split('\0') # pylint: disable=W1401
  1974. # Backslash is not anomalous
  1975. return []
  1976. def DiffZ(self, name, *args):
  1977. cmd = [name]
  1978. cmd.append('-z')
  1979. cmd.extend(args)
  1980. p = GitCommand(self._project,
  1981. cmd,
  1982. gitdir=self._gitdir,
  1983. bare = False,
  1984. capture_stdout = True,
  1985. capture_stderr = True)
  1986. try:
  1987. out = p.process.stdout.read()
  1988. r = {}
  1989. if out:
  1990. out = iter(out[:-1].split('\0')) # pylint: disable=W1401
  1991. while out:
  1992. try:
  1993. info = out.next()
  1994. path = out.next()
  1995. except StopIteration:
  1996. break
  1997. class _Info(object):
  1998. def __init__(self, path, omode, nmode, oid, nid, state):
  1999. self.path = path
  2000. self.src_path = None
  2001. self.old_mode = omode
  2002. self.new_mode = nmode
  2003. self.old_id = oid
  2004. self.new_id = nid
  2005. if len(state) == 1:
  2006. self.status = state
  2007. self.level = None
  2008. else:
  2009. self.status = state[:1]
  2010. self.level = state[1:]
  2011. while self.level.startswith('0'):
  2012. self.level = self.level[1:]
  2013. info = info[1:].split(' ')
  2014. info = _Info(path, *info)
  2015. if info.status in ('R', 'C'):
  2016. info.src_path = info.path
  2017. info.path = out.next()
  2018. r[info.path] = info
  2019. return r
  2020. finally:
  2021. p.Wait()
  2022. def GetHead(self):
  2023. if self._bare:
  2024. path = os.path.join(self._project.gitdir, HEAD)
  2025. else:
  2026. path = os.path.join(self._project.worktree, '.git', HEAD)
  2027. try:
  2028. fd = open(path, 'rb')
  2029. except IOError as e:
  2030. raise NoManifestException(path, str(e))
  2031. try:
  2032. line = fd.read()
  2033. finally:
  2034. fd.close()
  2035. try:
  2036. line = line.decode()
  2037. except AttributeError:
  2038. pass
  2039. if line.startswith('ref: '):
  2040. return line[5:-1]
  2041. return line[:-1]
  2042. def SetHead(self, ref, message=None):
  2043. cmdv = []
  2044. if message is not None:
  2045. cmdv.extend(['-m', message])
  2046. cmdv.append(HEAD)
  2047. cmdv.append(ref)
  2048. self.symbolic_ref(*cmdv)
  2049. def DetachHead(self, new, message=None):
  2050. cmdv = ['--no-deref']
  2051. if message is not None:
  2052. cmdv.extend(['-m', message])
  2053. cmdv.append(HEAD)
  2054. cmdv.append(new)
  2055. self.update_ref(*cmdv)
  2056. def UpdateRef(self, name, new, old=None,
  2057. message=None,
  2058. detach=False):
  2059. cmdv = []
  2060. if message is not None:
  2061. cmdv.extend(['-m', message])
  2062. if detach:
  2063. cmdv.append('--no-deref')
  2064. cmdv.append(name)
  2065. cmdv.append(new)
  2066. if old is not None:
  2067. cmdv.append(old)
  2068. self.update_ref(*cmdv)
  2069. def DeleteRef(self, name, old=None):
  2070. if not old:
  2071. old = self.rev_parse(name)
  2072. self.update_ref('-d', name, old)
  2073. self._project.bare_ref.deleted(name)
  2074. def rev_list(self, *args, **kw):
  2075. if 'format' in kw:
  2076. cmdv = ['log', '--pretty=format:%s' % kw['format']]
  2077. else:
  2078. cmdv = ['rev-list']
  2079. cmdv.extend(args)
  2080. p = GitCommand(self._project,
  2081. cmdv,
  2082. bare = self._bare,
  2083. gitdir=self._gitdir,
  2084. capture_stdout = True,
  2085. capture_stderr = True)
  2086. r = []
  2087. for line in p.process.stdout:
  2088. if line[-1] == '\n':
  2089. line = line[:-1]
  2090. r.append(line)
  2091. if p.Wait() != 0:
  2092. raise GitError('%s rev-list %s: %s' % (
  2093. self._project.name,
  2094. str(args),
  2095. p.stderr))
  2096. return r
  2097. def __getattr__(self, name):
  2098. """Allow arbitrary git commands using pythonic syntax.
  2099. This allows you to do things like:
  2100. git_obj.rev_parse('HEAD')
  2101. Since we don't have a 'rev_parse' method defined, the __getattr__ will
  2102. run. We'll replace the '_' with a '-' and try to run a git command.
  2103. Any other positional arguments will be passed to the git command, and the
  2104. following keyword arguments are supported:
  2105. config: An optional dict of git config options to be passed with '-c'.
  2106. Args:
  2107. name: The name of the git command to call. Any '_' characters will
  2108. be replaced with '-'.
  2109. Returns:
  2110. A callable object that will try to call git with the named command.
  2111. """
  2112. name = name.replace('_', '-')
  2113. def runner(*args, **kwargs):
  2114. cmdv = []
  2115. config = kwargs.pop('config', None)
  2116. for k in kwargs:
  2117. raise TypeError('%s() got an unexpected keyword argument %r'
  2118. % (name, k))
  2119. if config is not None:
  2120. if not git_require((1, 7, 2)):
  2121. raise ValueError('cannot set config on command line for %s()'
  2122. % name)
  2123. for k, v in config.items():
  2124. cmdv.append('-c')
  2125. cmdv.append('%s=%s' % (k, v))
  2126. cmdv.append(name)
  2127. cmdv.extend(args)
  2128. p = GitCommand(self._project,
  2129. cmdv,
  2130. bare = self._bare,
  2131. gitdir=self._gitdir,
  2132. capture_stdout = True,
  2133. capture_stderr = True)
  2134. if p.Wait() != 0:
  2135. raise GitError('%s %s: %s' % (
  2136. self._project.name,
  2137. name,
  2138. p.stderr))
  2139. r = p.stdout
  2140. try:
  2141. r = r.decode('utf-8')
  2142. except AttributeError:
  2143. pass
  2144. if r.endswith('\n') and r.index('\n') == len(r) - 1:
  2145. return r[:-1]
  2146. return r
  2147. return runner
  2148. class _PriorSyncFailedError(Exception):
  2149. def __str__(self):
  2150. return 'prior sync failed; rebase still in progress'
  2151. class _DirtyError(Exception):
  2152. def __str__(self):
  2153. return 'contains uncommitted changes'
  2154. class _InfoMessage(object):
  2155. def __init__(self, project, text):
  2156. self.project = project
  2157. self.text = text
  2158. def Print(self, syncbuf):
  2159. syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
  2160. syncbuf.out.nl()
  2161. class _Failure(object):
  2162. def __init__(self, project, why):
  2163. self.project = project
  2164. self.why = why
  2165. def Print(self, syncbuf):
  2166. syncbuf.out.fail('error: %s/: %s',
  2167. self.project.relpath,
  2168. str(self.why))
  2169. syncbuf.out.nl()
  2170. class _Later(object):
  2171. def __init__(self, project, action):
  2172. self.project = project
  2173. self.action = action
  2174. def Run(self, syncbuf):
  2175. out = syncbuf.out
  2176. out.project('project %s/', self.project.relpath)
  2177. out.nl()
  2178. try:
  2179. self.action()
  2180. out.nl()
  2181. return True
  2182. except GitError:
  2183. out.nl()
  2184. return False
  2185. class _SyncColoring(Coloring):
  2186. def __init__(self, config):
  2187. Coloring.__init__(self, config, 'reposync')
  2188. self.project = self.printer('header', attr = 'bold')
  2189. self.info = self.printer('info')
  2190. self.fail = self.printer('fail', fg='red')
  2191. class SyncBuffer(object):
  2192. def __init__(self, config, detach_head=False):
  2193. self._messages = []
  2194. self._failures = []
  2195. self._later_queue1 = []
  2196. self._later_queue2 = []
  2197. self.out = _SyncColoring(config)
  2198. self.out.redirect(sys.stderr)
  2199. self.detach_head = detach_head
  2200. self.clean = True
  2201. def info(self, project, fmt, *args):
  2202. self._messages.append(_InfoMessage(project, fmt % args))
  2203. def fail(self, project, err=None):
  2204. self._failures.append(_Failure(project, err))
  2205. self.clean = False
  2206. def later1(self, project, what):
  2207. self._later_queue1.append(_Later(project, what))
  2208. def later2(self, project, what):
  2209. self._later_queue2.append(_Later(project, what))
  2210. def Finish(self):
  2211. self._PrintMessages()
  2212. self._RunLater()
  2213. self._PrintMessages()
  2214. return self.clean
  2215. def _RunLater(self):
  2216. for q in ['_later_queue1', '_later_queue2']:
  2217. if not self._RunQueue(q):
  2218. return
  2219. def _RunQueue(self, queue):
  2220. for m in getattr(self, queue):
  2221. if not m.Run(self):
  2222. self.clean = False
  2223. return False
  2224. setattr(self, queue, [])
  2225. return True
  2226. def _PrintMessages(self):
  2227. for m in self._messages:
  2228. m.Print(self)
  2229. for m in self._failures:
  2230. m.Print(self)
  2231. self._messages = []
  2232. self._failures = []
  2233. class MetaProject(Project):
  2234. """A special project housed under .repo.
  2235. """
  2236. def __init__(self, manifest, name, gitdir, worktree):
  2237. Project.__init__(self,
  2238. manifest = manifest,
  2239. name = name,
  2240. gitdir = gitdir,
  2241. objdir = gitdir,
  2242. worktree = worktree,
  2243. remote = RemoteSpec('origin'),
  2244. relpath = '.repo/%s' % name,
  2245. revisionExpr = 'refs/heads/master',
  2246. revisionId = None,
  2247. groups = None)
  2248. def PreSync(self):
  2249. if self.Exists:
  2250. cb = self.CurrentBranch
  2251. if cb:
  2252. base = self.GetBranch(cb).merge
  2253. if base:
  2254. self.revisionExpr = base
  2255. self.revisionId = None
  2256. def MetaBranchSwitch(self, target):
  2257. """ Prepare MetaProject for manifest branch switch
  2258. """
  2259. # detach and delete manifest branch, allowing a new
  2260. # branch to take over
  2261. syncbuf = SyncBuffer(self.config, detach_head = True)
  2262. self.Sync_LocalHalf(syncbuf)
  2263. syncbuf.Finish()
  2264. return GitCommand(self,
  2265. ['update-ref', '-d', 'refs/heads/default'],
  2266. capture_stdout = True,
  2267. capture_stderr = True).Wait() == 0
  2268. @property
  2269. def LastFetch(self):
  2270. try:
  2271. fh = os.path.join(self.gitdir, 'FETCH_HEAD')
  2272. return os.path.getmtime(fh)
  2273. except OSError:
  2274. return 0
  2275. @property
  2276. def HasChanges(self):
  2277. """Has the remote received new commits not yet checked out?
  2278. """
  2279. if not self.remote or not self.revisionExpr:
  2280. return False
  2281. all_refs = self.bare_ref.all
  2282. revid = self.GetRevisionId(all_refs)
  2283. head = self.work_git.GetHead()
  2284. if head.startswith(R_HEADS):
  2285. try:
  2286. head = all_refs[head]
  2287. except KeyError:
  2288. head = None
  2289. if revid == head:
  2290. return False
  2291. elif self._revlist(not_rev(HEAD), revid):
  2292. return True
  2293. return False