aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bot_update: refactor giant git_checkout into two.

Browse Source 
Change-Id: I2c023a538bfe62955238b39034ed7460286789e5
Reviewed-on: https://chromium-review.googlesource.com/771454
Reviewed-by: Jao-ke Chin-Lee <jchinlee@chromium.org>
Commit-Queue: Andrii Shyshkalov <tandrii@chromium.org>
changes/54/771454/2
Andrii Shyshkalov 8 years ago committed by Commit Bot
parent 3d50d9c5b3
commit 1f620f0550
1 changed files with 91 additions and 85 deletions
Show Stats Download Patch File Download Diff File

176
recipes/recipe_modules/bot_update/resources/bot_update.py
Unescape Escape View File

@ -561,8 +561,8 @@ def _maybe_break_locks(checkout_path):
raise
def git_checkout(solutions, revisions, shallow, refs, git_cache_dir,
cleanup_dir):
def git_checkouts(solutions, revisions, shallow, refs, git_cache_dir,
cleanup_dir):
build_dir = os.getcwd()
# Before we do anything, break all git_cache locks.
if path.isdir(git_cache_dir):
@ -574,92 +574,98 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir,
raise Exception('%s exists after cache unlock' % filename)
first_solution = True
for sln in solutions:
# Just in case we're hitting a different git server than the one from
# which the target revision was polled, we retry some.
done = False
# One minute (5 tries with exp. backoff). We retry at least once regardless
# of deadline in case initial fetch takes longer than the deadline but does
# not contain the required revision.
deadline = time.time() + 60
tries = 0
while not done:
name = sln['name']
url = sln['url']
if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
# Experiments show there's little to be gained from
# a shallow clone of src.
shallow = False
sln_dir = path.join(build_dir, name)
s = ['--shallow'] if shallow else []
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
'--cache-dir', git_cache_dir] + s + [url])
for ref in refs:
populate_cmd.extend(['--ref', ref])
git(*populate_cmd)
mirror_dir = git(
'cache', 'exists', '--quiet',
'--cache-dir', git_cache_dir, url).strip()
clone_cmd = (
'clone', '--no-checkout', '--local', '--shared', mirror_dir, sln_dir)
_git_checkout(sln, build_dir, revisions, shallow, refs, git_cache_dir,
cleanup_dir)
if first_solution:
git_ref = git('log', '--format=%H', '--max-count=1',
cwd=path.join(build_dir, sln['name'])
).strip()
first_solution = False
return git_ref
try:
# If repo deletion was aborted midway, it may have left .git in broken
# state.
if path.exists(sln_dir) and is_broken_repo_dir(sln_dir):
print 'Git repo %s appears to be broken, removing it' % sln_dir
remove(sln_dir, cleanup_dir)
# Use "tries=1", since we retry manually in this loop.
if not path.isdir(sln_dir):
git(*clone_cmd)
else:
git('remote', 'set-url', 'origin', mirror_dir, cwd=sln_dir)
git('fetch', 'origin', cwd=sln_dir)
for ref in refs:
refspec = '%s:%s' % (ref, ref.lstrip('+'))
git('fetch', 'origin', refspec, cwd=sln_dir)
# Windows sometimes has trouble deleting files.
# This can make git commands that rely on locks fail.
# Try a few times in case Windows has trouble again (and again).
if sys.platform.startswith('win'):
tries = 3
while tries:
try:
_maybe_break_locks(sln_dir)
break
except Exception:
tries -= 1
revision = get_target_revision(name, url, revisions) or 'HEAD'
force_revision(sln_dir, revision)
done = True
except SubprocessFailed as e:
# Exited abnormally, theres probably something wrong.
print 'Something failed: %s.' % str(e)
# Only kick in deadline after trying once, in case the revision hasn't
# yet propagated.
if tries >= 1 and time.time() > deadline:
overrun = time.time() - deadline
print 'Ran %s seconds past deadline. Aborting.' % overrun
raise
# Lets wipe the checkout and try again.
tries += 1
sleep_secs = 2**tries
print 'waiting %s seconds and trying again...' % sleep_secs
time.sleep(sleep_secs)
def _git_checkout(sln, build_dir, revisions, shallow, refs, git_cache_dir,
cleanup_dir):
# Just in case we're hitting a different git server than the one from
# which the target revision was polled, we retry some.
done = False
# One minute (5 tries with exp. backoff). We retry at least once regardless
# of deadline in case initial fetch takes longer than the deadline but does
# not contain the required revision.
deadline = time.time() + 60
tries = 0
while not done:
name = sln['name']
url = sln['url']
if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
# Experiments show there's little to be gained from
# a shallow clone of src.
shallow = False
sln_dir = path.join(build_dir, name)
s = ['--shallow'] if shallow else []
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
'--cache-dir', git_cache_dir] + s + [url])
for ref in refs:
populate_cmd.extend(['--ref', ref])
git(*populate_cmd)
mirror_dir = git(
'cache', 'exists', '--quiet',
'--cache-dir', git_cache_dir, url).strip()
clone_cmd = (
'clone', '--no-checkout', '--local', '--shared', mirror_dir, sln_dir)
try:
# If repo deletion was aborted midway, it may have left .git in broken
# state.
if path.exists(sln_dir) and is_broken_repo_dir(sln_dir):
print 'Git repo %s appears to be broken, removing it' % sln_dir
remove(sln_dir, cleanup_dir)
git('clean', '-dff', cwd=sln_dir)
# Use "tries=1", since we retry manually in this loop.
if not path.isdir(sln_dir):
git(*clone_cmd)
else:
git('remote', 'set-url', 'origin', mirror_dir, cwd=sln_dir)
git('fetch', 'origin', cwd=sln_dir)
for ref in refs:
refspec = '%s:%s' % (ref, ref.lstrip('+'))
git('fetch', 'origin', refspec, cwd=sln_dir)
# Windows sometimes has trouble deleting files.
# This can make git commands that rely on locks fail.
# Try a few times in case Windows has trouble again (and again).
if sys.platform.startswith('win'):
tries = 3
while tries:
try:
_maybe_break_locks(sln_dir)
break
except Exception:
tries -= 1
revision = get_target_revision(name, url, revisions) or 'HEAD'
force_revision(sln_dir, revision)
done = True
except SubprocessFailed as e:
# Exited abnormally, theres probably something wrong.
print 'Something failed: %s.' % str(e)
# Only kick in deadline after trying once, in case the revision hasn't
# yet propagated.
if tries >= 1 and time.time() > deadline:
overrun = time.time() - deadline
print 'Ran %s seconds past deadline. Aborting.' % overrun
raise
if first_solution:
git_ref = git('log', '--format=%H', '--max-count=1',
cwd=sln_dir).strip()
first_solution = False
return git_ref
# Lets wipe the checkout and try again.
tries += 1
sleep_secs = 2**tries
print 'waiting %s seconds and trying again...' % sleep_secs
time.sleep(sleep_secs)
remove(sln_dir, cleanup_dir)
git('clean', '-dff', cwd=sln_dir)
def _download(url):
@ -833,8 +839,8 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
# invoking DEPS.
print 'Fetching Git checkout'
git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir,
cleanup_dir)
git_ref = git_checkouts(solutions, revisions, shallow, refs, git_cache_dir,
cleanup_dir)
print '===Processing patch solutions==='
already_patched = []

Write Preview
Loading…
Cancel
Save