aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revamped terminal output for update.

Browse Source 
Features:

- Non-verbose output is now limited to a one-line progress
indicator.

- Verbose output is now collated per subprocess.  As soon as a
subprocess finishes, its full output is dumped to terminal.

- Verbose output is prefixed with timestamps representing elapsed
time since the beginning of the gclient invocation.

- git progress indicators ("Receiving objects", etc.) are limited to
one line every 10 seconds.

- In both verbose and non-verbose mode, if a failure occurs, the
full output of the failed update operation is dumped to terminal
just before exit.

- In the event that updates are progressing, but slowly,
"Still working" messages will be printed periodically, to pacify
users and buildbots.

BUG=
R=hinoka@google.com

Review URL: https://codereview.chromium.org/227163002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@262500 0039d316-1c4b-4281-b951-d872f2087c98
experimental/szager/collated-output
szager@chromium.org 11 years ago
parent c1c9c4f882
commit fe0d1902b3
7 changed files with 303 additions and 189 deletions
Show Stats Download Patch File Download Diff File

29
gclient.py
Unescape Escape View File

@ -444,7 +444,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
parent_url = self.parent.parsed_url
if isinstance(parent_url, self.FileImpl):
parent_url = parent_url.file_location
scm = gclient_scm.CreateSCM(parent_url, self.root.root_dir, None)
scm = gclient_scm.CreateSCM(
parent_url, self.root.root_dir, None, self.outbuf)
parsed_url = scm.FullUrlForRelativeUrl(url)
else:
parsed_url = url
@ -657,7 +658,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
# pylint: disable=E1103
options.revision = parsed_url.GetRevision()
self._used_scm = gclient_scm.SVNWrapper(
parsed_url.GetPath(), self.root.root_dir, self.name)
parsed_url.GetPath(), self.root.root_dir, self.name,
out_cb=work_queue.out_cb)
self._used_scm.RunCommand('updatesingle',
options, args + [parsed_url.GetFilename()], file_list)
else:
@ -667,7 +669,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
self.maybeGetParentRevision(
command, options, parsed_url, self.parent.name, revision_overrides)
self._used_scm = gclient_scm.CreateSCM(
parsed_url, self.root.root_dir, self.name)
parsed_url, self.root.root_dir, self.name, self.outbuf,
out_cb=work_queue.out_cb)
self._got_revision = self._used_scm.RunCommand(command, options, args,
file_list)
if file_list:
@ -724,7 +727,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
match = re.match('^Binary file ([^\0]+) matches$', line)
if match:
print 'Binary file %s matches' % mod_path(match.group(1))
print 'Binary file %s matches\n' % mod_path(match.group(1))
return
items = line.split('\0')
@ -1050,7 +1053,8 @@ solutions = [
solutions."""
for dep in self.dependencies:
if dep.managed and dep.url:
scm = gclient_scm.CreateSCM(dep.url, self.root_dir, dep.name)
scm = gclient_scm.CreateSCM(
dep.url, self.root_dir, dep.name, self.outbuf)
actual_url = scm.GetActualRemoteURL(self._options)
if actual_url and not scm.DoesRemoteURLMatch(self._options):
raise gclient_utils.Error('''
@ -1234,7 +1238,8 @@ want to set 'managed': False in .gclient.
'It appears your safesync_url (%s) is not working properly\n'
'(as it returned an empty response). Check your config.' %
dep.safesync_url)
scm = gclient_scm.CreateSCM(dep.url, dep.root.root_dir, dep.name)
scm = gclient_scm.CreateSCM(
dep.url, dep.root.root_dir, dep.name, self.outbuf)
safe_rev = scm.GetUsableRev(rev, self._options)
if self._options.verbose:
print('Using safesync_url revision: %s.\n' % safe_rev)
@ -1265,7 +1270,8 @@ want to set 'managed': False in .gclient.
elif command == 'recurse':
pm = Progress(' '.join(args), 1)
work_queue = gclient_utils.ExecutionQueue(
self._options.jobs, pm, ignore_requirements=ignore_requirements)
self._options.jobs, pm, ignore_requirements=ignore_requirements,
verbose=self._options.verbose)
for s in self.dependencies:
work_queue.enqueue(s)
work_queue.flush(revision_overrides, command, args, options=self._options)
@ -1301,7 +1307,8 @@ want to set 'managed': False in .gclient.
if (entry not in entries and
(not any(path.startswith(entry + '/') for path in entries)) and
os.path.exists(e_dir)):
scm = gclient_scm.CreateSCM(prev_url, self.root_dir, entry_fixed)
scm = gclient_scm.CreateSCM(
prev_url, self.root_dir, entry_fixed, self.outbuf)
# Check to see if this directory is now part of a higher-up checkout.
if scm.GetCheckoutRoot() in full_entries:
@ -1332,7 +1339,8 @@ want to set 'managed': False in .gclient.
if not self.dependencies:
raise gclient_utils.Error('No solution specified')
# Load all the settings.
work_queue = gclient_utils.ExecutionQueue(self._options.jobs, None, False)
work_queue = gclient_utils.ExecutionQueue(
self._options.jobs, None, False, verbose=self._options.verbose)
for s in self.dependencies:
work_queue.enqueue(s)
work_queue.flush({}, None, [], options=self._options)
@ -1346,7 +1354,8 @@ want to set 'managed': False in .gclient.
else:
original_url = dep.parsed_url
url, _ = gclient_utils.SplitUrlRevision(original_url)
scm = gclient_scm.CreateSCM(original_url, self.root_dir, dep.name)
scm = gclient_scm.CreateSCM(
original_url, self.root_dir, dep.name, self.outbuf)
if not os.path.isdir(scm.checkout_path):
return None
return '%s@%s' % (url, scm.revinfo(self._options, [], None))

215
gclient_scm.py
Unescape Escape View File

@ -4,6 +4,8 @@
"""Gclient-specific SCM-specific operations."""
from __future__ import print_function
import logging
import os
import posixpath
@ -35,11 +37,12 @@ class DiffFiltererWrapper(object):
original_prefix = "--- "
working_prefix = "+++ "
def __init__(self, relpath):
def __init__(self, relpath, print_func):
# Note that we always use '/' as the path separator to be
# consistent with svn's cygwin-style output on Windows
self._relpath = relpath.replace("\\", "/")
self._current_file = None
self._print_func = print_func
def SetCurrentFile(self, current_file):
self._current_file = current_file
@ -59,7 +62,7 @@ class DiffFiltererWrapper(object):
if (line.startswith(self.original_prefix) or
line.startswith(self.working_prefix)):
line = self._Replace(line)
print(line)
self._print_func(line)
class SvnDiffFilterer(DiffFiltererWrapper):
@ -94,7 +97,7 @@ def GetScmName(url):
return None
def CreateSCM(url, root_dir=None, relpath=None):
def CreateSCM(url, root_dir=None, relpath=None, out_fh=None, out_cb=None):
SCM_MAP = {
'svn' : SVNWrapper,
'git' : GitWrapper,
@ -106,7 +109,7 @@ def CreateSCM(url, root_dir=None, relpath=None):
scm_class = SCM_MAP[scm_name]
if not scm_class.BinaryExists():
raise gclient_utils.Error('%s command not found' % scm_name)
return scm_class(url, root_dir, relpath)
return scm_class(url, root_dir, relpath, out_fh, out_cb)
# SCMWrapper base class
@ -116,7 +119,9 @@ class SCMWrapper(object):
This is the abstraction layer to bind to different SCM.
"""
def __init__(self, url=None, root_dir=None, relpath=None):
def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
out_cb=None):
self.url = url
self._root_dir = root_dir
if self._root_dir:
@ -126,6 +131,16 @@ class SCMWrapper(object):
self.relpath = self.relpath.replace('/', os.sep)
if self.relpath and self._root_dir:
self.checkout_path = os.path.join(self._root_dir, self.relpath)
if out_fh is None:
out_fh = sys.stdout
self.out_fh = out_fh
self.out_cb = out_cb
def Print(self, *args, **kwargs):
kwargs.setdefault('file', self.out_fh)
if kwargs.pop('timestamp', True):
self.out_fh.write('[%s] ' % gclient_utils.Elapsed())
print(*args, **kwargs)
def RunCommand(self, command, options, args, file_list=None):
commands = ['cleanup', 'update', 'updatesingle', 'revert',
@ -191,11 +206,12 @@ class GitWrapper(SCMWrapper):
cache_dir = None
def __init__(self, url=None, root_dir=None, relpath=None):
def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
out_cb=None):
"""Removes 'git+' fake prefix from git URL."""
if url.startswith('git+http://') or url.startswith('git+https://'):
url = url[4:]
SCMWrapper.__init__(self, url, root_dir, relpath)
SCMWrapper.__init__(self, url, root_dir, relpath, out_fh, out_cb)
@staticmethod
def BinaryExists():
@ -241,7 +257,7 @@ class GitWrapper(SCMWrapper):
gclient_utils.CheckCallAndFilter(
['git', 'diff', merge_base],
cwd=self.checkout_path,
filter_fn=GitDiffFilterer(self.relpath).Filter)
filter_fn=GitDiffFilterer(self.relpath).Filter, print_func=self.Print)
def UpdateSubmoduleConfig(self):
submod_cmd = ['git', 'config', '-f', '$toplevel/.git/config',
@ -316,7 +332,7 @@ class GitWrapper(SCMWrapper):
# expired yet. Use rev-list to get the corresponding revision.
# git rev-list -n 1 --before='time-stamp' branchname
if options.transitive:
print('Warning: --transitive only works for SVN repositories.')
self.Print('Warning: --transitive only works for SVN repositories.')
revision = default_rev
rev_str = ' at %s' % revision
@ -325,7 +341,7 @@ class GitWrapper(SCMWrapper):
printed_path = False
verbose = []
if options.verbose:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
verbose = ['--verbose']
printed_path = True
@ -352,13 +368,13 @@ class GitWrapper(SCMWrapper):
if not verbose:
# Make the output a little prettier. It's nice to have some whitespace
# between projects when cloning.
print('')
self.Print('')
return self._Capture(['rev-parse', '--verify', 'HEAD'])
if not managed:
self._UpdateBranchHeads(options, fetch=False)
self.UpdateSubmoduleConfig()
print ('________ unmanaged solution; skipping %s' % self.relpath)
self.Print('________ unmanaged solution; skipping %s' % self.relpath)
return self._Capture(['rev-parse', '--verify', 'HEAD'])
if not os.path.exists(os.path.join(self.checkout_path, '.git')):
@ -383,7 +399,7 @@ class GitWrapper(SCMWrapper):
subprocess2.capture(
['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote],
cwd=self.checkout_path).strip() != 'False'):
print('_____ switching %s to a new upstream' % self.relpath)
self.Print('_____ switching %s to a new upstream' % self.relpath)
# Make sure it's clean
self._CheckClean(rev_str)
# Switch over to the new upstream
@ -436,7 +452,7 @@ class GitWrapper(SCMWrapper):
remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'],
cwd=self.checkout_path)
if verbose:
print(remote_output)
self.Print(remote_output)
self._UpdateBranchHeads(options, fetch=True)
@ -453,7 +469,7 @@ class GitWrapper(SCMWrapper):
self._CheckDetachedHead(rev_str, options)
self._Capture(['checkout', '--quiet', '%s' % revision])
if not printed_path:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
elif current_type == 'hash':
# case 1
if scm.GIT.IsGitSvn(self.checkout_path) and upstream_branch is not None:
@ -482,7 +498,7 @@ class GitWrapper(SCMWrapper):
# case 4
new_base = revision.replace('heads', 'remotes/' + self.remote)
if not printed_path:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
switch_error = ("Switching upstream branch from %s to %s\n"
% (upstream_branch, new_base) +
"Please merge or rebase manually:\n" +
@ -494,7 +510,7 @@ class GitWrapper(SCMWrapper):
if files is not None:
files = self._Capture(['diff', upstream_branch, '--name-only']).split()
if verbose:
print('Trying fast-forward merge to branch : %s' % upstream_branch)
self.Print('Trying fast-forward merge to branch : %s' % upstream_branch)
try:
merge_args = ['merge']
if options.merge:
@ -502,12 +518,12 @@ class GitWrapper(SCMWrapper):
else:
merge_args.append('--ff-only')
merge_args.append(upstream_branch)
merge_output = scm.GIT.Capture(merge_args, cwd=self.checkout_path)
merge_output = self._Capture(merge_args)
except subprocess2.CalledProcessError as e:
if re.match('fatal: Not possible to fast-forward, aborting.', e.stderr):
files = []
if not printed_path:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
printed_path = True
while True:
try:
@ -529,34 +545,34 @@ class GitWrapper(SCMWrapper):
"cd %s && git " % self.checkout_path
+ "rebase %s" % upstream_branch)
elif re.match(r'skip|s', action, re.I):
print('Skipping %s' % self.relpath)
self.Print('Skipping %s' % self.relpath)
return
else:
print('Input not recognized')
self.Print('Input not recognized')
elif re.match("error: Your local changes to '.*' would be "
"overwritten by merge. Aborting.\nPlease, commit your "
"changes or stash them before you can merge.\n",
e.stderr):
if not printed_path:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
printed_path = True
raise gclient_utils.Error(e.stderr)
else:
# Some other problem happened with the merge
logging.error("Error during fast-forward merge in %s!" % self.relpath)
print(e.stderr)
self.Print(e.stderr)
raise
else:
# Fast-forward merge was successful
if not re.match('Already up-to-date.', merge_output) or verbose:
if not printed_path:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
printed_path = True
print(merge_output.strip())
self.Print(merge_output.strip())
if not verbose:
# Make the output a little prettier. It's nice to have some
# whitespace between projects when syncing.
print('')
self.Print('')
self.UpdateSubmoduleConfig()
if file_list is not None:
@ -571,7 +587,8 @@ class GitWrapper(SCMWrapper):
% (self.relpath, rev_str))
if verbose:
print('Checked out revision %s' % self.revinfo(options, (), None))
self.Print('Checked out revision %s' % self.revinfo(options, (), None),
timestamp=False)
# If --reset and --delete_unversioned_trees are specified, remove any
# untracked directories.
@ -585,7 +602,7 @@ class GitWrapper(SCMWrapper):
for path in (p for p in paths.splitlines() if p.endswith('/')):
full_path = os.path.join(self.checkout_path, path)
if not os.path.islink(full_path):
print('\n_____ removing unversioned directory %s' % path)
self.Print('_____ removing unversioned directory %s' % path)
gclient_utils.rmtree(full_path)
return self._Capture(['rev-parse', '--verify', 'HEAD'])
@ -599,7 +616,7 @@ class GitWrapper(SCMWrapper):
if not os.path.isdir(self.checkout_path):
# revert won't work if the directory doesn't exist. It needs to
# checkout instead.
print('\n_____ %s is missing, synching instead' % self.relpath)
self.Print('_____ %s is missing, synching instead' % self.relpath)
# Don't reuse the args.
return self.update(options, [], file_list)
@ -634,11 +651,12 @@ class GitWrapper(SCMWrapper):
def status(self, options, _args, file_list):
"""Display status information."""
if not os.path.isdir(self.checkout_path):
print(('\n________ couldn\'t run status in %s:\n'
'The directory does not exist.') % self.checkout_path)
self.Print('________ couldn\'t run status in %s:\n'
'The directory does not exist.' % self.checkout_path)
else:
merge_base = self._Capture(['merge-base', 'HEAD', self.remote])
self._Run(['diff', '--name-status', merge_base], options)
self._Run(['diff', '--name-status', merge_base], options,
stdout=self.out_fh)
if file_list is not None:
files = self._Capture(['diff', '--name-only', merge_base]).split()
file_list.extend([os.path.join(self.checkout_path, f) for f in files])
@ -672,14 +690,14 @@ class GitWrapper(SCMWrapper):
logging.debug('git config --get svn-remote.svn.fetch failed, '
'ignoring possible optimization.')
if options.verbose:
print('Running git svn fetch. This might take a while.\n')
self.Print('Running git svn fetch. This might take a while.\n')
scm.GIT.Capture(['svn', 'fetch'], cwd=self.checkout_path)
try:
sha1 = scm.GIT.GetBlessedSha1ForSvnRev(
cwd=self.checkout_path, rev=rev)
except gclient_utils.Error, e:
sha1 = e.message
print('\nWarning: Could not find a git revision with accurate\n'
self.Print('Warning: Could not find a git revision with accurate\n'
'.DEPS.git that maps to SVN revision %s. Sync-ing to\n'
'the closest sane git revision, which is:\n'
' %s\n' % (rev, e.message))
@ -741,7 +759,7 @@ class GitWrapper(SCMWrapper):
if not options.verbose:
# git clone doesn't seem to insert a newline properly before printing
# to stdout
print('')
self.Print('')
template_path = os.path.join(
os.path.dirname(THIS_FILE_PATH), 'git-templates')
cfg = gclient_utils.DefaultIndexPackConfig(self.url)
@ -766,11 +784,11 @@ class GitWrapper(SCMWrapper):
gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'),
os.path.join(self.checkout_path, '.git'))
except:
traceback.print_exc(file=sys.stderr)
traceback.print_exc(file=self.out_fh)
raise
finally:
if os.listdir(tmp_dir):
print('\n_____ removing non-empty tmp dir %s' % tmp_dir)
self.Print('_____ removing non-empty tmp dir %s' % tmp_dir)
gclient_utils.rmtree(tmp_dir)
if revision.startswith('refs/heads/'):
self._Run(
@ -778,7 +796,7 @@ class GitWrapper(SCMWrapper):
else:
# Squelch git's very verbose detached HEAD warning and use our own
self._Run(['checkout', '--quiet', revision], options)
print(
self.Print(
('Checked out %s to a detached HEAD. Before making any commits\n'
'in this repo, you should use \'git checkout <branch>\' to switch to\n'
'an existing branch or use \'git checkout %s -b <branch>\' to\n'
@ -807,16 +825,16 @@ class GitWrapper(SCMWrapper):
revision = newbase
action = 'merge' if merge else 'rebase'
if not printed_path:
print('\n_____ %s : Attempting %s onto %s...' % (
self.Print('_____ %s : Attempting %s onto %s...' % (
self.relpath, action, revision))
printed_path = True
else:
print('Attempting %s onto %s...' % (action, revision))
self.Print('Attempting %s onto %s...' % (action, revision))
if merge:
merge_output = self._Capture(['merge', revision])
if options.verbose:
print(merge_output)
self.Print(merge_output)
return
# Build the rebase command here using the args
@ -852,7 +870,7 @@ class GitWrapper(SCMWrapper):
"cd %s && git " % self.checkout_path
+ "%s" % ' '.join(rebase_cmd))
elif re.match(r'show|s', rebase_action, re.I):
print('\n%s' % e.stderr.strip())
self.Print('%s' % e.stderr.strip())
continue
else:
gclient_utils.Error("Input not recognized")
@ -862,18 +880,18 @@ class GitWrapper(SCMWrapper):
"Fix the conflict and run gclient again.\n"
"See 'man git-rebase' for details.\n")
else:
print(e.stdout.strip())
print('Rebase produced error output:\n%s' % e.stderr.strip())
self.Print(e.stdout.strip())
self.Print('Rebase produced error output:\n%s' % e.stderr.strip())
raise gclient_utils.Error("Unrecognized error, please merge or rebase "
"manually.\ncd %s && git " %
self.checkout_path
+ "%s" % ' '.join(rebase_cmd))
print(rebase_output.strip())
self.Print(rebase_output.strip())
if not options.verbose:
# Make the output a little prettier. It's nice to have some
# whitespace between projects when syncing.
print('')
self.Print('')
@staticmethod
def _CheckMinVersion(min_version):
@ -932,7 +950,7 @@ class GitWrapper(SCMWrapper):
name = ('saved-by-gclient-' +
self._Capture(['rev-parse', '--short', 'HEAD']))
self._Capture(['branch', '-f', name])
print('\n_____ found an unreferenced commit and saved it as \'%s\'' %
self.Print('_____ found an unreferenced commit and saved it as \'%s\'' %
name)
def _GetCurrentBranch(self):
@ -942,11 +960,10 @@ class GitWrapper(SCMWrapper):
return None
return branch
def _Capture(self, args, cwd=None):
return subprocess2.check_output(
['git'] + args,
stderr=subprocess2.VOID,
cwd=cwd or self.checkout_path).strip()
def _Capture(self, args, cwd=None, **kwargs):
kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stderr', subprocess2.PIPE)
return subprocess2.check_output(['git'] + args, **kwargs).strip()
def _UpdateBranchHeads(self, options, fetch=False):
"""Adds, and optionally fetches, "branch-heads" refspecs if requested."""
@ -963,26 +980,29 @@ class GitWrapper(SCMWrapper):
self._Run(fetch_cmd, options, retry=True)
def _Run(self, args, options, **kwargs):
kwargs.setdefault('cwd', self.checkout_path)
git_filter = not options.verbose
if git_filter:
kwargs['filter_fn'] = gclient_utils.GitFilter(kwargs.get('filter_fn'))
kwargs.setdefault('print_stdout', False)
# Don't prompt for passwords; just fail quickly and noisily.
# By default, git will use an interactive terminal prompt when a username/
# password is needed. That shouldn't happen in the chromium workflow,
# and if it does, then gclient may hide the prompt in the midst of a flood
# of terminal spew. The only indication that something has gone wrong
# will be when gclient hangs unresponsively. Instead, we disable the
# password prompt and simply allow git to fail noisily. The error
# message produced by git will be copied to gclient's output.
env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
env.setdefault('GIT_ASKPASS', 'true')
env.setdefault('SSH_ASKPASS', 'true')
else:
kwargs.setdefault('print_stdout', True)
cwd = kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stdout', self.out_fh)
filter_kwargs = { 'time_throttle': 10, 'out_fh': self.out_fh }
if self.out_cb:
filter_kwargs['predicate'] = self.out_cb
kwargs['filter_fn'] = git_filter = gclient_utils.GitFilter(**filter_kwargs)
kwargs.setdefault('print_stdout', False)
# Don't prompt for passwords; just fail quickly and noisily.
# By default, git will use an interactive terminal prompt when a username/
# password is needed. That shouldn't happen in the chromium workflow,
# and if it does, then gclient may hide the prompt in the midst of a flood
# of terminal spew. The only indication that something has gone wrong
# will be when gclient hangs unresponsively. Instead, we disable the
# password prompt and simply allow git to fail noisily. The error
# message produced by git will be copied to gclient's output.
env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
env.setdefault('GIT_ASKPASS', 'true')
env.setdefault('SSH_ASKPASS', 'true')
cmd = ['git'] + args
return gclient_utils.CheckCallAndFilterAndHeader(cmd, **kwargs)
header = "running '%s' in '%s'" % (' '.join(cmd), cwd)
git_filter(header)
return gclient_utils.CheckCallAndFilter(cmd, **kwargs)
class SVNWrapper(SCMWrapper):
@ -1032,7 +1052,7 @@ class SVNWrapper(SCMWrapper):
['svn', 'diff', '-x', '--ignore-eol-style'] + args,
cwd=self.checkout_path,
print_stdout=False,
filter_fn=SvnDiffFilterer(self.relpath).Filter)
filter_fn=SvnDiffFilterer(self.relpath).Filter, print_func=self.Print)
def update(self, options, args, file_list):
"""Runs svn to update or transparently checkout the working copy.
@ -1045,12 +1065,12 @@ class SVNWrapper(SCMWrapper):
# Only update if git or hg is not controlling the directory.
git_path = os.path.join(self.checkout_path, '.git')
if os.path.exists(git_path):
print('________ found .git directory; skipping %s' % self.relpath)
self.Print('________ found .git directory; skipping %s' % self.relpath)
return
hg_path = os.path.join(self.checkout_path, '.hg')
if os.path.exists(hg_path):
print('________ found .hg directory; skipping %s' % self.relpath)
self.Print('________ found .hg directory; skipping %s' % self.relpath)
return
if args:
@ -1086,7 +1106,7 @@ class SVNWrapper(SCMWrapper):
[], os.path.join(self.checkout_path, '.'))
except (gclient_utils.Error, subprocess2.CalledProcessError):
if options.reset and options.delete_unversioned_trees:
print 'Removing troublesome path %s' % self.checkout_path
self.Print('Removing troublesome path %s' % self.checkout_path)
gclient_utils.rmtree(self.checkout_path)
exists = False
else:
@ -1127,14 +1147,14 @@ class SVNWrapper(SCMWrapper):
latest_checkout = sorted_items[-1]
tempdir = tempfile.mkdtemp()
print 'Downloading %s...' % latest_checkout
self.Print('Downloading %s...' % latest_checkout)
code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
if code:
print '%s\n%s' % (out, err)
self.Print('%s\n%s' % (out, err))
raise Exception()
filename = latest_checkout.split('/')[-1]
tarball = os.path.join(tempdir, filename)
print 'Unpacking into %s...' % self.checkout_path
self.Print('Unpacking into %s...' % self.checkout_path)
gclient_utils.safe_makedirs(self.checkout_path)
# TODO(hinoka): Use 7z for windows.
cmd = ['tar', '--extract', '--ungzip',
@ -1143,7 +1163,7 @@ class SVNWrapper(SCMWrapper):
gclient_utils.CheckCallAndFilter(
cmd, stdout=sys.stdout, print_stdout=True)
print 'Deleting temp file'
self.Print('Deleting temp file')
gclient_utils.rmtree(tempdir)
# Rewrite the repository root to match.
@ -1154,14 +1174,14 @@ class SVNWrapper(SCMWrapper):
tarball_parsed.netloc)
if tarball_root != local_root:
print 'Switching repository root to %s' % local_root
self.Print('Switching repository root to %s' % local_root)
self._Run(['switch', '--relocate', tarball_root,
local_root, self.checkout_path],
options)
except Exception as e:
print 'We tried to get a source tarball but failed.'
print 'Resuming normal operations.'
print str(e)
self.Print('We tried to get a source tarball but failed.')
self.Print('Resuming normal operations.')
self.Print(str(e))
gclient_utils.safe_makedirs(os.path.dirname(self.checkout_path))
# We need to checkout.
@ -1171,7 +1191,7 @@ class SVNWrapper(SCMWrapper):
return self.Svnversion()
if not managed:
print ('________ unmanaged solution; skipping %s' % self.relpath)
self.Print(('________ unmanaged solution; skipping %s' % self.relpath))
return self.Svnversion()
if 'URL' not in from_info:
@ -1201,12 +1221,13 @@ class SVNWrapper(SCMWrapper):
assert not os.path.isabs(d[1])
path_to_remove = os.path.normpath(
os.path.join(self.checkout_path, d[1]))
print 'Removing troublesome path %s' % path_to_remove
self.Print('Removing troublesome path %s' % path_to_remove)
gclient_utils.rmtree(path_to_remove)
else:
print 'Not removing troublesome path %s automatically.' % d[1]
self.Print(
'Not removing troublesome path %s automatically.' % d[1])
if d[0][0] == '!':
print 'You can pass --force to enable automatic removal.'
self.Print('You can pass --force to enable automatic removal.')
raise e
# Retrieve the current HEAD version because svn is slow at null updates.
@ -1226,7 +1247,7 @@ class SVNWrapper(SCMWrapper):
can_switch = ((from_info['Repository Root'] != to_info['Repository Root'])
and (from_info['UUID'] == to_info['UUID']))
if can_switch:
print('\n_____ relocating %s to a new checkout' % self.relpath)
self.Print('_____ relocating %s to a new checkout' % self.relpath)
# We have different roots, so check if we can switch --relocate.
# Subversion only permits this if the repository UUIDs match.
# Perform the switch --relocate, then rewrite the from_url
@ -1254,7 +1275,7 @@ class SVNWrapper(SCMWrapper):
'there is local changes in %s. Delete the directory and '
'try again.') % (url, self.checkout_path))
# Ok delete it.
print('\n_____ switching %s to a new checkout' % self.relpath)
self.Print('_____ switching %s to a new checkout' % self.relpath)
gclient_utils.rmtree(self.checkout_path)
# We need to checkout.
command = ['checkout', url, self.checkout_path]
@ -1266,7 +1287,7 @@ class SVNWrapper(SCMWrapper):
# number of the existing directory, then we don't need to bother updating.
if not options.force and str(from_info['Revision']) == revision:
if options.verbose or not forced_revision:
print('\n_____ %s%s' % (self.relpath, rev_str))
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
else:
command = ['update', self.checkout_path]
command = self._AddAdditionalUpdateFlags(command, options, revision)
@ -1280,7 +1301,7 @@ class SVNWrapper(SCMWrapper):
if (status[0][0] == '?'
and os.path.isdir(full_path)
and not os.path.islink(full_path)):
print('\n_____ removing unversioned directory %s' % status[1])
self.Print('_____ removing unversioned directory %s' % status[1])
gclient_utils.rmtree(full_path)
return self.Svnversion()
@ -1323,20 +1344,20 @@ class SVNWrapper(SCMWrapper):
gclient_utils.rmtree(self.checkout_path)
# svn revert won't work if the directory doesn't exist. It needs to
# checkout instead.
print('\n_____ %s is missing, synching instead' % self.relpath)
self.Print('_____ %s is missing, synching instead' % self.relpath)
# Don't reuse the args.
return self.update(options, [], file_list)
if not os.path.isdir(os.path.join(self.checkout_path, '.svn')):
if os.path.isdir(os.path.join(self.checkout_path, '.git')):
print('________ found .git directory; skipping %s' % self.relpath)
self.Print('________ found .git directory; skipping %s' % self.relpath)
return
if os.path.isdir(os.path.join(self.checkout_path, '.hg')):
print('________ found .hg directory; skipping %s' % self.relpath)
self.Print('________ found .hg directory; skipping %s' % self.relpath)
return
if not options.force:
raise gclient_utils.Error('Invalid checkout path, aborting')
print(
self.Print(
'\n_____ %s is not a valid svn checkout, synching instead' %
self.relpath)
gclient_utils.rmtree(self.checkout_path)
@ -1349,7 +1370,7 @@ class SVNWrapper(SCMWrapper):
if logging.getLogger().isEnabledFor(logging.INFO):
logging.info('%s%s' % (file_status[0], file_status[1]))
else:
print(os.path.join(self.checkout_path, file_status[1]))
self.Print(os.path.join(self.checkout_path, file_status[1]))
scm.SVN.Revert(self.checkout_path, callback=printcb)
# Revert() may delete the directory altogether.
@ -1382,7 +1403,7 @@ class SVNWrapper(SCMWrapper):
command = ['status'] + args
if not os.path.isdir(self.checkout_path):
# svn status won't work if the directory doesn't exist.
print(('\n________ couldn\'t run \'%s\' in \'%s\':\n'
self.Print(('\n________ couldn\'t run \'%s\' in \'%s\':\n'
'The directory does not exist.') %
(' '.join(command), self.checkout_path))
# There's no file list to retrieve.

122
gclient_utils.py
Unescape Escape View File

@ -6,6 +6,7 @@
import codecs
import cStringIO
import datetime
import logging
import os
import pipes
@ -25,6 +26,7 @@ import subprocess2
RETRY_MAX = 3
RETRY_INITIAL_SLEEP = 0.5
START = datetime.datetime.now()
_WARNINGS = []
@ -47,6 +49,12 @@ class Error(Exception):
super(Error, self).__init__(msg, *args, **kwargs)
def Elapsed(until=None):
if until is None:
until = datetime.datetime.now()
return str(until - START).partition('.')[0]
def PrintWarnings():
"""Prints any accumulated warnings."""
if _WARNINGS:
@ -483,12 +491,8 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None,
output.write(in_byte)
if print_stdout:
stdout.write(in_byte)
if in_byte != '\r':
if in_byte != '\n':
in_line += in_byte
else:
filter_fn(in_line)
in_line = ''
if in_byte not in ['\r', '\n']:
in_line += in_byte
else:
filter_fn(in_line)
in_line = ''
@ -525,9 +529,9 @@ class GitFilter(object):
Allows a custom function to skip certain lines (predicate), and will throttle
the output of percentage completed lines to only output every X seconds.
"""
PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*')
def __init__(self, time_throttle=0, predicate=None):
def __init__(self, time_throttle=0, predicate=None, out_fh=None):
"""
Args:
time_throttle (int): GitFilter will throttle 'noisy' output (such as the
@ -535,10 +539,13 @@ class GitFilter(object):
seconds apart.
predicate (f(line)): An optional function which is invoked for every line.
The line will be skipped if predicate(line) returns False.
out_fh: File handle to write output to.
"""
self.last_time = 0
self.time_throttle = time_throttle
self.predicate = predicate
self.out_fh = out_fh or sys.stdout
self.progress_prefix = None
def __call__(self, line):
# git uses an escape sequence to clear the line; elide it.
@ -549,11 +556,14 @@ class GitFilter(object):
return
now = time.time()
match = self.PERCENT_RE.match(line)
if not match:
self.last_time = 0
if (now - self.last_time) >= self.time_throttle:
self.last_time = now
print line
if match:
if match.group(1) != self.progress_prefix:
self.progress_prefix = match.group(1)
elif now - self.last_time < self.time_throttle:
return
self.last_time = now
self.out_fh.write('[%s] ' % Elapsed())
print >> self.out_fh, line
def FindGclientRoot(from_dir, filename='.gclient'):
@ -683,6 +693,8 @@ class WorkItem(object):
def __init__(self, name):
# A unique string representing this work item.
self._name = name
self.outbuf = cStringIO.StringIO()
self.start = self.finish = None
def run(self, work_queue):
"""work_queue is passed as keyword argument so it should be
@ -704,7 +716,7 @@ class ExecutionQueue(object):
Methods of this class are thread safe.
"""
def __init__(self, jobs, progress, ignore_requirements):
def __init__(self, jobs, progress, ignore_requirements, verbose=False):
"""jobs specifies the number of concurrent tasks to allow. progress is a
Progress instance."""
# Set when a thread is done or a new item is enqueued.
@ -725,6 +737,9 @@ class ExecutionQueue(object):
self.progress.update(0)
self.ignore_requirements = ignore_requirements
self.verbose = verbose
self.last_join = None
self.last_subproc_output = None
def enqueue(self, d):
"""Enqueue one Dependency to be executed later once its requirements are
@ -743,9 +758,30 @@ class ExecutionQueue(object):
finally:
self.ready_cond.release()
def out_cb(self, _):
self.last_subproc_output = datetime.datetime.now()
return True
@staticmethod
def format_task_output(task, comment=''):
if comment:
comment = ' (%s)' % comment
if task.start and task.finish:
elapsed = ' (Elapsed: %s)' % (
str(task.finish - task.start).partition('.')[0])
else:
elapsed = ''
return """
%s%s%s
----------------------------------------
%s
----------------------------------------""" % (
task.name, comment, task.outbuf.getvalue().strip(), elapsed)
def flush(self, *args, **kwargs):
"""Runs all enqueued items until all are executed."""
kwargs['work_queue'] = self
self.last_subproc_output = self.last_join = datetime.datetime.now()
self.ready_cond.acquire()
try:
while True:
@ -778,6 +814,17 @@ class ExecutionQueue(object):
# We need to poll here otherwise Ctrl-C isn't processed.
try:
self.ready_cond.wait(10)
# If we haven't printed to terminal for a while, but we have received
# spew from a suprocess, let the user know we're still progressing.
now = datetime.datetime.now()
if (now - self.last_join > datetime.timedelta(seconds=60) and
self.last_subproc_output > self.last_join):
if self.progress:
print >> sys.stdout, ''
elapsed = Elapsed()
print >> sys.stdout, '[%s] Still working on:' % elapsed
for task in self.running:
print >> sys.stdout, '[%s] %s' % (elapsed, task.item.name)
except KeyboardInterrupt:
# Help debugging by printing some information:
print >> sys.stderr, (
@ -788,7 +835,10 @@ class ExecutionQueue(object):
', '.join(self.ran),
len(self.running)))
for i in self.queued:
print >> sys.stderr, '%s: %s' % (i.name, ', '.join(i.requirements))
print >> sys.stderr, '%s (not started): %s' % (
i.name, ', '.join(i.requirements))
for i in self.running:
print >> sys.stderr, self.format_task_output(i.item, 'interrupted')
raise
# Something happened: self.enqueue() or a thread terminated. Loop again.
finally:
@ -796,11 +846,14 @@ class ExecutionQueue(object):
assert not self.running, 'Now guaranteed to be single-threaded'
if not self.exceptions.empty():
if self.progress:
print >> sys.stdout, ''
# To get back the stack location correctly, the raise a, b, c form must be
# used, passing a tuple as the first argument doesn't work.
e = self.exceptions.get()
e, task = self.exceptions.get()
print >> sys.stderr, self.format_task_output(task.item, 'ERROR')
raise e[0], e[1], e[2]
if self.progress:
elif self.progress:
self.progress.end()
def _flush_terminated_threads(self):
@ -812,7 +865,10 @@ class ExecutionQueue(object):
self.running.append(t)
else:
t.join()
self.last_join = datetime.datetime.now()
sys.stdout.flush()
if self.verbose:
print >> sys.stdout, self.format_task_output(t.item)
if self.progress:
self.progress.update(1, t.item.name)
if t.item.name in self.ran:
@ -832,10 +888,26 @@ class ExecutionQueue(object):
else:
# Run the 'thread' inside the main thread. Don't try to catch any
# exception.
task_item.run(*args, **kwargs)
self.ran.append(task_item.name)
if self.progress:
self.progress.update(1, ', '.join(t.item.name for t in self.running))
try:
task_item.start = datetime.datetime.now()
print >> task_item.outbuf, '[%s] Started.' % Elapsed(task_item.start)
task_item.run(*args, **kwargs)
task_item.finish = datetime.datetime.now()
print >> task_item.outbuf, '[%s] Finished.' % Elapsed(task_item.finish)
self.ran.append(task_item.name)
if self.verbose:
if self.progress:
print >> sys.stdout, ''
print >> sys.stdout, self.format_task_output(task_item)
if self.progress:
self.progress.update(1, ', '.join(t.item.name for t in self.running))
except KeyboardInterrupt:
print >> sys.stderr, self.format_task_output(task_item, 'interrupted')
raise
except Exception:
print >> sys.stderr, self.format_task_output(task_item, 'ERROR')
raise
class _Worker(threading.Thread):
"""One thread to execute one WorkItem."""
@ -853,17 +925,21 @@ class ExecutionQueue(object):
logging.debug('_Worker.run(%s)' % self.item.name)
work_queue = self.kwargs['work_queue']
try:
self.item.start = datetime.datetime.now()
print >> self.item.outbuf, '[%s] Started.' % Elapsed(self.item.start)
self.item.run(*self.args, **self.kwargs)
self.item.finish = datetime.datetime.now()
print >> self.item.outbuf, '[%s] Finished.' % Elapsed(self.item.finish)
except KeyboardInterrupt:
logging.info('Caught KeyboardInterrupt in thread %s', self.item.name)
logging.info(str(sys.exc_info()))
work_queue.exceptions.put(sys.exc_info())
work_queue.exceptions.put((sys.exc_info(), self))
raise
except Exception:
# Catch exception location.
logging.info('Caught exception in thread %s', self.item.name)
logging.info(str(sys.exc_info()))
work_queue.exceptions.put(sys.exc_info())
work_queue.exceptions.put((sys.exc_info(), self))
finally:
logging.info('_Worker.run(%s) done', self.item.name)
work_queue.ready_cond.acquire()

39
tests/gclient_scm_test.py
Unescape Escape View File

@ -13,6 +13,7 @@ from subprocess import Popen, PIPE, STDOUT
import logging
import os
import re
import sys
import tempfile
import unittest
@ -28,6 +29,14 @@ import subprocess2
# Shortcut since this function is used often
join = gclient_scm.os.path.join
TIMESTAMP_RE = re.compile('\[[0-9]{1,2}:[0-9]{2}:[0-9]{2}\] (.*)', re.DOTALL)
def strip_timestamps(value):
lines = value.splitlines(True)
for i in xrange(len(lines)):
m = TIMESTAMP_RE.match(lines[i])
if m:
lines[i] = m.group(1)
return ''.join(lines)
# Access to a protected member XXX of a client class
# pylint: disable=W0212
@ -89,6 +98,12 @@ class SVNWrapperTestCase(BaseTestCase):
self.jobs = 1
self.delete_unversioned_trees = False
def checkstdout(self, expected):
value = sys.stdout.getvalue()
sys.stdout.close()
# pylint: disable=E1101
self.assertEquals(expected, strip_timestamps(value))
def Options(self, *args, **kwargs):
return self.OptionsObject(*args, **kwargs)
@ -179,7 +194,7 @@ class SVNWrapperTestCase(BaseTestCase):
relpath=self.relpath)
scm.revert(options, self.args, files_list)
self.checkstdout(
('\n_____ %s is missing, synching instead\n' % self.relpath))
('_____ %s is missing, synching instead\n' % self.relpath))
def testRevertNoDotSvn(self):
options = self.Options(verbose=True, force=True)
@ -416,7 +431,7 @@ class SVNWrapperTestCase(BaseTestCase):
scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
relpath=self.relpath)
scm.update(options, (), files_list)
self.checkstdout('\n_____ %s at 42\n' % self.relpath)
self.checkstdout('_____ %s at 42\n' % self.relpath)
def testUpdateResetDeleteUnversionedTrees(self):
options = self.Options(verbose=True)
@ -461,8 +476,8 @@ class SVNWrapperTestCase(BaseTestCase):
files_list = []
scm.update(options, (), files_list)
self.checkstdout(
('\n_____ %s at 42\n'
'\n_____ removing unversioned directory dir\n') % self.relpath)
('_____ %s at 42\n'
'_____ removing unversioned directory dir\n') % self.relpath)
def testUpdateSingleCheckout(self):
options = self.Options(verbose=True)
@ -509,7 +524,7 @@ class SVNWrapperTestCase(BaseTestCase):
scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
relpath=self.relpath)
scm.updatesingle(options, ['DEPS'], files_list)
self.checkstdout('\n_____ %s at 42\n' % self.relpath)
self.checkstdout('_____ %s at 42\n' % self.relpath)
def testUpdateSingleCheckoutSVN14(self):
options = self.Options(verbose=True)
@ -581,7 +596,7 @@ class SVNWrapperTestCase(BaseTestCase):
relpath=self.relpath)
scm.updatesingle(options, ['DEPS'], files_list)
self.checkstdout(
('\n_____ %s at 42\n' % self.relpath))
('_____ %s at 42\n' % self.relpath))
def testUpdateSingleUpdate(self):
options = self.Options(verbose=True)
@ -616,7 +631,7 @@ class SVNWrapperTestCase(BaseTestCase):
scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
relpath=self.relpath)
scm.updatesingle(options, ['DEPS'], files_list)
self.checkstdout('\n_____ %s at 42\n' % self.relpath)
self.checkstdout('_____ %s at 42\n' % self.relpath)
def testUpdateGit(self):
options = self.Options(verbose=True)
@ -745,6 +760,12 @@ from :3
def Options(self, *args, **kwargs):
return self.OptionsObject(*args, **kwargs)
def checkstdout(self, expected):
value = sys.stdout.getvalue()
sys.stdout.close()
# pylint: disable=E1101
self.assertEquals(expected, strip_timestamps(value))
@staticmethod
def CreateGitRepo(git_import, path):
"""Do it for real."""
@ -895,7 +916,7 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
scm.status(options, self.args, file_list)
self.assertEquals(file_list, [file_path])
self.checkstdout(
('\n________ running \'git diff --name-status '
('running \'git diff --name-status '
'069c602044c5388d2d15c3f875b057c852003458\' in \'%s\'\nM\ta\n') %
join(self.root_dir, '.'))
@ -915,7 +936,7 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
expected_file_list = [join(self.base_path, x) for x in ['a', 'b']]
self.assertEquals(sorted(file_list), expected_file_list)
self.checkstdout(
('\n________ running \'git diff --name-status '
('running \'git diff --name-status '
'069c602044c5388d2d15c3f875b057c852003458\' in \'%s\'\nM\ta\nM\tb\n') %
join(self.root_dir, '.'))

71
tests/gclient_smoketest.py
Unescape Escape View File

@ -544,7 +544,7 @@ class GClientSmokeSVN(GClientSmokeBase):
out = self.splitBlock(results[0])
# src, src/other is missing, src/other, src/third_party/foo is missing,
# src/third_party/foo, 2 svn hooks, 3 related to File().
self.assertEquals(10, len(out))
self.assertEquals( 8, len(out))
self.checkString('', results[1])
self.assertEquals(0, results[2])
tree = self.mangle_svn_tree(
@ -597,14 +597,14 @@ class GClientSmokeSVN(GClientSmokeBase):
['running', join(self.root_dir, 'src', 'third_party', 'fpp')],
['running', join(self.root_dir, 'src', 'third_party', 'prout')]])
out = self.svnBlockCleanup(out)
self.checkString('other', out[0][1])
self.checkString(join('third_party', 'fpp'), out[0][2])
self.checkString(join('third_party', 'prout'), out[0][3])
self.checkString('hi', out[1][1])
self.assertEquals(4, len(out[0]))
self.assertEquals(2, len(out[1]))
self.assertEquals(1, len(out[2]))
self.assertEquals(1, len(out[3]))
self.checkString('other', out[0][5])
self.checkString(join('third_party', 'fpp'), out[0][7])
self.checkString(join('third_party', 'prout'), out[0][8])
self.checkString('hi', out[1][5])
self.assertEquals(9, len(out[0]))
self.assertEquals(7, len(out[1]))
self.assertEquals(6, len(out[2]))
self.assertEquals(6, len(out[3]))
self.assertEquals(4, len(out))
# Revert implies --force implies running hooks without looking at pattern
@ -613,7 +613,7 @@ class GClientSmokeSVN(GClientSmokeBase):
# the file list after some ___ running 'svn status'
results = self.gclient(['revert', '--deps', 'mac', '--jobs', '1'])
out = self.splitBlock(results[0])
self.assertEquals(7, len(out))
self.assertEquals(4, len(out))
self.checkString('', results[1])
self.assertEquals(0, results[2])
tree = self.mangle_svn_tree(
@ -848,7 +848,7 @@ class GClientSmokeGIT(GClientSmokeBase):
# Test unversioned checkout.
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '1'],
['running', 'running', 'running', 'running', 'running'])
['running', 'running'])
# TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must
# add sync parsing to get the list of updated files.
tree = self.mangle_git_tree(('repo_1@2', 'src'),
@ -867,7 +867,7 @@ class GClientSmokeGIT(GClientSmokeBase):
['sync', '--jobs', '1', '--revision',
'src@' + self.githash('repo_1', 1),
'--deps', 'mac', '--delete_unversioned_trees'],
['running', 'running', 'deleting'])
['deleting'])
tree = self.mangle_git_tree(('repo_1@1', 'src'),
('repo_2@2', 'src/repo2'),
('repo_3@1', 'src/repo2/repo3'),
@ -877,7 +877,7 @@ class GClientSmokeGIT(GClientSmokeBase):
# Test incremental sync: delete-unversioned_trees isn't there.
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '1'],
['running', 'running', 'running'])
['running', 'running'])
tree = self.mangle_git_tree(('repo_1@2', 'src'),
('repo_2@1', 'src/repo2'),
('repo_3@1', 'src/repo2/repo3'),
@ -895,7 +895,7 @@ class GClientSmokeGIT(GClientSmokeBase):
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '1',
'--revision', 'invalid@' + self.githash('repo_1', 1)],
['running', 'running', 'running', 'running', 'running'],
['running', 'running'],
'Please fix your script, having invalid --revision flags '
'will soon considered an error.\n')
tree = self.mangle_git_tree(('repo_1@2', 'src'),
@ -913,7 +913,7 @@ class GClientSmokeGIT(GClientSmokeBase):
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '1',
'--revision', self.githash('repo_1', 1)],
['running', 'running', 'running', 'running'])
[])
tree = self.mangle_git_tree(('repo_1@1', 'src'),
('repo_2@2', 'src/repo2'),
('repo_3@1', 'src/repo2/repo3'),
@ -928,7 +928,7 @@ class GClientSmokeGIT(GClientSmokeBase):
# Test unversioned checkout.
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '8'],
['running', 'running', 'running', 'running', 'running'],
['running', 'running'],
untangle=True)
# TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must
# add sync parsing to get the list of updated files.
@ -948,7 +948,7 @@ class GClientSmokeGIT(GClientSmokeBase):
self.parseGclient(
['sync', '--revision', 'src@' + self.githash('repo_1', 1),
'--deps', 'mac', '--delete_unversioned_trees', '--jobs', '1'],
[ 'running', 'running', 'deleting'],
['deleting'],
untangle=True)
tree = self.mangle_git_tree(('repo_1@1', 'src'),
('repo_2@2', 'src/repo2'),
@ -959,7 +959,7 @@ class GClientSmokeGIT(GClientSmokeBase):
# Test incremental sync: delete-unversioned_trees isn't there.
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '8'],
['running', 'running', 'running'],
['running', 'running'],
untangle=True)
tree = self.mangle_git_tree(('repo_1@2', 'src'),
('repo_2@1', 'src/repo2'),
@ -1001,16 +1001,13 @@ class GClientSmokeGIT(GClientSmokeBase):
return
self.gclient(['config', self.git_base + 'repo_5', '--name', 'src'])
expectation = [
('running', self.root_dir), # git clone repo_5
('running', self.root_dir), # pre-deps hook
('running', self.root_dir), # git clone repo_1
('running', self.root_dir), # git clone repo_1
]
out = self.parseGclient(['sync', '--deps', 'mac', '--jobs=1',
'--revision', 'src@' + self.githash('repo_5', 2)],
expectation)
self.assertEquals(2, len(out[1]))
self.assertEquals('pre-deps hook', out[1][1])
self.assertEquals(2, len(out[0]))
self.assertEquals('pre-deps hook', out[0][1])
tree = self.mangle_git_tree(('repo_5@2', 'src'),
('repo_1@2', 'src/repo1'),
('repo_2@1', 'src/repo2')
@ -1054,7 +1051,6 @@ class GClientSmokeGIT(GClientSmokeBase):
return
self.gclient(['config', self.git_base + 'repo_5', '--name', 'src'])
expectated_stdout = [
('running', self.root_dir), # git clone repo_5
('running', self.root_dir), # pre-deps hook
('running', self.root_dir), # pre-deps hook (fails)
]
@ -1134,9 +1130,6 @@ class GClientSmokeGITMutates(GClientSmokeBase):
self.gclient(['sync', '--deps', 'mac'])
write(join(self.root_dir, 'src', 'repo2', 'hi'), 'Hey!')
expected1 = ('running', os.path.join(self.root_dir, 'src'))
expected2 = ('running', os.path.join(expected1[1], 'repo2'))
expected3 = ('running', os.path.join(expected2[1], 'repo_renamed'))
out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], [])
# TODO(maruel): http://crosbug.com/3584 It should output the unversioned
# files.
@ -1147,13 +1140,9 @@ class GClientSmokeGITMutates(GClientSmokeBase):
# there should be two results for each. The last two results should reflect
# writing git_hooked1 and git_hooked2. There's only one result for the third
# because it is clean and has no output for 'git clean'.
expected4 = ('running', self.root_dir)
out = self.parseGclient(['revert', '--deps', 'mac', '--jobs', '1'],
[expected1, expected1,
expected2, expected2,
expected3,
expected4, expected4])
self.assertEquals(7, len(out))
['running', 'running'])
self.assertEquals(2, len(out))
tree = self.mangle_git_tree(('repo_1@3', 'src'),
('repo_2@1', 'src/repo2'),
('repo_3@2', 'src/repo2/repo_renamed'))
@ -1171,11 +1160,8 @@ class GClientSmokeGITMutates(GClientSmokeBase):
'custom_vars': {'r2hash': self.FAKE_REPOS.git_hashes['repo_2'][-1][0] }
}])
out = self.parseGclient(['revert', '--deps', 'mac', '--jobs', '1'],
[expected1, expected1,
expected2, expected2,
expected3,
expected4, expected4])
self.assertEquals(7, len(out))
['running', 'running'])
self.assertEquals(2, len(out))
tree = self.mangle_git_tree(('repo_1@3', 'src'),
('repo_2@3', 'src/repo2'),
('repo_3@2', 'src/repo2/repo_renamed'))
@ -1204,12 +1190,12 @@ class GClientSmokeBoth(GClientSmokeBase):
'{"name": "src-git",'
'"url": "' + self.git_base + 'repo_1"}]'])
self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'],
['running', 'running', 'running',
['running', 'running',
# This is due to the way svn update is called for a single
# file when File() is used in a DEPS file.
('running', self.root_dir + '/src/file/other'),
'running', 'running', 'running', 'running',
'running', 'running', 'running', 'running'])
'running', 'running'])
tree = self.mangle_git_tree(('repo_1@2', 'src-git'),
('repo_2@1', 'src/repo2'),
('repo_3@2', 'src/repo2/repo_renamed'))
@ -1240,7 +1226,7 @@ class GClientSmokeBoth(GClientSmokeBase):
self.checkString('', stderr)
self.assertEquals(0, returncode)
results = self.splitBlock(stdout)
self.assertEquals(12, len(results))
self.assertEquals(9, len(results))
tree = self.mangle_git_tree(('repo_1@2', 'src-git'),
('repo_2@1', 'src/repo2'),
('repo_3@2', 'src/repo2/repo_renamed'))
@ -1267,8 +1253,7 @@ class GClientSmokeBoth(GClientSmokeBase):
self.parseGclient(
['sync', '--deps', 'mac', '--jobs', '1', '--revision', '1',
'-r', 'src-git@' + self.githash('repo_1', 1)],
['running', 'running', 'running', 'running', 'running',
'running', 'running', 'running'],
['running', 'running', 'running', 'running'],
expected_stderr=
'You must specify the full solution name like --revision src@1\n'
'when you have multiple solutions setup in your .gclient file.\n'

2
tests/gclient_test.py
Unescape Escape View File

@ -72,7 +72,7 @@ class GclientTest(trial_dir.TestCase):
os.chdir(self.previous_dir)
super(GclientTest, self).tearDown()
def _createscm(self, parsed_url, root_dir, name):
def _createscm(self, parsed_url, root_dir, name, out_fh=None, out_cb=None):
self.assertTrue(parsed_url.startswith('svn://example.com/'), parsed_url)
self.assertTrue(root_dir.startswith(self.root_dir), root_dir)
return SCMMock(self, parsed_url)

14
third_party/repo/progress.py vendored
Unescape Escape View File

@ -59,15 +59,17 @@ class Progress(object):
return
if self._total <= 0:
sys.stdout.write('%s: %d, done.\n' % (
text = '%s: %d, done.' % (
self._title,
self._done))
sys.stdout.flush()
self._done)
else:
p = (100 * self._done) / self._total
sys.stdout.write('%s: %3d%% (%d/%d), done.\n' % (
text = '%s: %3d%% (%d/%d), done.' % (
self._title,
p,
self._done,
self._total))
sys.stdout.flush()
self._total)
spaces = max(self._width - len(text), 0)
sys.stdout.write('%s%*s\n' % (text, spaces, ''))
sys.stdout.flush()

Write Preview
Loading…
Cancel
Save