First commit of depot_tools/fetch.py
This is the bare minimum to have fetch.py working for chromium. The DEPS file pulls the new tools/recipes repo in, which will contain the recipe for chromium and all other checkout recipes eventually (followup CL coming shortly). fetch and fetch.bat simply invoke fetch.py. fetch.py takes a recipe and some arguments on the command line, invokes the recipe to get its instructions, and performs the actual checkout. Currently only supports Gclient-Git-Svn checkout (the kind needed for Chromium), but other checkout types will be pulled from annotated_checkout.py soon. Review URL: https://codereview.chromium.org/13463006 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@192130 0039d316-1c4b-4281-b951-d872f2087c98experimental/szager/collated-output
agable@chromium.org
12 years ago
parent
07d149fbb0
commit
cc023508e4
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
base_dir=$(dirname "$0")
|
||||
|
||||
PYTHONDONTWRITEBYTECODE=1 exec python "$base_dir/fetch.py" "$@"
|
||||
@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
:: Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
:: Use of this source code is governed by a BSD-style license that can be
|
||||
:: found in the LICENSE file.
|
||||
|
||||
:: This is required with cygwin only.
|
||||
PATH=%~dp0;%PATH%
|
||||
|
||||
:: Defer control.
|
||||
%~dp0python "%~dp0\fetch.py" %*
|
||||
@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
Tool to perform checkouts in one easy command line!
|
||||
|
||||
Usage:
|
||||
fetch <recipe> [--property=value [--property2=value2 ...]]
|
||||
|
||||
This script is a wrapper around various version control and repository
|
||||
checkout commands. It requires a |recipe| name, fetches data from that
|
||||
recipe in depot_tools/recipes, and then performs all necessary inits,
|
||||
checkouts, pulls, fetches, etc.
|
||||
|
||||
Optional arguments may be passed on the command line in key-value pairs.
|
||||
These parameters will be passed through to the recipe's main method.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import pipes
|
||||
|
||||
|
||||
SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
#################################################
|
||||
# Checkout class definitions.
|
||||
#################################################
|
||||
class Checkout(object):
|
||||
"""Base class for implementing different types of checkouts.
|
||||
|
||||
Attributes:
|
||||
|base|: the absolute path of the directory in which this script is run.
|
||||
|spec|: the spec for this checkout as returned by the recipe. Different
|
||||
subclasses will expect different keys in this dictionary.
|
||||
|root|: the directory into which the checkout will be performed, as returned
|
||||
by the recipe. This is a relative path from |base|.
|
||||
"""
|
||||
def __init__(self, spec, root):
|
||||
self.base = os.getcwd()
|
||||
self.spec = spec
|
||||
self.root = root
|
||||
|
||||
def exists(self):
|
||||
pass
|
||||
|
||||
def init(self):
|
||||
pass
|
||||
|
||||
def sync(self):
|
||||
pass
|
||||
|
||||
|
||||
class GclientCheckout(Checkout):
|
||||
|
||||
@staticmethod
|
||||
def run_gclient(*cmd, **kwargs):
|
||||
print 'Running: gclient %s' % ' '.join(pipes.quote(x) for x in cmd)
|
||||
return subprocess.check_call(('gclient',) + cmd, **kwargs)
|
||||
|
||||
|
||||
class GitCheckout(Checkout):
|
||||
|
||||
@staticmethod
|
||||
def run_git(*cmd, **kwargs):
|
||||
print 'Running: git %s' % ' '.join(pipes.quote(x) for x in cmd)
|
||||
return subprocess.check_call(('git',) + cmd, **kwargs)
|
||||
|
||||
|
||||
class GclientGitSvnCheckout(GclientCheckout, GitCheckout):
|
||||
|
||||
def __init__(self, spec, root):
|
||||
super(GclientGitSvnCheckout, self).__init__(spec, root)
|
||||
assert 'solutions' in self.spec
|
||||
keys = ['solutions', 'target_os', 'target_os_only']
|
||||
gclient_spec = '\n'.join('%s = %s' % (key, self.spec[key])
|
||||
for key in self.spec if key in keys)
|
||||
self.spec['gclient_spec'] = gclient_spec
|
||||
assert 'svn_url' in self.spec
|
||||
assert 'svn_branch' in self.spec
|
||||
assert 'svn_ref' in self.spec
|
||||
|
||||
def exists(self):
|
||||
return os.path.exists(os.path.join(os.getcwd(), self.root))
|
||||
|
||||
def init(self):
|
||||
# Configure and do the gclient checkout.
|
||||
self.run_gclient('config', '--spec', self.spec['gclient_spec'])
|
||||
self.run_gclient('sync')
|
||||
|
||||
# Configure git.
|
||||
wd = os.path.join(self.base, self.root)
|
||||
self.run_git(
|
||||
'submodule', 'foreach',
|
||||
'git config -f $toplevel/.git/config submodule.$name.ignore all',
|
||||
cwd=wd)
|
||||
self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd)
|
||||
|
||||
# Configure git-svn.
|
||||
self.run_git('svn', 'init', '--prefix=origin/', '-T',
|
||||
self.spec['svn_branch'], self.spec['svn_url'], cwd=wd)
|
||||
self.run_git('config', 'svn-remote.svn.fetch', self.spec['svn_branch'] +
|
||||
':refs/remotes/origin/' + self.spec['svn_ref'], cwd=wd)
|
||||
self.run_git('svn', 'fetch', cwd=wd)
|
||||
|
||||
# Configure git-svn submodules, if any.
|
||||
submodules = json.loads(self.spec.get('submodule_git_svn_spec', '{}'))
|
||||
for path, subspec in submodules.iteritems():
|
||||
subspec = submodules[path]
|
||||
ospath = os.path.join(*path.split('/'))
|
||||
wd = os.path.join(self.base, self.root, ospath)
|
||||
self.run_git('svn', 'init', '--prefix=origin/', '-T',
|
||||
subspec['svn_branch'], subspec['svn_url'], cwd=wd)
|
||||
self.run_git('config', '--replace', 'svn-remote.svn.fetch',
|
||||
subspec['svn_branch'] + ':refs/remotes/origin/' +
|
||||
subspec['svn_ref'], cwd=wd)
|
||||
self.run_git('svn', 'fetch', cwd=wd)
|
||||
|
||||
|
||||
CHECKOUT_TYPE_MAP = {
|
||||
'gclient': GclientCheckout,
|
||||
'gclient_git_svn': GclientGitSvnCheckout,
|
||||
'git': GitCheckout,
|
||||
}
|
||||
|
||||
|
||||
def CheckoutFactory(type_name, spec, root):
|
||||
"""Factory to build Checkout class instances."""
|
||||
class_ = CHECKOUT_TYPE_MAP.get(type_name)
|
||||
if not class_:
|
||||
raise KeyError('unrecognized checkout type: %s' % type_name)
|
||||
return class_(spec, root)
|
||||
|
||||
|
||||
#################################################
|
||||
# Utility function and file entry point.
|
||||
#################################################
|
||||
def usage(msg=None):
|
||||
"""Print help and exit."""
|
||||
if msg:
|
||||
print 'Error:', msg
|
||||
|
||||
print (
|
||||
"""
|
||||
usage: %s <recipe> [--property=value [--property2=value2 ...]]
|
||||
""" % os.path.basename(sys.argv[0]))
|
||||
sys.exit(bool(msg))
|
||||
|
||||
|
||||
def handle_args(argv):
|
||||
"""Gets the recipe name from the command line arguments."""
|
||||
if len(argv) <= 1:
|
||||
usage('Must specify a recipe.')
|
||||
|
||||
def looks_like_arg(arg):
|
||||
return arg.startswith('--') and arg.count('=') == 1
|
||||
|
||||
bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
|
||||
if bad_parms:
|
||||
usage('Got bad arguments %s' % bad_parms)
|
||||
|
||||
recipe = argv[1]
|
||||
props = argv[2:]
|
||||
return recipe, props
|
||||
|
||||
|
||||
def run_recipe_fetch(recipe, props, aliased=False):
|
||||
"""Invoke a recipe's fetch method with the passed-through args
|
||||
and return its json output as a python object."""
|
||||
recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe))
|
||||
cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props
|
||||
result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
spec = json.loads(result)
|
||||
if 'alias' in spec:
|
||||
assert not aliased
|
||||
return run_recipe_fetch(
|
||||
spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True)
|
||||
cmd = [sys.executable, recipe_path + '.py', 'root']
|
||||
result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
root = json.loads(result)
|
||||
return spec, root
|
||||
|
||||
|
||||
def run(spec, root):
|
||||
"""Perform a checkout with the given type and configuration.
|
||||
|
||||
Args:
|
||||
spec: Checkout configuration returned by the the recipe's fetch_spec
|
||||
method (checkout type, repository url, etc.).
|
||||
root: The directory into which the repo expects to be checkout out.
|
||||
"""
|
||||
assert 'type' in spec
|
||||
checkout_type = spec['type']
|
||||
checkout_spec = spec['%s_spec' % checkout_type]
|
||||
try:
|
||||
checkout = CheckoutFactory(checkout_type, checkout_spec, root)
|
||||
except KeyError:
|
||||
return 1
|
||||
if checkout.exists():
|
||||
print 'You appear to already have this checkout.'
|
||||
print 'Aborting to avoid clobbering your work.'
|
||||
return 1
|
||||
checkout.init()
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
recipe, props = handle_args(sys.argv)
|
||||
spec, root = run_recipe_fetch(recipe, props)
|
||||
return run(spec, root)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,48 @@
|
||||
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# pylint: disable=F0401
|
||||
import recipe_util
|
||||
import sys
|
||||
|
||||
# pylint: disable=W0232
|
||||
class Chromium(recipe_util.Recipe):
|
||||
"""Basic Recipe class for Chromium."""
|
||||
|
||||
@staticmethod
|
||||
def fetch_spec(props):
|
||||
url = 'https://chromium.googlesource.com/chromium/src.git'
|
||||
solution = { 'name' :'src',
|
||||
'url' : url,
|
||||
'deps_file': '.DEPS.git',
|
||||
'managed' : True,
|
||||
'custom_deps': {},
|
||||
'safesync_url': '',
|
||||
}
|
||||
if props.get('webkit_rev', '') == 'ToT':
|
||||
solution['custom_vars'] = {'webkit_rev': ''}
|
||||
spec = {
|
||||
'solutions': [solution],
|
||||
'svn_url': 'svn://svn.chromium.org/chrome',
|
||||
'svn_branch': 'trunk/src',
|
||||
'svn_ref': 'git-svn',
|
||||
}
|
||||
if props.get('submodule_git_svn_spec'):
|
||||
spec['submodule_git_svn_spec'] = props['submodule_git_svn_spec']
|
||||
return {
|
||||
'type': 'gclient_git_svn',
|
||||
'gclient_git_svn_spec': spec
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def expected_root(_props):
|
||||
return 'src'
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
return Chromium().handle_args(argv)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""This module holds utilities which make writing recipes easier."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
class Recipe(object):
|
||||
"""Base class for all recipes.
|
||||
|
||||
Provides methods that are expected to be overridden by child classes. Also
|
||||
provides an command-line parsing method that converts the unified command-line
|
||||
interface used in depot_tools to the unified python interface defined here."""
|
||||
|
||||
@staticmethod
|
||||
def fetch_spec(_props):
|
||||
"""Returns instructions to check out the project, conditioned on |props|."""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def expected_root(_props):
|
||||
"""Returns the directory into which the checkout will be performed."""
|
||||
raise NotImplementedError
|
||||
|
||||
def handle_args(self, argv):
|
||||
"""Passes the command-line arguments through to the appropriate method."""
|
||||
methods = {'fetch': self.fetch_spec,
|
||||
'root': self.expected_root}
|
||||
if len(argv) <= 1 or argv[1] not in methods:
|
||||
print 'Must specify a a fetch/root action'
|
||||
return 1
|
||||
|
||||
def looks_like_arg(arg):
|
||||
return arg.startswith('--') and arg.count('=') == 1
|
||||
|
||||
bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
|
||||
if bad_parms:
|
||||
print 'Got bad arguments %s' % bad_parms
|
||||
return 1
|
||||
|
||||
method = methods[argv[1]]
|
||||
props = dict(x.split('=', 1) for x in (y.lstrip('-') for y in argv[2:]))
|
||||
|
||||
self.output(method(props))
|
||||
|
||||
@staticmethod
|
||||
def output(data):
|
||||
print(json.dumps(data))
|
||||
Loading…
Reference in New Issue