From 38b04f839e0f50f1a4d5592f6257d3690142bcb4 Mon Sep 17 00:00:00 2001 From: "msb@chromium.org" Date: Tue, 13 Jul 2010 23:03:34 +0000 Subject: [PATCH] Add the repo tool which is going to be used for iterative tasks across multiple git repos Review URL: http://codereview.chromium.org/2792012 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@52240 0039d316-1c4b-4281-b951-d872f2087c98 --- repo | 596 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 596 insertions(+) create mode 100644 repo diff --git a/repo b/repo new file mode 100644 index 000000000..ff7c41888 --- /dev/null +++ b/repo @@ -0,0 +1,596 @@ +#!/bin/sh + +## repo default configuration +## +REPO_URL='git://android.git.kernel.org/tools/repo.git' +REPO_REV='stable' + +# Copyright (C) 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +magic='--calling-python-from-/bin/sh--' +"""exec" python -E "$0" "$@" """#$magic" +if __name__ == '__main__': + import sys + if sys.argv[-1] == '#%s' % magic: + del sys.argv[-1] +del magic + +# increment this whenever we make important changes to this script +VERSION = (1, 9) + +# increment this if the MAINTAINER_KEYS block is modified +KEYRING_VERSION = (1,0) +MAINTAINER_KEYS = """ + + Repo Maintainer +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.2.2 (GNU/Linux) + +mQGiBEj3ugERBACrLJh/ZPyVSKeClMuznFIrsQ+hpNnmJGw1a9GXKYKk8qHPhAZf +WKtrBqAVMNRLhL85oSlekRz98u41H5si5zcuv+IXJDF5MJYcB8f22wAy15lUqPWi +VCkk1l8qqLiuW0fo+ZkPY5qOgrvc0HW1SmdH649uNwqCbcKb6CxaTxzhOwCgj3AP +xI1WfzLqdJjsm1Nq98L0cLcD/iNsILCuw44PRds3J75YP0pze7YF/6WFMB6QSFGu +aUX1FsTTztKNXGms8i5b2l1B8JaLRWq/jOnZzyl1zrUJhkc0JgyZW5oNLGyWGhKD +Fxp5YpHuIuMImopWEMFIRQNrvlg+YVK8t3FpdI1RY0LYqha8pPzANhEYgSfoVzOb +fbfbA/4ioOrxy8ifSoga7ITyZMA+XbW8bx33WXutO9N7SPKS/AK2JpasSEVLZcON +ae5hvAEGVXKxVPDjJBmIc2cOe7kOKSi3OxLzBqrjS2rnjiP4o0ekhZIe4+ocwVOg +e0PLlH5avCqihGRhpoqDRsmpzSHzJIxtoeb+GgGEX8KkUsVAhbQpUmVwbyBNYWlu +dGFpbmVyIDxyZXBvQGFuZHJvaWQua2VybmVsLm9yZz6IYAQTEQIAIAUCSPe6AQIb +AwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEBZTDV6SD1xl1GEAn0x/OKQpy7qI +6G73NJviU0IUMtftAKCFMUhGb/0bZvQ8Rm3QCUpWHyEIu7kEDQRI97ogEBAA2wI6 +5fs9y/rMwD6dkD/vK9v4C9mOn1IL5JCPYMJBVSci+9ED4ChzYvfq7wOcj9qIvaE0 +GwCt2ar7Q56me5J+byhSb32Rqsw/r3Vo5cZMH80N4cjesGuSXOGyEWTe4HYoxnHv +gF4EKI2LK7xfTUcxMtlyn52sUpkfKsCpUhFvdmbAiJE+jCkQZr1Z8u2KphV79Ou+ +P1N5IXY/XWOlq48Qf4MWCYlJFrB07xjUjLKMPDNDnm58L5byDrP/eHysKexpbakL +xCmYyfT6DV1SWLblpd2hie0sL3YejdtuBMYMS2rI7Yxb8kGuqkz+9l1qhwJtei94 +5MaretDy/d/JH/pRYkRf7L+ke7dpzrP+aJmcz9P1e6gq4NJsWejaALVASBiioqNf +QmtqSVzF1wkR5avZkFHuYvj6V/t1RrOZTXxkSk18KFMJRBZrdHFCWbc5qrVxUB6e +N5pja0NFIUCigLBV1c6I2DwiuboMNh18VtJJh+nwWeez/RueN4ig59gRTtkcc0PR +35tX2DR8+xCCFVW/NcJ4PSePYzCuuLvp1vEDHnj41R52Fz51hgddT4rBsp0nL+5I +socSOIIezw8T9vVzMY4ArCKFAVu2IVyBcahTfBS8q5EM63mONU6UVJEozfGljiMw +xuQ7JwKcw0AUEKTKG7aBgBaTAgT8TOevpvlw91cAAwUP/jRkyVi/0WAb0qlEaq/S +ouWxX1faR+vU3b+Y2/DGjtXQMzG0qpetaTHC/AxxHpgt/dCkWI6ljYDnxgPLwG0a +Oasm94BjZc6vZwf1opFZUKsjOAAxRxNZyjUJKe4UZVuMTk6zo27Nt3LMnc0FO47v +FcOjRyquvgNOS818irVHUf12waDx8gszKxQTTtFxU5/ePB2jZmhP6oXSe4K/LG5T ++WBRPDrHiGPhCzJRzm9BP0lTnGCAj3o9W90STZa65RK7IaYpC8TB35JTBEbrrNCp +w6lzd74LnNEp5eMlKDnXzUAgAH0yzCQeMl7t33QCdYx2hRs2wtTQSjGfAiNmj/WW +Vl5Jn+2jCDnRLenKHwVRFsBX2e0BiRWt/i9Y8fjorLCXVj4z+7yW6DawdLkJorEo +p3v5ILwfC7hVx4jHSnOgZ65L9s8EQdVr1ckN9243yta7rNgwfcqb60ILMFF1BRk/ +0V7wCL+68UwwiQDvyMOQuqkysKLSDCLb7BFcyA7j6KG+5hpsREstFX2wK1yKeraz +5xGrFy8tfAaeBMIQ17gvFSp/suc9DYO0ICK2BISzq+F+ZiAKsjMYOBNdH/h0zobQ +HTHs37+/QLMomGEGKZMWi0dShU2J5mNRQu3Hhxl3hHDVbt5CeJBb26aQcQrFz69W +zE3GNvmJosh6leayjtI9P2A6iEkEGBECAAkFAkj3uiACGwwACgkQFlMNXpIPXGWp +TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2 +=CMiZ +-----END PGP PUBLIC KEY BLOCK----- +""" + +GIT = 'git' # our git command +MIN_GIT_VERSION = (1, 5, 4) # minimum supported git version +repodir = '.repo' # name of repo's private directory +S_repo = 'repo' # special repo reposiory +S_manifests = 'manifests' # special manifest repository +REPO_MAIN = S_repo + '/main.py' # main script + + +import optparse +import os +import re +import readline +import subprocess +import sys + +home_dot_repo = os.path.expanduser('~/.repoconfig') +gpg_dir = os.path.join(home_dot_repo, 'gnupg') + +extra_args = [] +init_optparse = optparse.OptionParser(usage="repo init -u url [options]") + +# Logging +group = init_optparse.add_option_group('Logging options') +group.add_option('-q', '--quiet', + dest="quiet", action="store_true", default=False, + help="be quiet") + +# Manifest +group = init_optparse.add_option_group('Manifest options') +group.add_option('-u', '--manifest-url', + dest='manifest_url', + help='manifest repository location', metavar='URL') +group.add_option('-o', '--origin', + dest='manifest_origin', + help="use REMOTE instead of 'origin' to track upstream", + metavar='REMOTE') +group.add_option('-b', '--manifest-branch', + dest='manifest_branch', + help='manifest branch or revision', metavar='REVISION') +group.add_option('-m', '--manifest-name', + dest='manifest_name', + help='initial manifest file (deprecated)', + metavar='NAME.xml') +group.add_option('--mirror', + dest='mirror', action='store_true', + help='mirror the forrest') + +# Tool +group = init_optparse.add_option_group('repo Version options') +group.add_option('--repo-url', + dest='repo_url', + help='repo repository location', metavar='URL') +group.add_option('--repo-branch', + dest='repo_branch', + help='repo branch or revision', metavar='REVISION') +group.add_option('--no-repo-verify', + dest='no_repo_verify', action='store_true', + help='do not verify repo source code') + + +class CloneFailure(Exception): + """Indicate the remote clone of repo itself failed. + """ + + +def _Init(args): + """Installs repo by cloning it over the network. + """ + opt, args = init_optparse.parse_args(args) + if args or not opt.manifest_url: + init_optparse.print_usage() + sys.exit(1) + + url = opt.repo_url + if not url: + url = REPO_URL + extra_args.append('--repo-url=%s' % url) + + branch = opt.repo_branch + if not branch: + branch = REPO_REV + extra_args.append('--repo-branch=%s' % branch) + + if branch.startswith('refs/heads/'): + branch = branch[len('refs/heads/'):] + if branch.startswith('refs/'): + print >>sys.stderr, "fatal: invalid branch name '%s'" % branch + raise CloneFailure() + + if not os.path.isdir(repodir): + try: + os.mkdir(repodir) + except OSError, e: + print >>sys.stderr, \ + 'fatal: cannot make %s directory: %s' % ( + repodir, e.strerror) + # Don't faise CloneFailure; that would delete the + # name. Instead exit immediately. + # + sys.exit(1) + + _CheckGitVersion() + try: + if _NeedSetupGnuPG(): + can_verify = _SetupGnuPG(opt.quiet) + else: + can_verify = True + + if not opt.quiet: + print >>sys.stderr, 'Getting repo ...' + print >>sys.stderr, ' from %s' % url + + dst = os.path.abspath(os.path.join(repodir, S_repo)) + _Clone(url, dst, opt.quiet) + + if can_verify and not opt.no_repo_verify: + rev = _Verify(dst, branch, opt.quiet) + else: + rev = 'refs/remotes/origin/%s^0' % branch + + _Checkout(dst, branch, rev, opt.quiet) + except CloneFailure: + if opt.quiet: + print >>sys.stderr, \ + 'fatal: repo init failed; run without --quiet to see why' + raise + + +def _CheckGitVersion(): + cmd = [GIT, '--version'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + ver_str = proc.stdout.read().strip() + proc.stdout.close() + proc.wait() + + if not ver_str.startswith('git version '): + print >>sys.stderr, 'error: "%s" unsupported' % ver_str + raise CloneFailure() + + ver_str = ver_str[len('git version '):].strip() + ver_act = tuple(map(lambda x: int(x), ver_str.split('.')[0:3])) + if ver_act < MIN_GIT_VERSION: + need = '.'.join(map(lambda x: str(x), MIN_GIT_VERSION)) + print >>sys.stderr, 'fatal: git %s or later required' % need + raise CloneFailure() + + +def _NeedSetupGnuPG(): + if not os.path.isdir(home_dot_repo): + return True + + kv = os.path.join(home_dot_repo, 'keyring-version') + if not os.path.exists(kv): + return True + + kv = open(kv).read() + if not kv: + return True + + kv = tuple(map(lambda x: int(x), kv.split('.'))) + if kv < KEYRING_VERSION: + return True + return False + + +def _SetupGnuPG(quiet): + if not os.path.isdir(home_dot_repo): + try: + os.mkdir(home_dot_repo) + except OSError, e: + print >>sys.stderr, \ + 'fatal: cannot make %s directory: %s' % ( + home_dot_repo, e.strerror) + sys.exit(1) + + if not os.path.isdir(gpg_dir): + try: + os.mkdir(gpg_dir, 0700) + except OSError, e: + print >>sys.stderr, \ + 'fatal: cannot make %s directory: %s' % ( + gpg_dir, e.strerror) + sys.exit(1) + + env = dict(os.environ) + env['GNUPGHOME'] = gpg_dir + + cmd = ['gpg', '--import'] + try: + proc = subprocess.Popen(cmd, + env = env, + stdin = subprocess.PIPE) + except OSError, e: + if not quiet: + print >>sys.stderr, 'warning: gpg (GnuPG) is not available.' + print >>sys.stderr, 'warning: Installing it is strongly encouraged.' + print >>sys.stderr + return False + + proc.stdin.write(MAINTAINER_KEYS) + proc.stdin.close() + + if proc.wait() != 0: + print >>sys.stderr, 'fatal: registering repo maintainer keys failed' + sys.exit(1) + print + + fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w') + fd.write('.'.join(map(lambda x: str(x), KEYRING_VERSION)) + '\n') + fd.close() + return True + + +def _SetConfig(local, name, value): + """Set a git configuration option to the specified value. + """ + cmd = [GIT, 'config', name, value] + if subprocess.Popen(cmd, cwd = local).wait() != 0: + raise CloneFailure() + + +def _Fetch(local, quiet, *args): + cmd = [GIT, 'fetch'] + if quiet: + cmd.append('--quiet') + err = subprocess.PIPE + else: + err = None + cmd.extend(args) + cmd.append('origin') + + proc = subprocess.Popen(cmd, cwd = local, stderr = err) + if err: + proc.stderr.read() + proc.stderr.close() + if proc.wait() != 0: + raise CloneFailure() + + +def _Clone(url, local, quiet): + """Clones a git repository to a new subdirectory of repodir + """ + try: + os.mkdir(local) + except OSError, e: + print >>sys.stderr, \ + 'fatal: cannot make %s directory: %s' \ + % (local, e.strerror) + raise CloneFailure() + + cmd = [GIT, 'init', '--quiet'] + try: + proc = subprocess.Popen(cmd, cwd = local) + except OSError, e: + print >>sys.stderr + print >>sys.stderr, "fatal: '%s' is not available" % GIT + print >>sys.stderr, 'fatal: %s' % e + print >>sys.stderr + print >>sys.stderr, 'Please make sure %s is installed'\ + ' and in your path.' % GIT + raise CloneFailure() + if proc.wait() != 0: + print >>sys.stderr, 'fatal: could not create %s' % local + raise CloneFailure() + + _SetConfig(local, 'remote.origin.url', url) + _SetConfig(local, 'remote.origin.fetch', + '+refs/heads/*:refs/remotes/origin/*') + _Fetch(local, quiet) + _Fetch(local, quiet, '--tags') + + +def _Verify(cwd, branch, quiet): + """Verify the branch has been signed by a tag. + """ + cmd = [GIT, 'describe', 'origin/%s' % branch] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd = cwd) + cur = proc.stdout.read().strip() + proc.stdout.close() + + proc.stderr.read() + proc.stderr.close() + + if proc.wait() != 0 or not cur: + print >>sys.stderr + print >>sys.stderr,\ + "fatal: branch '%s' has not been signed" \ + % branch + raise CloneFailure() + + m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur) + if m: + cur = m.group(1) + if not quiet: + print >>sys.stderr + print >>sys.stderr, \ + "info: Ignoring branch '%s'; using tagged release '%s'" \ + % (branch, cur) + print >>sys.stderr + + env = dict(os.environ) + env['GNUPGHOME'] = gpg_dir + + cmd = [GIT, 'tag', '-v', cur] + proc = subprocess.Popen(cmd, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + cwd = cwd, + env = env) + out = proc.stdout.read() + proc.stdout.close() + + err = proc.stderr.read() + proc.stderr.close() + + if proc.wait() != 0: + print >>sys.stderr + print >>sys.stderr, out + print >>sys.stderr, err + print >>sys.stderr + raise CloneFailure() + return '%s^0' % cur + + +def _Checkout(cwd, branch, rev, quiet): + """Checkout an upstream branch into the repository and track it. + """ + cmd = [GIT, 'update-ref', 'refs/heads/default', rev] + if subprocess.Popen(cmd, cwd = cwd).wait() != 0: + raise CloneFailure() + + _SetConfig(cwd, 'branch.default.remote', 'origin') + _SetConfig(cwd, 'branch.default.merge', 'refs/heads/%s' % branch) + + cmd = [GIT, 'symbolic-ref', 'HEAD', 'refs/heads/default'] + if subprocess.Popen(cmd, cwd = cwd).wait() != 0: + raise CloneFailure() + + cmd = [GIT, 'read-tree', '--reset', '-u'] + if not quiet: + cmd.append('-v') + cmd.append('HEAD') + if subprocess.Popen(cmd, cwd = cwd).wait() != 0: + raise CloneFailure() + + +def _FindRepo(): + """Look for a repo installation, starting at the current directory. + """ + dir = os.getcwd() + repo = None + + while dir != '/' and not repo: + repo = os.path.join(dir, repodir, REPO_MAIN) + if not os.path.isfile(repo): + repo = None + dir = os.path.dirname(dir) + return (repo, os.path.join(dir, repodir)) + + +class _Options: + help = False + + +def _ParseArguments(args): + cmd = None + opt = _Options() + arg = [] + + for i in xrange(0, len(args)): + a = args[i] + if a == '-h' or a == '--help': + opt.help = True + + elif not a.startswith('-'): + cmd = a + arg = args[i + 1:] + break + return cmd, opt, arg + + +def _Usage(): + print >>sys.stderr,\ +"""usage: repo COMMAND [ARGS] + +repo is not yet installed. Use "repo init" to install it here. + +The most commonly used repo commands are: + + init Install repo in the current working directory + help Display detailed help on a command + +For access to the full online help, install repo ("repo init"). +""" + sys.exit(1) + + +def _Help(args): + if args: + if args[0] == 'init': + init_optparse.print_help() + else: + print >>sys.stderr,\ + "error: '%s' is not a bootstrap command.\n"\ + ' For access to online help, install repo ("repo init").'\ + % args[0] + else: + _Usage() + sys.exit(1) + + +def _NotInstalled(): + print >>sys.stderr,\ +'error: repo is not installed. Use "repo init" to install it here.' + sys.exit(1) + + +def _NoCommands(cmd): + print >>sys.stderr,\ +"""error: command '%s' requires repo to be installed first. + Use "repo init" to install it here.""" % cmd + sys.exit(1) + + +def _RunSelf(wrapper_path): + my_dir = os.path.dirname(wrapper_path) + my_main = os.path.join(my_dir, 'main.py') + my_git = os.path.join(my_dir, '.git') + + if os.path.isfile(my_main) and os.path.isdir(my_git): + for name in ['git_config.py', + 'project.py', + 'subcmds']: + if not os.path.exists(os.path.join(my_dir, name)): + return None, None + return my_main, my_git + return None, None + + +def _SetDefaultsTo(gitdir): + global REPO_URL + global REPO_REV + + REPO_URL = gitdir + proc = subprocess.Popen([GIT, + '--git-dir=%s' % gitdir, + 'symbolic-ref', + 'HEAD'], + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + REPO_REV = proc.stdout.read().strip() + proc.stdout.close() + + proc.stderr.read() + proc.stderr.close() + + if proc.wait() != 0: + print >>sys.stderr, 'fatal: %s has no current branch' % gitdir + sys.exit(1) + + +def main(orig_args): + main, dir = _FindRepo() + cmd, opt, args = _ParseArguments(orig_args) + + wrapper_path = os.path.abspath(__file__) + my_main, my_git = _RunSelf(wrapper_path) + + if not main: + if opt.help: + _Usage() + if cmd == 'help': + _Help(args) + if not cmd: + _NotInstalled() + if cmd == 'init': + if my_git: + _SetDefaultsTo(my_git) + try: + _Init(args) + except CloneFailure: + for root, dirs, files in os.walk(repodir, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(repodir) + sys.exit(1) + main, dir = _FindRepo() + else: + _NoCommands(cmd) + + if my_main: + main = my_main + + ver_str = '.'.join(map(lambda x: str(x), VERSION)) + me = [main, + '--repo-dir=%s' % dir, + '--wrapper-version=%s' % ver_str, + '--wrapper-path=%s' % wrapper_path, + '--'] + me.extend(orig_args) + me.extend(extra_args) + try: + os.execv(main, me) + except OSError, e: + print >>sys.stderr, "fatal: unable to start %s" % main + print >>sys.stderr, "fatal: %s" % e + sys.exit(148) + + +if __name__ == '__main__': + main(sys.argv[1:]) +