Create "git cache" command.
The git cache command is a central place to manage a machine's git cache. It provides two subcommands: * populate -- creates or updates the cache of a given repository * exists -- prints the path to the cache of a repo, if it exists Gclient, deps2git, bot_update, and any other tools that touch the cache will be able to use this command to make sure that everyone is interacting with the cache in the same way. R=hinoka@chromium.org, iannucci@chromium.org BUG=339168 Review URL: https://codereview.chromium.org/164823002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@253007 0039d316-1c4b-4281-b951-d872f2087c98experimental/szager/collated-output
agable@chromium.org
12 years ago
parent
b460ebe723
commit
5a306a2aa6
@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# Copyright 2014 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# git-cache -- a git-command for managing local caches of remote repositories.
|
||||
|
||||
# Test if this script is running under a MSys install. If it is, we will
|
||||
# hardcode the path to Python where possible.
|
||||
OUTPUT="$(uname | grep 'MINGW')"
|
||||
MINGW=$?
|
||||
|
||||
if [ $MINGW = 0 ]; then
|
||||
base_dir="${0%\\*}"
|
||||
else
|
||||
base_dir=$(dirname "$0")
|
||||
fi
|
||||
|
||||
# Uncomment this line if you never use gclient.
|
||||
# "$base_dir"/update_depot_tools
|
||||
|
||||
if [ -e "$base_dir/python.bat" -a $MINGW = 0 ]; then
|
||||
PYTHONDONTWRITEBYTECODE=1 cmd.exe //c "$base_dir\\python.bat" "$base_dir\\git_cache.py" "$@"
|
||||
else
|
||||
PYTHONDONTWRITEBYTECODE=1 exec "$base_dir/git_cache.py" "$@"
|
||||
fi
|
||||
@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2014 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""A git command for managing a local cache of git repositories."""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import tempfile
|
||||
import subprocess
|
||||
import sys
|
||||
import urlparse
|
||||
|
||||
import gclient_utils
|
||||
import subcommand
|
||||
|
||||
|
||||
GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
|
||||
|
||||
|
||||
def NormalizeUrl(url):
|
||||
"""Convert a git url to a normalized form."""
|
||||
parsed = urlparse.urlparse(url)
|
||||
norm_url = 'https://' + parsed.netloc + parsed.path
|
||||
if not norm_url.endswith('.git'):
|
||||
norm_url += '.git'
|
||||
return norm_url
|
||||
|
||||
|
||||
def UrlToCacheDir(url):
|
||||
"""Convert a git url to a normalized form for the cache dir path."""
|
||||
parsed = urlparse.urlparse(url)
|
||||
norm_url = parsed.netloc + parsed.path
|
||||
if norm_url.endswith('.git'):
|
||||
norm_url = norm_url[:-len('.git')]
|
||||
return norm_url.replace('-', '--').replace('/', '-').lower()
|
||||
|
||||
|
||||
def RunGit(cmd, **kwargs):
|
||||
"""Run git in a subprocess."""
|
||||
kwargs.setdefault('cwd', os.getcwd())
|
||||
if kwargs.get('filter_fn'):
|
||||
kwargs['filter_fn'] = gclient_utils.GitFilter(kwargs.get('filter_fn'))
|
||||
kwargs.setdefault('print_stdout', False)
|
||||
env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
|
||||
env.setdefault('GIT_ASKPASS', 'true')
|
||||
env.setdefault('SSH_ASKPASS', 'true')
|
||||
else:
|
||||
kwargs.setdefault('print_stdout', True)
|
||||
stdout = kwargs.get('stdout', sys.stdout)
|
||||
print >> stdout, 'running "git %s" in "%s"' % (' '.join(cmd), kwargs['cwd'])
|
||||
gclient_utils.CheckCallAndFilter([GIT_EXECUTABLE] + cmd, **kwargs)
|
||||
|
||||
|
||||
class LockError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Lockfile(object):
|
||||
"""Class to represent a cross-platform process-specific lockfile."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = os.path.abspath(path)
|
||||
self.lockfile = self.path + ".lock"
|
||||
self.pid = os.getpid()
|
||||
|
||||
def _read_pid(self):
|
||||
"""Read the pid stored in the lockfile.
|
||||
|
||||
Note: This method is potentially racy. By the time it returns the lockfile
|
||||
may have been unlocked, removed, or stolen by some other process.
|
||||
"""
|
||||
try:
|
||||
with open(self.lockfile, 'r') as f:
|
||||
pid = int(f.readline().strip())
|
||||
except (IOError, ValueError):
|
||||
pid = None
|
||||
return pid
|
||||
|
||||
def _make_lockfile(self):
|
||||
"""Safely creates a lockfile containing the current pid."""
|
||||
open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
fd = os.open(self.lockfile, open_flags, 0o644)
|
||||
f = os.fdopen(fd, 'w')
|
||||
print >> f, self.pid
|
||||
f.close()
|
||||
|
||||
def _remove_lockfile(self):
|
||||
"""Delete the lockfile. Complains (implicitly) if it doesn't exist."""
|
||||
os.remove(self.lockfile)
|
||||
|
||||
def lock(self):
|
||||
"""Acquire the lock.
|
||||
|
||||
Note: This is a NON-BLOCKING FAIL-FAST operation.
|
||||
Do. Or do not. There is no try.
|
||||
"""
|
||||
try:
|
||||
self._make_lockfile()
|
||||
except OSError as e:
|
||||
if e.errno == errno.EEXIST:
|
||||
raise LockError("%s is already locked" % self.path)
|
||||
else:
|
||||
raise LockError("Failed to create %s (err %s)" % (self.path, e.errno))
|
||||
|
||||
def unlock(self):
|
||||
"""Release the lock."""
|
||||
if not self.is_locked():
|
||||
raise LockError("%s is not locked" % self.path)
|
||||
if not self.i_am_locking():
|
||||
raise LockError("%s is locked, but not by me" % self.path)
|
||||
self._remove_lockfile()
|
||||
|
||||
def break_lock(self):
|
||||
"""Remove the lock, even if it was created by someone else."""
|
||||
try:
|
||||
self._remove_lockfile()
|
||||
return True
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENOENT:
|
||||
return False
|
||||
else:
|
||||
raise
|
||||
|
||||
def is_locked(self):
|
||||
"""Test if the file is locked by anyone.
|
||||
|
||||
Note: This method is potentially racy. By the time it returns the lockfile
|
||||
may have been unlocked, removed, or stolen by some other process.
|
||||
"""
|
||||
return os.path.exists(self.lockfile)
|
||||
|
||||
def i_am_locking(self):
|
||||
"""Test if the file is locked by this process."""
|
||||
return self.is_locked() and self.pid == self._read_pid()
|
||||
|
||||
def __enter__(self):
|
||||
self.lock()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
self.unlock()
|
||||
|
||||
|
||||
@subcommand.usage('[url of repo to check for caching]')
|
||||
def CMDexists(parser, args):
|
||||
"""Check to see if there already is a cache of the given repo."""
|
||||
options, args = parser.parse_args(args)
|
||||
if not len(args) == 1:
|
||||
parser.error('git cache exists only takes exactly one repo url.')
|
||||
url = args[0]
|
||||
repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
|
||||
flag_file = os.path.join(repo_dir, 'config')
|
||||
if os.path.isdir(repo_dir) and os.path.isfile(flag_file):
|
||||
print repo_dir
|
||||
return 0
|
||||
return 1
|
||||
|
||||
|
||||
@subcommand.usage('[url of repo to add to or update in cache]')
|
||||
def CMDpopulate(parser, args):
|
||||
"""Ensure that the cache has all up-to-date objects for the given repo."""
|
||||
parser.add_option('--depth', type='int',
|
||||
help='Only cache DEPTH commits of history')
|
||||
parser.add_option('--shallow', '-s', action='store_true',
|
||||
help='Only cache 10000 commits of history')
|
||||
parser.add_option('--ref', action='append',
|
||||
help='Specify additional refs to be fetched')
|
||||
options, args = parser.parse_args(args)
|
||||
if options.shallow and not options.depth:
|
||||
options.depth = 10000
|
||||
if not len(args) == 1:
|
||||
parser.error('git cache populate only takes exactly one repo url.')
|
||||
url = args[0]
|
||||
|
||||
gclient_utils.safe_makedirs(options.cache_dir)
|
||||
repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
|
||||
|
||||
v = []
|
||||
filter_fn = lambda l: '[up to date]' not in l
|
||||
if options.verbose:
|
||||
v = ['-v', '--progress']
|
||||
filter_fn = None
|
||||
|
||||
d = []
|
||||
if options.depth:
|
||||
d = ['--depth', '%d' % options.depth]
|
||||
|
||||
def _config(directory):
|
||||
RunGit(['config', 'core.deltaBaseCacheLimit', '2g'],
|
||||
cwd=directory)
|
||||
RunGit(['config', 'remote.origin.url', NormalizeUrl(url)],
|
||||
cwd=directory)
|
||||
RunGit(['config', '--replace-all', 'remote.origin.fetch',
|
||||
'+refs/heads/*:refs/heads/*'],
|
||||
cwd=directory)
|
||||
for ref in options.ref or []:
|
||||
ref = ref.rstrip('/')
|
||||
refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
|
||||
RunGit(['config', '--add', 'remote.origin.fetch', refspec],
|
||||
cwd=directory)
|
||||
|
||||
with Lockfile(repo_dir):
|
||||
# Setup from scratch if the repo is new or is in a bad state.
|
||||
if not os.path.exists(os.path.join(repo_dir, 'config')):
|
||||
gclient_utils.rmtree(repo_dir)
|
||||
tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
|
||||
dir=options.cache_dir)
|
||||
RunGit(['init', '--bare'], cwd=tempdir)
|
||||
_config(tempdir)
|
||||
fetch_cmd = ['fetch'] + v + d + ['--tags', 'origin']
|
||||
RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
|
||||
os.rename(tempdir, repo_dir)
|
||||
else:
|
||||
_config(repo_dir)
|
||||
if options.depth and os.path.exists(os.path.join(repo_dir, 'shallow')):
|
||||
logging.warn('Shallow fetch requested, but repo cache already exists.')
|
||||
fetch_cmd = ['fetch'] + v + ['--update-shallow', '--tags', 'origin']
|
||||
RunGit(fetch_cmd, filter_fn=filter_fn, cwd=repo_dir, retry=True)
|
||||
|
||||
|
||||
@subcommand.usage('[url of repo to unlock, or -a|--all]')
|
||||
def CMDunlock(parser, args):
|
||||
"""Unlock one or all repos if their lock files are still around."""
|
||||
parser.add_option('--force', '-f', action='store_true',
|
||||
help='Actually perform the action')
|
||||
parser.add_option('--all', '-a', action='store_true',
|
||||
help='Unlock all repository caches')
|
||||
options, args = parser.parse_args(args)
|
||||
if len(args) > 1 or (len(args) == 0 and not options.all):
|
||||
parser.error('git cache unlock takes exactly one repo url, or --all')
|
||||
|
||||
if not options.all:
|
||||
url = args[0]
|
||||
repo_dirs = [os.path.join(options.cache_dir, UrlToCacheDir(url))]
|
||||
else:
|
||||
repo_dirs = [path for path in os.listdir(options.cache_dir)
|
||||
if os.path.isdir(path)]
|
||||
lockfiles = [repo_dir + '.lock' for repo_dir in repo_dirs
|
||||
if os.path.exists(repo_dir + '.lock')]
|
||||
|
||||
if not options.force:
|
||||
parser.error('git cache unlock requires -f|--force to do anything. '
|
||||
'Refusing to unlock the following repo caches: '
|
||||
', '.join(lockfiles))
|
||||
|
||||
unlocked = []
|
||||
untouched = []
|
||||
for repo_dir in repo_dirs:
|
||||
lf = Lockfile(repo_dir)
|
||||
if lf.break_lock():
|
||||
unlocked.append(repo_dir)
|
||||
else:
|
||||
untouched.append(repo_dir)
|
||||
|
||||
if unlocked:
|
||||
logging.info('Broke locks on these caches: %s' % unlocked)
|
||||
if untouched:
|
||||
logging.debug('Did not touch these caches: %s' % untouched)
|
||||
|
||||
|
||||
class OptionParser(optparse.OptionParser):
|
||||
"""Wrapper class for OptionParser to handle global options."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
|
||||
self.add_option('-c', '--cache-dir',
|
||||
help='Path to the directory containing the cache')
|
||||
self.add_option('-v', '--verbose', action='count', default=0,
|
||||
help='Increase verbosity (can be passed multiple times)')
|
||||
|
||||
def parse_args(self, args=None, values=None):
|
||||
options, args = optparse.OptionParser.parse_args(self, args, values)
|
||||
|
||||
try:
|
||||
global_cache_dir = subprocess.check_output(
|
||||
[GIT_EXECUTABLE, 'config', '--global', 'cache.cachepath']).strip()
|
||||
if options.cache_dir:
|
||||
logging.warn('Overriding globally-configured cache directory.')
|
||||
else:
|
||||
options.cache_dir = global_cache_dir
|
||||
except subprocess.CalledProcessError:
|
||||
if not options.cache_dir:
|
||||
self.error('No cache directory specified on command line '
|
||||
'or in cache.cachepath.')
|
||||
options.cache_dir = os.path.abspath(options.cache_dir)
|
||||
|
||||
levels = [logging.WARNING, logging.INFO, logging.DEBUG]
|
||||
logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
|
||||
|
||||
return options, args
|
||||
|
||||
|
||||
def main(argv):
|
||||
dispatcher = subcommand.CommandDispatcher(__name__)
|
||||
return dispatcher.execute(OptionParser(), argv)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Loading…
Reference in New Issue