You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
739 lines
25 KiB
Python
739 lines
25 KiB
Python
#!/usr/bin/env python
|
|
# Copyright 2014 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""A git command for managing a local cache of git repositories."""
|
|
|
|
from __future__ import print_function
|
|
import errno
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import subprocess
|
|
import sys
|
|
import urlparse
|
|
import zipfile
|
|
|
|
from download_from_google_storage import Gsutil
|
|
import gclient_utils
|
|
import subcommand
|
|
|
|
# Analogous to gc.autopacklimit git config.
|
|
GC_AUTOPACKLIMIT = 50
|
|
|
|
GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.'
|
|
|
|
try:
|
|
# pylint: disable=E0602
|
|
WinErr = WindowsError
|
|
except NameError:
|
|
class WinErr(Exception):
|
|
pass
|
|
|
|
class LockError(Exception):
|
|
pass
|
|
|
|
class ClobberNeeded(Exception):
|
|
pass
|
|
|
|
class Lockfile(object):
|
|
"""Class to represent a cross-platform process-specific lockfile."""
|
|
|
|
def __init__(self, path, timeout=0):
|
|
self.path = os.path.abspath(path)
|
|
self.timeout = timeout
|
|
self.lockfile = self.path + ".lock"
|
|
self.pid = os.getpid()
|
|
|
|
def _read_pid(self):
|
|
"""Read the pid stored in the lockfile.
|
|
|
|
Note: This method is potentially racy. By the time it returns the lockfile
|
|
may have been unlocked, removed, or stolen by some other process.
|
|
"""
|
|
try:
|
|
with open(self.lockfile, 'r') as f:
|
|
pid = int(f.readline().strip())
|
|
except (IOError, ValueError):
|
|
pid = None
|
|
return pid
|
|
|
|
def _make_lockfile(self):
|
|
"""Safely creates a lockfile containing the current pid."""
|
|
open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
fd = os.open(self.lockfile, open_flags, 0o644)
|
|
f = os.fdopen(fd, 'w')
|
|
print(self.pid, file=f)
|
|
f.close()
|
|
|
|
def _remove_lockfile(self):
|
|
"""Delete the lockfile. Complains (implicitly) if it doesn't exist.
|
|
|
|
See gclient_utils.py:rmtree docstring for more explanation on the
|
|
windows case.
|
|
"""
|
|
if sys.platform == 'win32':
|
|
lockfile = os.path.normcase(self.lockfile)
|
|
for _ in xrange(3):
|
|
exitcode = subprocess.call(['cmd.exe', '/c',
|
|
'del', '/f', '/q', lockfile])
|
|
if exitcode == 0:
|
|
return
|
|
time.sleep(3)
|
|
raise LockError('Failed to remove lock: %s' % lockfile)
|
|
else:
|
|
os.remove(self.lockfile)
|
|
|
|
def lock(self):
|
|
"""Acquire the lock.
|
|
|
|
This will block with a deadline of self.timeout seconds.
|
|
"""
|
|
elapsed = 0
|
|
while True:
|
|
try:
|
|
self._make_lockfile()
|
|
return
|
|
except OSError as e:
|
|
if elapsed < self.timeout:
|
|
sleep_time = max(10, min(3, self.timeout - elapsed))
|
|
logging.info('Could not create git cache lockfile; '
|
|
'will retry after sleep(%d).', sleep_time);
|
|
elapsed += sleep_time
|
|
time.sleep(sleep_time)
|
|
continue
|
|
if e.errno == errno.EEXIST:
|
|
raise LockError("%s is already locked" % self.path)
|
|
else:
|
|
raise LockError("Failed to create %s (err %s)" % (self.path, e.errno))
|
|
|
|
def unlock(self):
|
|
"""Release the lock."""
|
|
try:
|
|
if not self.is_locked():
|
|
raise LockError("%s is not locked" % self.path)
|
|
if not self.i_am_locking():
|
|
raise LockError("%s is locked, but not by me" % self.path)
|
|
self._remove_lockfile()
|
|
except WinErr:
|
|
# Windows is unreliable when it comes to file locking. YMMV.
|
|
pass
|
|
|
|
def break_lock(self):
|
|
"""Remove the lock, even if it was created by someone else."""
|
|
try:
|
|
self._remove_lockfile()
|
|
return True
|
|
except OSError as exc:
|
|
if exc.errno == errno.ENOENT:
|
|
return False
|
|
else:
|
|
raise
|
|
|
|
def is_locked(self):
|
|
"""Test if the file is locked by anyone.
|
|
|
|
Note: This method is potentially racy. By the time it returns the lockfile
|
|
may have been unlocked, removed, or stolen by some other process.
|
|
"""
|
|
return os.path.exists(self.lockfile)
|
|
|
|
def i_am_locking(self):
|
|
"""Test if the file is locked by this process."""
|
|
return self.is_locked() and self.pid == self._read_pid()
|
|
|
|
|
|
class Mirror(object):
|
|
|
|
git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
|
|
gsutil_exe = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
|
|
cachepath_lock = threading.Lock()
|
|
|
|
@staticmethod
|
|
def parse_fetch_spec(spec):
|
|
"""Parses and canonicalizes a fetch spec.
|
|
|
|
Returns (fetchspec, value_regex), where value_regex can be used
|
|
with 'git config --replace-all'.
|
|
"""
|
|
parts = spec.split(':', 1)
|
|
src = parts[0].lstrip('+').rstrip('/')
|
|
if not src.startswith('refs/'):
|
|
src = 'refs/heads/%s' % src
|
|
dest = parts[1].rstrip('/') if len(parts) > 1 else src
|
|
regex = r'\+%s:.*' % src.replace('*', r'\*')
|
|
return ('+%s:%s' % (src, dest), regex)
|
|
|
|
def __init__(self, url, refs=None, print_func=None):
|
|
self.url = url
|
|
self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
|
|
self.basedir = self.UrlToCacheDir(url)
|
|
self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
|
|
if print_func:
|
|
self.print = self.print_without_file
|
|
self.print_func = print_func
|
|
else:
|
|
self.print = print
|
|
|
|
def print_without_file(self, message, **kwargs):
|
|
self.print_func(message)
|
|
|
|
@property
|
|
def bootstrap_bucket(self):
|
|
if 'chrome-internal' in self.url:
|
|
return 'chrome-git-cache'
|
|
else:
|
|
return 'chromium-git-cache'
|
|
|
|
@classmethod
|
|
def FromPath(cls, path):
|
|
return cls(cls.CacheDirToUrl(path))
|
|
|
|
@staticmethod
|
|
def UrlToCacheDir(url):
|
|
"""Convert a git url to a normalized form for the cache dir path."""
|
|
parsed = urlparse.urlparse(url)
|
|
norm_url = parsed.netloc + parsed.path
|
|
if norm_url.endswith('.git'):
|
|
norm_url = norm_url[:-len('.git')]
|
|
return norm_url.replace('-', '--').replace('/', '-').lower()
|
|
|
|
@staticmethod
|
|
def CacheDirToUrl(path):
|
|
"""Convert a cache dir path to its corresponding url."""
|
|
netpath = re.sub(r'\b-\b', '/', os.path.basename(path)).replace('--', '-')
|
|
return 'https://%s' % netpath
|
|
|
|
@classmethod
|
|
def SetCachePath(cls, cachepath):
|
|
with cls.cachepath_lock:
|
|
setattr(cls, 'cachepath', cachepath)
|
|
|
|
@classmethod
|
|
def GetCachePath(cls):
|
|
with cls.cachepath_lock:
|
|
if not hasattr(cls, 'cachepath'):
|
|
try:
|
|
cachepath = subprocess.check_output(
|
|
[cls.git_exe, 'config', '--global', 'cache.cachepath']).strip()
|
|
except subprocess.CalledProcessError:
|
|
cachepath = None
|
|
if not cachepath:
|
|
raise RuntimeError(
|
|
'No global cache.cachepath git configuration found.')
|
|
setattr(cls, 'cachepath', cachepath)
|
|
return getattr(cls, 'cachepath')
|
|
|
|
def RunGit(self, cmd, **kwargs):
|
|
"""Run git in a subprocess."""
|
|
cwd = kwargs.setdefault('cwd', self.mirror_path)
|
|
kwargs.setdefault('print_stdout', False)
|
|
kwargs.setdefault('filter_fn', self.print)
|
|
env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
|
|
env.setdefault('GIT_ASKPASS', 'true')
|
|
env.setdefault('SSH_ASKPASS', 'true')
|
|
self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
|
|
gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
|
|
|
|
def config(self, cwd=None):
|
|
if cwd is None:
|
|
cwd = self.mirror_path
|
|
|
|
# Don't run git-gc in a daemon. Bad things can happen if it gets killed.
|
|
try:
|
|
self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd)
|
|
except subprocess.CalledProcessError:
|
|
# Hard error, need to clobber.
|
|
raise ClobberNeeded()
|
|
|
|
# Don't combine pack files into one big pack file. It's really slow for
|
|
# repositories, and there's no way to track progress and make sure it's
|
|
# not stuck.
|
|
self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd)
|
|
|
|
# Allocate more RAM for cache-ing delta chains, for better performance
|
|
# of "Resolving deltas".
|
|
self.RunGit(['config', 'core.deltaBaseCacheLimit',
|
|
gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
|
|
|
|
self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
|
|
self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
|
|
'+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd)
|
|
for spec, value_regex in self.fetch_specs:
|
|
self.RunGit(
|
|
['config', '--replace-all', 'remote.origin.fetch', spec, value_regex],
|
|
cwd=cwd)
|
|
|
|
def bootstrap_repo(self, directory):
|
|
"""Bootstrap the repo from Google Stroage if possible.
|
|
|
|
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
|
|
"""
|
|
|
|
python_fallback = False
|
|
if (sys.platform.startswith('win') and
|
|
not gclient_utils.FindExecutable('7z')):
|
|
python_fallback = True
|
|
elif sys.platform.startswith('darwin'):
|
|
# The OSX version of unzip doesn't support zip64.
|
|
python_fallback = True
|
|
elif not gclient_utils.FindExecutable('unzip'):
|
|
python_fallback = True
|
|
|
|
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
|
|
gsutil = Gsutil(self.gsutil_exe, boto_path=None)
|
|
# Get the most recent version of the zipfile.
|
|
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
|
|
ls_out_sorted = sorted(ls_out.splitlines())
|
|
if not ls_out_sorted:
|
|
# This repo is not on Google Storage.
|
|
return False
|
|
latest_checkout = ls_out_sorted[-1]
|
|
|
|
# Download zip file to a temporary directory.
|
|
try:
|
|
tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
|
|
self.print('Downloading %s' % latest_checkout)
|
|
code = gsutil.call('cp', latest_checkout, tempdir)
|
|
if code:
|
|
return False
|
|
filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
|
|
|
|
# Unpack the file with 7z on Windows, unzip on linux, or fallback.
|
|
if not python_fallback:
|
|
if sys.platform.startswith('win'):
|
|
cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
|
|
else:
|
|
cmd = ['unzip', filename, '-d', directory]
|
|
retcode = subprocess.call(cmd)
|
|
else:
|
|
try:
|
|
with zipfile.ZipFile(filename, 'r') as f:
|
|
f.printdir()
|
|
f.extractall(directory)
|
|
except Exception as e:
|
|
self.print('Encountered error: %s' % str(e), file=sys.stderr)
|
|
retcode = 1
|
|
else:
|
|
retcode = 0
|
|
finally:
|
|
# Clean up the downloaded zipfile.
|
|
gclient_utils.rm_file_or_tree(tempdir)
|
|
|
|
if retcode:
|
|
self.print(
|
|
'Extracting bootstrap zipfile %s failed.\n'
|
|
'Resuming normal operations.' % filename)
|
|
return False
|
|
return True
|
|
|
|
def exists(self):
|
|
return os.path.isfile(os.path.join(self.mirror_path, 'config'))
|
|
|
|
def _preserve_fetchspec(self):
|
|
"""Read and preserve remote.origin.fetch from an existing mirror.
|
|
|
|
This modifies self.fetch_specs.
|
|
"""
|
|
if not self.exists():
|
|
return
|
|
try:
|
|
config_fetchspecs = subprocess.check_output(
|
|
[self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
|
|
cwd=self.mirror_path)
|
|
for fetchspec in config_fetchspecs.splitlines():
|
|
self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
|
|
except subprocess.CalledProcessError:
|
|
logging.warn('Tried and failed to preserve remote.origin.fetch from the '
|
|
'existing cache directory. You may need to manually edit '
|
|
'%s and "git cache fetch" again.'
|
|
% os.path.join(self.mirror_path, 'config'))
|
|
|
|
def _ensure_bootstrapped(self, depth, bootstrap, force=False):
|
|
tempdir = None
|
|
pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
|
|
pack_files = []
|
|
|
|
if os.path.isdir(pack_dir):
|
|
pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
|
|
|
|
should_bootstrap = (force or
|
|
not self.exists() or
|
|
len(pack_files) > GC_AUTOPACKLIMIT)
|
|
if should_bootstrap:
|
|
if self.exists():
|
|
# Re-bootstrapping an existing mirror; preserve existing fetch spec.
|
|
self._preserve_fetchspec()
|
|
tempdir = tempfile.mkdtemp(
|
|
prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
|
|
bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
|
|
if bootstrapped:
|
|
# Bootstrap succeeded; delete previous cache, if any.
|
|
gclient_utils.rmtree(self.mirror_path)
|
|
elif not self.exists():
|
|
# Bootstrap failed, no previous cache; start with a bare git dir.
|
|
self.RunGit(['init', '--bare'], cwd=tempdir)
|
|
else:
|
|
# Bootstrap failed, previous cache exists; warn and continue.
|
|
logging.warn(
|
|
'Git cache has a lot of pack files (%d). Tried to re-bootstrap '
|
|
'but failed. Continuing with non-optimized repository.'
|
|
% len(pack_files))
|
|
gclient_utils.rmtree(tempdir)
|
|
tempdir = None
|
|
else:
|
|
if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
|
|
logging.warn(
|
|
'Shallow fetch requested, but repo cache already exists.')
|
|
return tempdir
|
|
|
|
def _fetch(self, rundir, verbose, depth):
|
|
self.config(rundir)
|
|
v = []
|
|
d = []
|
|
if verbose:
|
|
v = ['-v', '--progress']
|
|
if depth:
|
|
d = ['--depth', str(depth)]
|
|
fetch_cmd = ['fetch'] + v + d + ['origin']
|
|
fetch_specs = subprocess.check_output(
|
|
[self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
|
|
cwd=rundir).strip().splitlines()
|
|
for spec in fetch_specs:
|
|
try:
|
|
self.print('Fetching %s' % spec)
|
|
self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
|
|
except subprocess.CalledProcessError:
|
|
if spec == '+refs/heads/*:refs/heads/*':
|
|
raise ClobberNeeded() # Corrupted cache.
|
|
logging.warn('Fetch of %s failed' % spec)
|
|
|
|
def populate(self, depth=None, shallow=False, bootstrap=False,
|
|
verbose=False, ignore_lock=False, lock_timeout=0):
|
|
assert self.GetCachePath()
|
|
if shallow and not depth:
|
|
depth = 10000
|
|
gclient_utils.safe_makedirs(self.GetCachePath())
|
|
|
|
lockfile = Lockfile(self.mirror_path, lock_timeout)
|
|
if not ignore_lock:
|
|
lockfile.lock()
|
|
|
|
tempdir = None
|
|
try:
|
|
tempdir = self._ensure_bootstrapped(depth, bootstrap)
|
|
rundir = tempdir or self.mirror_path
|
|
self._fetch(rundir, verbose, depth)
|
|
except ClobberNeeded:
|
|
# This is a major failure, we need to clean and force a bootstrap.
|
|
gclient_utils.rmtree(rundir)
|
|
self.print(GIT_CACHE_CORRUPT_MESSAGE)
|
|
tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
|
|
assert tempdir
|
|
self._fetch(tempdir or self.mirror_path, verbose, depth)
|
|
finally:
|
|
if tempdir:
|
|
try:
|
|
if os.path.exists(self.mirror_path):
|
|
gclient_utils.rmtree(self.mirror_path)
|
|
os.rename(tempdir, self.mirror_path)
|
|
except OSError as e:
|
|
# This is somehow racy on Windows.
|
|
# Catching OSError because WindowsError isn't portable and
|
|
# pylint complains.
|
|
self.print('Error moving %s to %s: %s' % (tempdir, self.mirror_path,
|
|
str(e)))
|
|
if not ignore_lock:
|
|
lockfile.unlock()
|
|
|
|
def update_bootstrap(self, prune=False):
|
|
# The files are named <git number>.zip
|
|
gen_number = subprocess.check_output(
|
|
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
|
|
# Run Garbage Collect to compress packfile.
|
|
self.RunGit(['gc', '--prune=all'])
|
|
# Creating a temp file and then deleting it ensures we can use this name.
|
|
_, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
|
|
os.remove(tmp_zipfile)
|
|
subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
|
|
gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
|
|
gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
|
|
dest_name = '%s/%s.zip' % (gs_folder, gen_number)
|
|
gsutil.call('cp', tmp_zipfile, dest_name)
|
|
os.remove(tmp_zipfile)
|
|
|
|
# Remove all other files in the same directory.
|
|
if prune:
|
|
_, ls_out, _ = gsutil.check_call('ls', gs_folder)
|
|
for filename in ls_out.splitlines():
|
|
if filename == dest_name:
|
|
continue
|
|
gsutil.call('rm', filename)
|
|
|
|
@staticmethod
|
|
def DeleteTmpPackFiles(path):
|
|
pack_dir = os.path.join(path, 'objects', 'pack')
|
|
if not os.path.isdir(pack_dir):
|
|
return
|
|
pack_files = [f for f in os.listdir(pack_dir) if
|
|
f.startswith('.tmp-') or f.startswith('tmp_pack_')]
|
|
for f in pack_files:
|
|
f = os.path.join(pack_dir, f)
|
|
try:
|
|
os.remove(f)
|
|
logging.warn('Deleted stale temporary pack file %s' % f)
|
|
except OSError:
|
|
logging.warn('Unable to delete temporary pack file %s' % f)
|
|
|
|
@classmethod
|
|
def BreakLocks(cls, path):
|
|
did_unlock = False
|
|
lf = Lockfile(path)
|
|
if lf.break_lock():
|
|
did_unlock = True
|
|
# Look for lock files that might have been left behind by an interrupted
|
|
# git process.
|
|
lf = os.path.join(path, 'config.lock')
|
|
if os.path.exists(lf):
|
|
os.remove(lf)
|
|
did_unlock = True
|
|
cls.DeleteTmpPackFiles(path)
|
|
return did_unlock
|
|
|
|
def unlock(self):
|
|
return self.BreakLocks(self.mirror_path)
|
|
|
|
@classmethod
|
|
def UnlockAll(cls):
|
|
cachepath = cls.GetCachePath()
|
|
if not cachepath:
|
|
return
|
|
dirlist = os.listdir(cachepath)
|
|
repo_dirs = set([os.path.join(cachepath, path) for path in dirlist
|
|
if os.path.isdir(os.path.join(cachepath, path))])
|
|
for dirent in dirlist:
|
|
if dirent.startswith('_cache_tmp') or dirent.startswith('tmp'):
|
|
gclient_utils.rm_file_or_tree(os.path.join(cachepath, dirent))
|
|
elif (dirent.endswith('.lock') and
|
|
os.path.isfile(os.path.join(cachepath, dirent))):
|
|
repo_dirs.add(os.path.join(cachepath, dirent[:-5]))
|
|
|
|
unlocked_repos = []
|
|
for repo_dir in repo_dirs:
|
|
if cls.BreakLocks(repo_dir):
|
|
unlocked_repos.append(repo_dir)
|
|
|
|
return unlocked_repos
|
|
|
|
@subcommand.usage('[url of repo to check for caching]')
|
|
def CMDexists(parser, args):
|
|
"""Check to see if there already is a cache of the given repo."""
|
|
_, args = parser.parse_args(args)
|
|
if not len(args) == 1:
|
|
parser.error('git cache exists only takes exactly one repo url.')
|
|
url = args[0]
|
|
mirror = Mirror(url)
|
|
if mirror.exists():
|
|
print(mirror.mirror_path)
|
|
return 0
|
|
return 1
|
|
|
|
|
|
@subcommand.usage('[url of repo to create a bootstrap zip file]')
|
|
def CMDupdate_bootstrap(parser, args):
|
|
"""Create and uploads a bootstrap tarball."""
|
|
# Lets just assert we can't do this on Windows.
|
|
if sys.platform.startswith('win'):
|
|
print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
|
|
return 1
|
|
|
|
parser.add_option('--prune', action='store_true',
|
|
help='Prune all other cached zipballs of the same repo.')
|
|
|
|
# First, we need to ensure the cache is populated.
|
|
populate_args = args[:]
|
|
populate_args.append('--no-bootstrap')
|
|
CMDpopulate(parser, populate_args)
|
|
|
|
# Get the repo directory.
|
|
options, args = parser.parse_args(args)
|
|
url = args[0]
|
|
mirror = Mirror(url)
|
|
mirror.update_bootstrap(options.prune)
|
|
return 0
|
|
|
|
|
|
@subcommand.usage('[url of repo to add to or update in cache]')
|
|
def CMDpopulate(parser, args):
|
|
"""Ensure that the cache has all up-to-date objects for the given repo."""
|
|
parser.add_option('--depth', type='int',
|
|
help='Only cache DEPTH commits of history')
|
|
parser.add_option('--shallow', '-s', action='store_true',
|
|
help='Only cache 10000 commits of history')
|
|
parser.add_option('--ref', action='append',
|
|
help='Specify additional refs to be fetched')
|
|
parser.add_option('--no_bootstrap', '--no-bootstrap',
|
|
action='store_true',
|
|
help='Don\'t bootstrap from Google Storage')
|
|
parser.add_option('--ignore_locks', '--ignore-locks',
|
|
action='store_true',
|
|
help='Don\'t try to lock repository')
|
|
|
|
options, args = parser.parse_args(args)
|
|
if not len(args) == 1:
|
|
parser.error('git cache populate only takes exactly one repo url.')
|
|
url = args[0]
|
|
|
|
mirror = Mirror(url, refs=options.ref)
|
|
kwargs = {
|
|
'verbose': options.verbose,
|
|
'shallow': options.shallow,
|
|
'bootstrap': not options.no_bootstrap,
|
|
'ignore_lock': options.ignore_locks,
|
|
'lock_timeout': options.timeout,
|
|
}
|
|
if options.depth:
|
|
kwargs['depth'] = options.depth
|
|
mirror.populate(**kwargs)
|
|
|
|
|
|
@subcommand.usage('Fetch new commits into cache and current checkout')
|
|
def CMDfetch(parser, args):
|
|
"""Update mirror, and fetch in cwd."""
|
|
parser.add_option('--all', action='store_true', help='Fetch all remotes')
|
|
parser.add_option('--no_bootstrap', '--no-bootstrap',
|
|
action='store_true',
|
|
help='Don\'t (re)bootstrap from Google Storage')
|
|
options, args = parser.parse_args(args)
|
|
|
|
# Figure out which remotes to fetch. This mimics the behavior of regular
|
|
# 'git fetch'. Note that in the case of "stacked" or "pipelined" branches,
|
|
# this will NOT try to traverse up the branching structure to find the
|
|
# ultimate remote to update.
|
|
remotes = []
|
|
if options.all:
|
|
assert not args, 'fatal: fetch --all does not take a repository argument'
|
|
remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines()
|
|
elif args:
|
|
remotes = args
|
|
else:
|
|
current_branch = subprocess.check_output(
|
|
[Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
|
|
if current_branch != 'HEAD':
|
|
upstream = subprocess.check_output(
|
|
[Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch]
|
|
).strip()
|
|
if upstream and upstream != '.':
|
|
remotes = [upstream]
|
|
if not remotes:
|
|
remotes = ['origin']
|
|
|
|
cachepath = Mirror.GetCachePath()
|
|
git_dir = os.path.abspath(subprocess.check_output(
|
|
[Mirror.git_exe, 'rev-parse', '--git-dir']))
|
|
git_dir = os.path.abspath(git_dir)
|
|
if git_dir.startswith(cachepath):
|
|
mirror = Mirror.FromPath(git_dir)
|
|
mirror.populate(
|
|
bootstrap=not options.no_bootstrap, lock_timeout=options.timeout)
|
|
return 0
|
|
for remote in remotes:
|
|
remote_url = subprocess.check_output(
|
|
[Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip()
|
|
if remote_url.startswith(cachepath):
|
|
mirror = Mirror.FromPath(remote_url)
|
|
mirror.print = lambda *args: None
|
|
print('Updating git cache...')
|
|
mirror.populate(
|
|
bootstrap=not options.no_bootstrap, lock_timeout=options.timeout)
|
|
subprocess.check_call([Mirror.git_exe, 'fetch', remote])
|
|
return 0
|
|
|
|
|
|
@subcommand.usage('[url of repo to unlock, or -a|--all]')
|
|
def CMDunlock(parser, args):
|
|
"""Unlock one or all repos if their lock files are still around."""
|
|
parser.add_option('--force', '-f', action='store_true',
|
|
help='Actually perform the action')
|
|
parser.add_option('--all', '-a', action='store_true',
|
|
help='Unlock all repository caches')
|
|
options, args = parser.parse_args(args)
|
|
if len(args) > 1 or (len(args) == 0 and not options.all):
|
|
parser.error('git cache unlock takes exactly one repo url, or --all')
|
|
|
|
if not options.force:
|
|
cachepath = Mirror.GetCachePath()
|
|
lockfiles = [os.path.join(cachepath, path)
|
|
for path in os.listdir(cachepath)
|
|
if path.endswith('.lock') and os.path.isfile(path)]
|
|
parser.error('git cache unlock requires -f|--force to do anything. '
|
|
'Refusing to unlock the following repo caches: '
|
|
', '.join(lockfiles))
|
|
|
|
unlocked_repos = []
|
|
if options.all:
|
|
unlocked_repos.extend(Mirror.UnlockAll())
|
|
else:
|
|
m = Mirror(args[0])
|
|
if m.unlock():
|
|
unlocked_repos.append(m.mirror_path)
|
|
|
|
if unlocked_repos:
|
|
logging.info('Broke locks on these caches:\n %s' % '\n '.join(
|
|
unlocked_repos))
|
|
|
|
|
|
class OptionParser(optparse.OptionParser):
|
|
"""Wrapper class for OptionParser to handle global options."""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
|
|
self.add_option('-c', '--cache-dir',
|
|
help='Path to the directory containing the cache')
|
|
self.add_option('-v', '--verbose', action='count', default=1,
|
|
help='Increase verbosity (can be passed multiple times)')
|
|
self.add_option('-q', '--quiet', action='store_true',
|
|
help='Suppress all extraneous output')
|
|
self.add_option('--timeout', type='int', default=0,
|
|
help='Timeout for acquiring cache lock, in seconds')
|
|
|
|
def parse_args(self, args=None, values=None):
|
|
options, args = optparse.OptionParser.parse_args(self, args, values)
|
|
if options.quiet:
|
|
options.verbose = 0
|
|
|
|
levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
|
|
logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
|
|
|
|
try:
|
|
global_cache_dir = Mirror.GetCachePath()
|
|
except RuntimeError:
|
|
global_cache_dir = None
|
|
if options.cache_dir:
|
|
if global_cache_dir and (
|
|
os.path.abspath(options.cache_dir) !=
|
|
os.path.abspath(global_cache_dir)):
|
|
logging.warn('Overriding globally-configured cache directory.')
|
|
Mirror.SetCachePath(options.cache_dir)
|
|
|
|
return options, args
|
|
|
|
|
|
def main(argv):
|
|
dispatcher = subcommand.CommandDispatcher(__name__)
|
|
return dispatcher.execute(OptionParser(), argv)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
sys.exit(main(sys.argv[1:]))
|
|
except KeyboardInterrupt:
|
|
sys.stderr.write('interrupted\n')
|
|
sys.exit(1)
|