diff --git a/.gitignore b/.gitignore index 8ceec20b7..b8290067d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ /tags # Ignore unittest related files. +/testing_support/_infra /testing_support/_rietveld /tests/git-svn-submodule/ /tests/subversion_config/README.txt diff --git a/fetch.py b/fetch.py index cf34404e4..0fbc37851 100755 --- a/fetch.py +++ b/fetch.py @@ -231,6 +231,7 @@ def usage(msg=None): Valid options: -h, --help, help Print this message. --nohooks Don't run hooks after checkout. + --force (dangerous) Don't look for existing .gclient file. -n, --dry-run Don't run commands, only print them. --no-history Perform shallow clones, don't fetch the full git history. @@ -255,6 +256,7 @@ def handle_args(argv): dry_run = False nohooks = False no_history = False + force = False while len(argv) >= 2: arg = argv[1] if not arg.startswith('-'): @@ -266,6 +268,8 @@ def handle_args(argv): nohooks = True elif arg == '--no-history': no_history = True + elif arg == '--force': + force = True else: usage('Invalid option %s.' % arg) @@ -279,8 +283,11 @@ def handle_args(argv): config = argv[1] props = argv[2:] return ( - optparse.Values( - {'dry_run':dry_run, 'nohooks':nohooks, 'no_history': no_history }), + optparse.Values({ + 'dry_run': dry_run, + 'nohooks': nohooks, + 'no_history': no_history, + 'force': force}), config, props) @@ -324,7 +331,7 @@ def run(options, spec, root): checkout = CheckoutFactory(checkout_type, options, checkout_spec, root) except KeyError: return 1 - if checkout.exists(): + if not options.force and checkout.exists(): print 'Your current directory appears to already contain, or be part of, ' print 'a checkout. "fetch" is used only to get new checkouts. Use ' print '"gclient sync" to update existing checkouts.' diff --git a/fetch_configs/infra.py b/fetch_configs/infra.py index 12e5c9646..29681b21b 100644 --- a/fetch_configs/infra.py +++ b/fetch_configs/infra.py @@ -13,7 +13,9 @@ class Infra(config_util.Config): """Basic Config class for the Infrastructure repositories.""" @staticmethod - def fetch_spec(_props): + def fetch_spec(props): + # This is used by [depot_tools]/testing_support/local_rietveld.py + managed = props.get('managed', 'false').lower() == 'true' return { 'type': 'gclient_git', 'gclient_git_spec': { @@ -22,7 +24,7 @@ class Infra(config_util.Config): 'name' : 'infra', 'url' : 'https://chromium.googlesource.com/infra/infra.git', 'deps_file': '.DEPS.git', - 'managed' : False, + 'managed' : managed, } ], }, diff --git a/testing_support/get_appengine.py b/testing_support/get_appengine.py deleted file mode 100755 index 98820ffb8..000000000 --- a/testing_support/get_appengine.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2011 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""This script is copied from -https://chromium.googlesource.com/infra/infra.git/+/master/bootstrap -""" - -import datetime -import logging -import optparse -import os -import re -import shutil -import sys -import time -import tempfile -import urllib2 -import zipfile - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - - -def get_gae_sdk_version(gae_path): - """Returns the installed GAE SDK version or None.""" - version_path = os.path.join(gae_path, 'VERSION') - if os.path.isfile(version_path): - values = dict( - map(lambda x: x.strip(), l.split(':')) - for l in open(version_path) if ':' in l) - if 'release' in values: - return values['release'].strip('"') - - -def get_latest_gae_sdk_url(name): - """Returns the url to get the latest GAE SDK and its version.""" - url = 'https://cloud.google.com/appengine/downloads.html' - logging.debug('%s', url) - content = urllib2.urlopen(url).read() - regexp = ( - r'(https\:\/\/storage.googleapis.com\/appengine-sdks\/featured\/' - + re.escape(name) + r'[0-9\.]+?\.zip)') - m = re.search(regexp, content) - url = m.group(1) - # Calculate the version from the url. - new_version = re.search(re.escape(name) + r'(.+?).zip', url).group(1) - # Upgrade to https - return url.replace('http://', 'https://'), new_version - - -def extract_zip(z, root_path): - """Extracts files in a zipfile but keep the executable bits.""" - count = 0 - for f in z.infolist(): - perm = (f.external_attr >> 16L) & 0777 - mtime = time.mktime(datetime.datetime(*f.date_time).timetuple()) - filepath = os.path.join(root_path, f.filename) - logging.debug('Extracting %s', f.filename) - if f.filename.endswith('/'): - os.mkdir(filepath, perm) - else: - z.extract(f, root_path) - os.chmod(filepath, perm) - count += 1 - os.utime(filepath, (mtime, mtime)) - print('Extracted %d files' % count) - - -def install_latest_gae_sdk(root_path, fetch_go, dry_run): - if fetch_go: - rootdir = 'go_appengine' - if sys.platform == 'darwin': - name = 'go_appengine_sdk_darwin_amd64-' - else: - # Add other platforms as needed. - name = 'go_appengine_sdk_linux_amd64-' - else: - rootdir = 'google_appengine' - name = 'google_appengine_' - - # The zip file already contains 'google_appengine' (for python) or - # 'go_appengine' (for go) in its path so it's a bit - # awkward to unzip otherwise. Hard code the path in for now. - gae_path = os.path.join(root_path, rootdir) - print('Looking up path %s' % gae_path) - version = get_gae_sdk_version(gae_path) - if version: - print('Found installed version %s' % version) - else: - print('Didn\'t find an SDK') - - url, new_version = get_latest_gae_sdk_url(name) - print('New version is %s' % new_version) - if version == new_version: - return 0 - - if os.path.isdir(gae_path): - print('Removing previous version') - if not dry_run: - shutil.rmtree(gae_path) - - print('Fetching %s' % url) - if not dry_run: - u = urllib2.urlopen(url) - with tempfile.NamedTemporaryFile() as f: - while True: - chunk = u.read(2 ** 20) - if not chunk: - break - f.write(chunk) - # Assuming we're extracting there. In fact, we have no idea. - print('Extracting into %s' % gae_path) - z = zipfile.ZipFile(f, 'r') - try: - extract_zip(z, root_path) - finally: - z.close() - return 0 - - -def main(): - parser = optparse.OptionParser(prog='python -m %s' % __package__) - parser.add_option('-v', '--verbose', action='store_true') - parser.add_option( - '-g', '--go', action='store_true', help='Defaults to python SDK') - parser.add_option( - '-d', '--dest', default=os.path.dirname(BASE_DIR), help='Output') - parser.add_option('--dry-run', action='store_true', help='Do not download') - options, args = parser.parse_args() - if args: - parser.error('Unsupported args: %s' % ' '.join(args)) - logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR) - return install_latest_gae_sdk( - os.path.abspath(options.dest), options.go, options.dry_run) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/testing_support/local_rietveld.py b/testing_support/local_rietveld.py index 83d232f4b..d1fdc0b92 100755 --- a/testing_support/local_rietveld.py +++ b/testing_support/local_rietveld.py @@ -26,6 +26,7 @@ except ImportError: os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) import subprocess2 +DEPOT_TOOLS=os.path.dirname(os.path.dirname(os.path.abspath(__file__))) class Failure(Exception): pass @@ -55,60 +56,54 @@ class LocalRietveld(object): self.base_dir = base_dir if not self.base_dir: self.base_dir = os.path.dirname(os.path.abspath(__file__)) - # TODO(maruel): This should be in /tmp but that would mean having to fetch - # everytime. This test is already annoyingly slow. self.rietveld = os.path.join(self.base_dir, '_rietveld') + self.infra = os.path.join(self.base_dir, '_infra') self.rietveld_app = os.path.join( - self.rietveld, 'appengine', 'chromium_rietveld') + self.infra, 'infra', 'appengine', 'chromium_rietveld') + self.dev_app = os.path.join( + self.infra, 'google_appengine', 'dev_appserver.py') self.test_server = None self.port = None self.tempdir = None - self.dev_app = None def install_prerequisites(self): - # First, install the Google AppEngine SDK. - cmd = [os.path.join(self.base_dir, 'get_appengine.py'), - '--dest=%s' % self.base_dir] - try: - subprocess2.check_call(cmd) - except (OSError, subprocess2.CalledProcessError), e: - raise Failure('Failed to run %s\n%s' % (cmd, e)) - sdk_path = os.path.join(self.base_dir, 'google_appengine') - self.dev_app = os.path.join(sdk_path, 'dev_appserver.py') - - if os.path.isdir(os.path.join(self.rietveld, '.hg')): - # Left over from mercurial. Delete it. - print('Deleting deprecated mercurial rietveld files...') + if os.path.exists(self.rietveld): + print "Removing old rietveld dir" shutil.rmtree(self.rietveld) - # Second, checkout rietveld if not available. - if not os.path.isdir(self.rietveld): - print('Checking out rietveld...') - try: - subprocess2.check_call(['git', 'init', self.rietveld]) - subprocess2.check_call( - ['git', 'remote', 'add', '-f', 'origin', - 'https://chromium.googlesource.com/infra/infra.git'], - cwd=self.rietveld) - subprocess2.check_call( - ['git', 'config', 'core.sparseCheckout', 'true'], - cwd=self.rietveld) - with file(os.path.join(self.rietveld, '.git/info/sparse-checkout'), - 'w') as sparse_file: - sparse_file.write('appengine/chromium_rietveld') - subprocess2.check_call( - ['git', 'pull', 'origin', 'master'], - cwd=self.rietveld) - except (OSError, subprocess2.CalledProcessError), e: - raise Failure('Failed to clone rietveld. \n%s' % e) - else: - print('Syncing rietveld...') - try: - subprocess2.check_call( - ['git', 'pull', 'origin', 'master'], - cwd=self.rietveld) - except (OSError, subprocess2.CalledProcessError), e: - raise Failure('Failed to sync rietveld\n%s' % e) + sdk_path = os.path.join(self.base_dir, 'google_appengine') + if os.path.exists(sdk_path): + print "Removing old appengine SDK dir" + shutil.rmtree(sdk_path) + + previous = os.environ.get('DEPOT_TOOLS_UPDATE') + os.environ['DEPOT_TOOLS_UPDATE'] = '0' + try: + if not os.path.isfile(os.path.join(self.infra, '.gclient')): + print('Checking out infra...') + shutil.rmtree(self.infra, ignore_errors=True) + try: + os.makedirs(self.infra) + subprocess2.call( + [sys.executable, os.path.join(DEPOT_TOOLS, 'fetch.py'), + '--force', 'infra', '--managed=true'], + cwd=self.infra) + except (OSError, subprocess2.CalledProcessError), e: + raise Failure('Failed to clone infra. \n%s' % e) + else: + print('Syncing infra...') + try: + subprocess2.call( + [sys.executable, os.path.join(DEPOT_TOOLS, 'gclient.py'), + 'sync', '--force'], + cwd=self.infra) + except (OSError, subprocess2.CalledProcessError), e: + raise Failure('Failed to sync infra. \n%s' % e) + finally: + if previous is None: + del os.environ['DEPOT_TOOLS_UPDATE'] + else: + os.environ['DEPOT_TOOLS_UPDATE'] = previous def start_server(self, verbose=False): self.install_prerequisites()