From f0149a3cb5c0594603d71bec286627c63cbdf51a Mon Sep 17 00:00:00 2001 From: "akuegel@chromium.org" Date: Mon, 1 Jun 2015 15:53:48 +0000 Subject: [PATCH] Install google appengine. This is needed to run the presubmit commit checks. Tested this CL by running git cl presubmit locally. BUG=420910 Review URL: https://codereview.chromium.org/1156923006 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295472 0039d316-1c4b-4281-b951-d872f2087c98 --- testing_support/get_appengine.py | 139 ++++++++++++++++++++++++++++++ testing_support/local_rietveld.py | 25 +++--- 2 files changed, 149 insertions(+), 15 deletions(-) create mode 100755 testing_support/get_appengine.py diff --git a/testing_support/get_appengine.py b/testing_support/get_appengine.py new file mode 100755 index 000000000..98820ffb8 --- /dev/null +++ b/testing_support/get_appengine.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This script is copied from +https://chromium.googlesource.com/infra/infra.git/+/master/bootstrap +""" + +import datetime +import logging +import optparse +import os +import re +import shutil +import sys +import time +import tempfile +import urllib2 +import zipfile + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def get_gae_sdk_version(gae_path): + """Returns the installed GAE SDK version or None.""" + version_path = os.path.join(gae_path, 'VERSION') + if os.path.isfile(version_path): + values = dict( + map(lambda x: x.strip(), l.split(':')) + for l in open(version_path) if ':' in l) + if 'release' in values: + return values['release'].strip('"') + + +def get_latest_gae_sdk_url(name): + """Returns the url to get the latest GAE SDK and its version.""" + url = 'https://cloud.google.com/appengine/downloads.html' + logging.debug('%s', url) + content = urllib2.urlopen(url).read() + regexp = ( + r'(https\:\/\/storage.googleapis.com\/appengine-sdks\/featured\/' + + re.escape(name) + r'[0-9\.]+?\.zip)') + m = re.search(regexp, content) + url = m.group(1) + # Calculate the version from the url. + new_version = re.search(re.escape(name) + r'(.+?).zip', url).group(1) + # Upgrade to https + return url.replace('http://', 'https://'), new_version + + +def extract_zip(z, root_path): + """Extracts files in a zipfile but keep the executable bits.""" + count = 0 + for f in z.infolist(): + perm = (f.external_attr >> 16L) & 0777 + mtime = time.mktime(datetime.datetime(*f.date_time).timetuple()) + filepath = os.path.join(root_path, f.filename) + logging.debug('Extracting %s', f.filename) + if f.filename.endswith('/'): + os.mkdir(filepath, perm) + else: + z.extract(f, root_path) + os.chmod(filepath, perm) + count += 1 + os.utime(filepath, (mtime, mtime)) + print('Extracted %d files' % count) + + +def install_latest_gae_sdk(root_path, fetch_go, dry_run): + if fetch_go: + rootdir = 'go_appengine' + if sys.platform == 'darwin': + name = 'go_appengine_sdk_darwin_amd64-' + else: + # Add other platforms as needed. + name = 'go_appengine_sdk_linux_amd64-' + else: + rootdir = 'google_appengine' + name = 'google_appengine_' + + # The zip file already contains 'google_appengine' (for python) or + # 'go_appengine' (for go) in its path so it's a bit + # awkward to unzip otherwise. Hard code the path in for now. + gae_path = os.path.join(root_path, rootdir) + print('Looking up path %s' % gae_path) + version = get_gae_sdk_version(gae_path) + if version: + print('Found installed version %s' % version) + else: + print('Didn\'t find an SDK') + + url, new_version = get_latest_gae_sdk_url(name) + print('New version is %s' % new_version) + if version == new_version: + return 0 + + if os.path.isdir(gae_path): + print('Removing previous version') + if not dry_run: + shutil.rmtree(gae_path) + + print('Fetching %s' % url) + if not dry_run: + u = urllib2.urlopen(url) + with tempfile.NamedTemporaryFile() as f: + while True: + chunk = u.read(2 ** 20) + if not chunk: + break + f.write(chunk) + # Assuming we're extracting there. In fact, we have no idea. + print('Extracting into %s' % gae_path) + z = zipfile.ZipFile(f, 'r') + try: + extract_zip(z, root_path) + finally: + z.close() + return 0 + + +def main(): + parser = optparse.OptionParser(prog='python -m %s' % __package__) + parser.add_option('-v', '--verbose', action='store_true') + parser.add_option( + '-g', '--go', action='store_true', help='Defaults to python SDK') + parser.add_option( + '-d', '--dest', default=os.path.dirname(BASE_DIR), help='Output') + parser.add_option('--dry-run', action='store_true', help='Do not download') + options, args = parser.parse_args() + if args: + parser.error('Unsupported args: %s' % ' '.join(args)) + logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR) + return install_latest_gae_sdk( + os.path.abspath(options.dest), options.go, options.dry_run) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/testing_support/local_rietveld.py b/testing_support/local_rietveld.py index 41b0166bf..83d232f4b 100755 --- a/testing_support/local_rietveld.py +++ b/testing_support/local_rietveld.py @@ -63,23 +63,18 @@ class LocalRietveld(object): self.test_server = None self.port = None self.tempdir = None - - # Find the GAE SDK - previous_dir = '' - self.sdk_path = '' - base_dir = self.base_dir - while base_dir != previous_dir: - previous_dir = base_dir - self.sdk_path = os.path.join(base_dir, 'google_appengine') - if not os.path.isfile(os.path.join(self.sdk_path, 'VERSION')): - base_dir = os.path.dirname(base_dir) - self.dev_app = os.path.join(self.sdk_path, 'dev_appserver.py') + self.dev_app = None def install_prerequisites(self): - # First, verify the Google AppEngine SDK is available. - if not os.path.isfile(self.dev_app): - raise Failure( - 'Install google_appengine sdk in %s or higher up' % self.base_dir) + # First, install the Google AppEngine SDK. + cmd = [os.path.join(self.base_dir, 'get_appengine.py'), + '--dest=%s' % self.base_dir] + try: + subprocess2.check_call(cmd) + except (OSError, subprocess2.CalledProcessError), e: + raise Failure('Failed to run %s\n%s' % (cmd, e)) + sdk_path = os.path.join(self.base_dir, 'google_appengine') + self.dev_app = os.path.join(sdk_path, 'dev_appserver.py') if os.path.isdir(os.path.join(self.rietveld, '.hg')): # Left over from mercurial. Delete it.