#!/usr/bin/env python # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Run a pinned gsutil.""" import argparse import shutil import zipfile import hashlib import base64 import os import sys import json import urllib import subprocess GSUTIL_URL = 'https://storage.googleapis.com/pub/' API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' THIS_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil') DEFAULT_FALLBACK_GSUTIL = os.path.join( THIS_DIR, 'third_party', 'gsutil', 'gsutil') class SubprocessError(Exception): pass class InvalidGsutilError(Exception): pass def call(args, verbose=True, **kwargs): kwargs['stdout'] = subprocess.PIPE kwargs['stderr'] = subprocess.STDOUT proc = subprocess.Popen(args, **kwargs) out = [] for line in proc.stdout: out.append(line) if verbose: sys.stdout.write(line) code = proc.wait() if code: raise SubprocessError('%s failed with %s' % (args, code)) return ''.join(out) def download_gsutil(version, target_dir): """Downloads gsutil into the target_dir.""" filename = 'gsutil_%s.zip' % version target_filename = os.path.join(target_dir, filename) # Check if the target exists already. if os.path.exists(target_filename): md5_calc = hashlib.md5() with open(target_filename, 'rb') as f: while True: buf = f.read(4096) if not buf: break md5_calc.update(buf) local_md5 = md5_calc.hexdigest() metadata_url = '%s%s' % (API_URL, filename) metadata = json.load(urllib.urlopen(metadata_url)) remote_md5 = base64.b64decode(metadata['md5Hash']) if local_md5 == remote_md5: return target_filename os.remove(target_filename) # Do the download. url = '%s%s' % (GSUTIL_URL, filename) u = urllib.urlopen(url) with open(target_filename, 'wb') as f: while True: buf = u.read(4096) if not buf: break f.write(buf) return target_filename def check_gsutil(gsutil_bin): """Run gsutil version and make sure it runs.""" try: call([sys.executable, gsutil_bin, 'version'], verbose=False) return True except SubprocessError: return False def ensure_gsutil(version, target): bin_dir = os.path.join(target, 'gsutil_%s' % version) gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil') if os.path.isfile(gsutil_bin) and check_gsutil(gsutil_bin): # Everything is awesome! we're all done here. return gsutil_bin if os.path.isdir(bin_dir): # Clean up if we're redownloading a corrupted gsutil. shutil.rmtree(bin_dir) cache_dir = os.path.join(target, '.cache_dir') if not os.path.isdir(cache_dir): os.makedirs(cache_dir) target_zip_filename = download_gsutil(version, cache_dir) with zipfile.ZipFile(target_zip_filename, 'r') as target_zip: target_zip.extractall(bin_dir) # Final check that the gsutil bin is okay. This should never fail. if not check_gsutil(gsutil_bin): raise InvalidGsutilError() return gsutil_bin def run_gsutil(force_version, fallback, target, args): if force_version: gsutil_bin = ensure_gsutil(force_version, target) else: gsutil_bin = fallback cmd = [sys.executable, gsutil_bin] + args call(cmd) def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--force-version') parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL) parser.add_argument('--target', default=DEFAULT_BIN_DIR) parser.add_argument('args', nargs=argparse.REMAINDER) args = parser.parse_args() return args.force_version, args.fallback, args.target, args.args def main(): force_version, fallback, target, args = parse_args() run_gsutil(force_version, fallback, target, args) if __name__ == '__main__': sys.exit(main())