diff --git a/download_from_google_storage.py b/download_from_google_storage.py index d416d1444..84bc8e30d 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -76,12 +76,15 @@ class Gsutil(object): MAX_TRIES = 5 RETRY_BASE_DELAY = 5.0 RETRY_DELAY_MULTIPLE = 1.3 + VPYTHON3 = ('vpython3.bat' + if GetNormalizedPlatform() == 'win32' else 'vpython3') - def __init__(self, path, boto_path=None): + def __init__(self, path, boto_path=None, version='4.28'): if not os.path.exists(path): raise FileNotFoundError('GSUtil not found in %s' % path) self.path = path self.boto_path = boto_path + self.version = version def get_sub_env(self): env = os.environ.copy() @@ -98,12 +101,12 @@ class Gsutil(object): return env def call(self, *args): - cmd = [sys.executable, self.path] + cmd = [self.VPYTHON3, self.path, '--force-version', self.version] cmd.extend(args) return subprocess2.call(cmd, env=self.get_sub_env()) def check_call(self, *args): - cmd = [sys.executable, self.path] + cmd = [self.VPYTHON3, self.path, '--force-version', self.version] cmd.extend(args) ((out, err), code) = subprocess2.communicate( cmd, diff --git a/gsutil.py b/gsutil.py index 6ed377385..d5423ad7e 100755 --- a/gsutil.py +++ b/gsutil.py @@ -29,14 +29,13 @@ import zipfile GSUTIL_URL = 'https://storage.googleapis.com/pub/' API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' - THIS_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil') +DEFAULT_FALLBACK_GSUTIL = os.path.join( + THIS_DIR, 'third_party', 'gsutil', 'gsutil') IS_WINDOWS = os.name == 'nt' -VERSION = '4.61' - class InvalidGsutilError(Exception): pass @@ -130,8 +129,11 @@ def ensure_gsutil(version, target, clean): return gsutil_bin -def run_gsutil(target, args, clean=False): - gsutil_bin = ensure_gsutil(VERSION, target, clean) +def run_gsutil(force_version, fallback, target, args, clean=False): + if force_version: + gsutil_bin = ensure_gsutil(force_version, target, clean) + else: + gsutil_bin = fallback disable_update = ['-o', 'GSUtil:software_update_check_period=0'] if sys.platform == 'cygwin': @@ -145,8 +147,13 @@ def run_gsutil(target, args, clean=False): sys.exit(subprocess.call(cmd)) assert sys.platform != 'cygwin' + # Run "gsutil" through "vpython". We need to do this because on GCE instances, + # expectations are made about Python having access to "google-compute-engine" + # and "boto" packages that are not met with non-system Python (e.g., bundles). cmd = [ - sys.executable, + 'vpython', + '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'), + '--', gsutil_bin ] + disable_update + args return subprocess.call(cmd, shell=IS_WINDOWS) @@ -156,19 +163,13 @@ def parse_args(): bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR) parser = argparse.ArgumentParser() - + parser.add_argument('--force-version', default='4.30') parser.add_argument('--clean', action='store_true', help='Clear any existing gsutil package, forcing a new download.') + parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL) parser.add_argument('--target', default=bin_dir, help='The target directory to download/store a gsutil version in. ' '(default is %(default)s).') - - # These two args exist for backwards-compatibility but are no-ops. - parser.add_argument('--force-version', default=VERSION, - help='(deprecated, this flag has no effect)') - parser.add_argument('--fallback', - help='(deprecated, this flag has no effect)') - parser.add_argument('args', nargs=argparse.REMAINDER) args, extras = parser.parse_known_args() @@ -181,7 +182,8 @@ def parse_args(): def main(): args = parse_args() - return run_gsutil(args.target, args.args, clean=args.clean) + return run_gsutil(args.force_version, args.fallback, args.target, args.args, + clean=args.clean) if __name__ == '__main__': diff --git a/gsutil.vpython b/gsutil.vpython new file mode 100644 index 000000000..aa47e5918 --- /dev/null +++ b/gsutil.vpython @@ -0,0 +1,115 @@ +# vpython VirtualEnv protobuf for "gsutil". +# +# See: +# https://chromium.googlesource.com/infra/luci/luci-go/+/HEAD/vpython/api/vpython/spec.proto +# +# This is a "vpython" VirtualEnv specification applied to invocations of "gsutil" +# by the bootstrap wrapper, "gsutil.py". It ensures that any Python distribution +# has the expected Python packages installed. +# +# This is specifically relevant on Google Compute Engine invocations of +# "gsutil", where a stock system-deployed file in "/etc/" explicitly specifies +# that the "google_compute_engine" and "boto" Python packages are available. + +python_version: "2.7" + +wheel < + name: "infra/python/wheels/google_compute_engine-py2_py3" + version: "version:2.6.2" +> +wheel < + name: "infra/python/wheels/boto-py2_py3" + version: "version:2.48.0" +> + +# "gsutil" on non-GCE can require PyOpenSSL, which, in turn, requires +# "cryptography". + +wheel: < + name: "infra/python/wheels/pyopenssl-py2_py3" + version: "version:17.2.0" +> + +## +# BEGIN "cryptography" dependencies. +## + +wheel: < + name: "infra/python/wheels/cryptography/${vpython_platform}" + version: "version:2.9.2" +> + +wheel: < + name: "infra/python/wheels/appdirs-py2_py3" + version: "version:1.4.3" +> + +wheel: < + name: "infra/python/wheels/enum34-py2" + version: "version:1.1.6" +> + +wheel: < + name: "infra/python/wheels/cffi/${vpython_platform}" + version: "version:1.14.5" +> + +wheel: < + name: "infra/python/wheels/idna-py2_py3" + version: "version:2.5" +> + +wheel: < + name: "infra/python/wheels/ipaddress-py2" + version: "version:1.0.18" +> + +wheel: < + name: "infra/python/wheels/packaging-py2_py3" + version: "version:16.8" +> + +wheel: < + name: "infra/python/wheels/pyasn1-py2_py3" + version: "version:0.2.3" +> + +wheel: < + name: "infra/python/wheels/pycparser-py2_py3" + version: "version:2.17" +> + +wheel: < + name: "infra/python/wheels/pyparsing-py2_py3" + version: "version:2.2.0" +> + +wheel: < + name: "infra/python/wheels/setuptools-py2_py3" + version: "version:34.3.2" +> + +wheel: < + name: "infra/python/wheels/six-py2_py3" + version: "version:1.10.0" +> + +## +# END "cryptography" dependencies. +## + +wheel: < + name: "infra/python/wheels/crcmod/${vpython_platform}" + version: "version:1.7" + match_tag: < + abi: "cp27mu" + platform: "manylinux1_i686" + > + match_tag: < + abi: "cp27mu" + platform: "manylinux1_x86_64" + > + match_tag: < + platform: "macosx_10_6_intel" + > +> diff --git a/tests/gsutil_test.py b/tests/gsutil_test.py index d9b7bbbd5..cd8129104 100755 --- a/tests/gsutil_test.py +++ b/tests/gsutil_test.py @@ -72,7 +72,7 @@ class GsutilUnitTests(unittest.TestCase): setattr(subprocess, 'call', self.old_call) def test_download_gsutil(self): - version = gsutil.VERSION + version = '4.2' filename = 'gsutil_%s.zip' % version full_filename = os.path.join(self.tempdir, filename) fake_file = b'This is gsutil.zip' @@ -113,7 +113,7 @@ class GsutilUnitTests(unittest.TestCase): self.assertEqual(self.fake.expectations, []) def test_ensure_gsutil_full(self): - version = gsutil.VERSION + version = '4.2' gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil') gsutil_bin = os.path.join(gsutil_dir, 'gsutil') gsutil_flag = os.path.join(gsutil_dir, 'install.flag') @@ -137,7 +137,7 @@ class GsutilUnitTests(unittest.TestCase): self.assertEqual(self.fake.expectations, []) def test_ensure_gsutil_short(self): - version = gsutil.VERSION + version = '4.2' gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil') gsutil_bin = os.path.join(gsutil_dir, 'gsutil') gsutil_flag = os.path.join(gsutil_dir, 'install.flag') diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index 2ecdf4e33..332bb9a47 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.