From f059ec936899adfd83f63fff46fdb61ae3a39537 Mon Sep 17 00:00:00 2001 From: Dirk Pranke Date: Mon, 17 May 2021 18:26:44 +0000 Subject: [PATCH] Update gsutil to use gsutil version 4.61, python3. This CL updates the gsutil.py wrapper to download and use v4.61 of GCP's gsutil, which is Python3-compatible. v4.61 appears to be fully self-contained and have all of the packages it needs vendored into it. So, there's no reason to use vpython anymore, and this CL removes that. Also, this CL removes the 'fallback' option to gsutil and the ability to force a version switch, as this should no longer be necessary (it was added for a migration back in 2014 but apparently this code was never removed afterwards). This CL also updates download_from_google_storage.py and upload_to_google_storage.py to similarly not have the version flags and to just use regular python3, not vpython3. Bug: 1184108 Change-Id: I0d1a8351dba2d3ad1f927afa333fb10959f19443 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2898439 Reviewed-by: Mike Frysinger Reviewed-by: Josip Sokcevic Reviewed-by: Robbie Iannucci Commit-Queue: Dirk Pranke --- download_from_google_storage.py | 9 +-- gsutil.py | 32 +++++---- gsutil.vpython | 115 -------------------------------- tests/gsutil_test.py | 6 +- upload_to_google_storage.py | 2 +- 5 files changed, 22 insertions(+), 142 deletions(-) delete mode 100644 gsutil.vpython diff --git a/download_from_google_storage.py b/download_from_google_storage.py index 84bc8e30db..d416d1444e 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -76,15 +76,12 @@ class Gsutil(object): MAX_TRIES = 5 RETRY_BASE_DELAY = 5.0 RETRY_DELAY_MULTIPLE = 1.3 - VPYTHON3 = ('vpython3.bat' - if GetNormalizedPlatform() == 'win32' else 'vpython3') - def __init__(self, path, boto_path=None, version='4.28'): + def __init__(self, path, boto_path=None): if not os.path.exists(path): raise FileNotFoundError('GSUtil not found in %s' % path) self.path = path self.boto_path = boto_path - self.version = version def get_sub_env(self): env = os.environ.copy() @@ -101,12 +98,12 @@ class Gsutil(object): return env def call(self, *args): - cmd = [self.VPYTHON3, self.path, '--force-version', self.version] + cmd = [sys.executable, self.path] cmd.extend(args) return subprocess2.call(cmd, env=self.get_sub_env()) def check_call(self, *args): - cmd = [self.VPYTHON3, self.path, '--force-version', self.version] + cmd = [sys.executable, self.path] cmd.extend(args) ((out, err), code) = subprocess2.communicate( cmd, diff --git a/gsutil.py b/gsutil.py index d5423ad7ef..6ed3773852 100755 --- a/gsutil.py +++ b/gsutil.py @@ -29,13 +29,14 @@ import zipfile GSUTIL_URL = 'https://storage.googleapis.com/pub/' API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' + THIS_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil') -DEFAULT_FALLBACK_GSUTIL = os.path.join( - THIS_DIR, 'third_party', 'gsutil', 'gsutil') IS_WINDOWS = os.name == 'nt' +VERSION = '4.61' + class InvalidGsutilError(Exception): pass @@ -129,11 +130,8 @@ def ensure_gsutil(version, target, clean): return gsutil_bin -def run_gsutil(force_version, fallback, target, args, clean=False): - if force_version: - gsutil_bin = ensure_gsutil(force_version, target, clean) - else: - gsutil_bin = fallback +def run_gsutil(target, args, clean=False): + gsutil_bin = ensure_gsutil(VERSION, target, clean) disable_update = ['-o', 'GSUtil:software_update_check_period=0'] if sys.platform == 'cygwin': @@ -147,13 +145,8 @@ def run_gsutil(force_version, fallback, target, args, clean=False): sys.exit(subprocess.call(cmd)) assert sys.platform != 'cygwin' - # Run "gsutil" through "vpython". We need to do this because on GCE instances, - # expectations are made about Python having access to "google-compute-engine" - # and "boto" packages that are not met with non-system Python (e.g., bundles). cmd = [ - 'vpython', - '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'), - '--', + sys.executable, gsutil_bin ] + disable_update + args return subprocess.call(cmd, shell=IS_WINDOWS) @@ -163,13 +156,19 @@ def parse_args(): bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR) parser = argparse.ArgumentParser() - parser.add_argument('--force-version', default='4.30') + parser.add_argument('--clean', action='store_true', help='Clear any existing gsutil package, forcing a new download.') - parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL) parser.add_argument('--target', default=bin_dir, help='The target directory to download/store a gsutil version in. ' '(default is %(default)s).') + + # These two args exist for backwards-compatibility but are no-ops. + parser.add_argument('--force-version', default=VERSION, + help='(deprecated, this flag has no effect)') + parser.add_argument('--fallback', + help='(deprecated, this flag has no effect)') + parser.add_argument('args', nargs=argparse.REMAINDER) args, extras = parser.parse_known_args() @@ -182,8 +181,7 @@ def parse_args(): def main(): args = parse_args() - return run_gsutil(args.force_version, args.fallback, args.target, args.args, - clean=args.clean) + return run_gsutil(args.target, args.args, clean=args.clean) if __name__ == '__main__': diff --git a/gsutil.vpython b/gsutil.vpython deleted file mode 100644 index aa47e5918f..0000000000 --- a/gsutil.vpython +++ /dev/null @@ -1,115 +0,0 @@ -# vpython VirtualEnv protobuf for "gsutil". -# -# See: -# https://chromium.googlesource.com/infra/luci/luci-go/+/HEAD/vpython/api/vpython/spec.proto -# -# This is a "vpython" VirtualEnv specification applied to invocations of "gsutil" -# by the bootstrap wrapper, "gsutil.py". It ensures that any Python distribution -# has the expected Python packages installed. -# -# This is specifically relevant on Google Compute Engine invocations of -# "gsutil", where a stock system-deployed file in "/etc/" explicitly specifies -# that the "google_compute_engine" and "boto" Python packages are available. - -python_version: "2.7" - -wheel < - name: "infra/python/wheels/google_compute_engine-py2_py3" - version: "version:2.6.2" -> -wheel < - name: "infra/python/wheels/boto-py2_py3" - version: "version:2.48.0" -> - -# "gsutil" on non-GCE can require PyOpenSSL, which, in turn, requires -# "cryptography". - -wheel: < - name: "infra/python/wheels/pyopenssl-py2_py3" - version: "version:17.2.0" -> - -## -# BEGIN "cryptography" dependencies. -## - -wheel: < - name: "infra/python/wheels/cryptography/${vpython_platform}" - version: "version:2.9.2" -> - -wheel: < - name: "infra/python/wheels/appdirs-py2_py3" - version: "version:1.4.3" -> - -wheel: < - name: "infra/python/wheels/enum34-py2" - version: "version:1.1.6" -> - -wheel: < - name: "infra/python/wheels/cffi/${vpython_platform}" - version: "version:1.14.5" -> - -wheel: < - name: "infra/python/wheels/idna-py2_py3" - version: "version:2.5" -> - -wheel: < - name: "infra/python/wheels/ipaddress-py2" - version: "version:1.0.18" -> - -wheel: < - name: "infra/python/wheels/packaging-py2_py3" - version: "version:16.8" -> - -wheel: < - name: "infra/python/wheels/pyasn1-py2_py3" - version: "version:0.2.3" -> - -wheel: < - name: "infra/python/wheels/pycparser-py2_py3" - version: "version:2.17" -> - -wheel: < - name: "infra/python/wheels/pyparsing-py2_py3" - version: "version:2.2.0" -> - -wheel: < - name: "infra/python/wheels/setuptools-py2_py3" - version: "version:34.3.2" -> - -wheel: < - name: "infra/python/wheels/six-py2_py3" - version: "version:1.10.0" -> - -## -# END "cryptography" dependencies. -## - -wheel: < - name: "infra/python/wheels/crcmod/${vpython_platform}" - version: "version:1.7" - match_tag: < - abi: "cp27mu" - platform: "manylinux1_i686" - > - match_tag: < - abi: "cp27mu" - platform: "manylinux1_x86_64" - > - match_tag: < - platform: "macosx_10_6_intel" - > -> diff --git a/tests/gsutil_test.py b/tests/gsutil_test.py index cd81291046..d9b7bbbd5c 100755 --- a/tests/gsutil_test.py +++ b/tests/gsutil_test.py @@ -72,7 +72,7 @@ class GsutilUnitTests(unittest.TestCase): setattr(subprocess, 'call', self.old_call) def test_download_gsutil(self): - version = '4.2' + version = gsutil.VERSION filename = 'gsutil_%s.zip' % version full_filename = os.path.join(self.tempdir, filename) fake_file = b'This is gsutil.zip' @@ -113,7 +113,7 @@ class GsutilUnitTests(unittest.TestCase): self.assertEqual(self.fake.expectations, []) def test_ensure_gsutil_full(self): - version = '4.2' + version = gsutil.VERSION gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil') gsutil_bin = os.path.join(gsutil_dir, 'gsutil') gsutil_flag = os.path.join(gsutil_dir, 'install.flag') @@ -137,7 +137,7 @@ class GsutilUnitTests(unittest.TestCase): self.assertEqual(self.fake.expectations, []) def test_ensure_gsutil_short(self): - version = '4.2' + version = gsutil.VERSION gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil') gsutil_bin = os.path.join(gsutil_dir, 'gsutil') gsutil_flag = os.path.join(gsutil_dir, 'install.flag') diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index 332bb9a47b..2ecdf4e337 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.