Revert "Reland "Update gsutil to use gsutil version 4.61, python3.""

This reverts commit e53a593956. Reason for revert: Speculative revert for build failures on Mac11 bots: crbug.com/1219065. Original change's description: > Reland "Update gsutil to use gsutil version 4.61, python3." > > This reverts commit af121aeec9. > > Reason for revert: re-landing with a switch back to vpython to get the compiled C extension version of crcmod for performance. > > Original change's description: > > Revert "Update gsutil to use gsutil version 4.61, python3." > > > > This reverts commit f059ec9368. > > > > Reason for revert: Reverting because we probably need to be using vpython and a compiled crcmod instead. See, e.g.,. b/188591640. > > > > Original change's description: > > > Update gsutil to use gsutil version 4.61, python3. > > > > > > This CL updates the gsutil.py wrapper to download and use > > > v4.61 of GCP's gsutil, which is Python3-compatible. > > > > > > v4.61 appears to be fully self-contained and have all of the > > > packages it needs vendored into it. So, there's no reason to > > > use vpython anymore, and this CL removes that. > > > > > > Also, this CL removes the 'fallback' option to gsutil and > > > the ability to force a version switch, as this should no > > > longer be necessary (it was added for a migration back in 2014 > > > but apparently this code was never removed afterwards). > > > > > > This CL also updates download_from_google_storage.py and > > > upload_to_google_storage.py to similarly not have the version flags > > > and to just use regular python3, not vpython3. > > > > > > Bug: 1184108 > > > Change-Id: I0d1a8351dba2d3ad1f927afa333fb10959f19443 > > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2898439 > > > Reviewed-by: Mike Frysinger <vapier@chromium.org> > > > Reviewed-by: Josip Sokcevic <sokcevic@google.com> > > > Reviewed-by: Robbie Iannucci <iannucci@chromium.org> > > > Commit-Queue: Dirk Pranke <dpranke@google.com> > > > > Bug: 1184108 > > Change-Id: I8e21a9a40d81e4e185642f866855b6838f80f1c2 > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2905904 > > Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com> > > Commit-Queue: Dirk Pranke <dpranke@google.com> > > Bug: 1184108 > Change-Id: I5d6d6d06842e08517488471c144972818fcf02ff > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2907155 > Reviewed-by: Mike Frysinger <vapier@chromium.org> > Reviewed-by: Josip Sokcevic <sokcevic@google.com> > Commit-Queue: Dirk Pranke <dpranke@google.com> Bug: 1184108 Change-Id: Ib0757e0873d6e6b39621d50d327d959f0fa43720 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2957473 Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com> Commit-Queue: Dirk Pranke <dpranke@google.com>
4 years ago · ef579a1192
parent e5d455cca7
commit ef579a1192
6 changed files with 140 additions and 40 deletions
--- a/download_from_google_storage.py
+++ b/download_from_google_storage.py
@ -79,11 +79,12 @@ class Gsutil(object):
  VPYTHON3 = ('vpython3.bat'
              if GetNormalizedPlatform() == 'win32' else 'vpython3')

-  def __init__(self, path, boto_path=None):
+  def __init__(self, path, boto_path=None, version='4.28'):
    if not os.path.exists(path):
      raise FileNotFoundError('GSUtil not found in %s' % path)
    self.path = path
    self.boto_path = boto_path
+    self.version = version

  def get_sub_env(self):
    env = os.environ.copy()
@ -100,12 +101,12 @@ class Gsutil(object):
    return env

  def call(self, *args):
-    cmd = [self.VPYTHON3, self.path]
+    cmd = [self.VPYTHON3, self.path, '--force-version', self.version]
    cmd.extend(args)
    return subprocess2.call(cmd, env=self.get_sub_env())

  def check_call(self, *args):
-    cmd = [self.VPYTHON3, self.path]
+    cmd = [self.VPYTHON3, self.path, '--force-version', self.version]
    cmd.extend(args)
    ((out, err), code) = subprocess2.communicate(
        cmd,
--- a/gsutil.py
+++ b/gsutil.py
@ -31,11 +31,11 @@ API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/'

 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
+DEFAULT_FALLBACK_GSUTIL = os.path.join(
+    THIS_DIR, 'third_party', 'gsutil', 'gsutil')

 IS_WINDOWS = os.name == 'nt'

-VERSION = '4.61'
-

 class InvalidGsutilError(Exception):
  pass
@ -129,8 +129,11 @@ def ensure_gsutil(version, target, clean):
  return gsutil_bin


-def run_gsutil(target, args, clean=False):
-  gsutil_bin = ensure_gsutil(VERSION, target, clean)
+def run_gsutil(force_version, fallback, target, args, clean=False):
+  if force_version:
+    gsutil_bin = ensure_gsutil(force_version, target, clean)
+  else:
+    gsutil_bin = fallback
  disable_update = ['-o', 'GSUtil:software_update_check_period=0']

  if sys.platform == 'cygwin':
@ -144,8 +147,13 @@ def run_gsutil(target, args, clean=False):
    sys.exit(subprocess.call(cmd))
  assert sys.platform != 'cygwin'

+  # Run "gsutil" through "vpython". We need to do this because on GCE instances,
+  # expectations are made about Python having access to "google-compute-engine"
+  # and "boto" packages that are not met with non-system Python (e.g., bundles).
  cmd = [
-      sys.executable,
+      'vpython',
+      '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'),
+      '--',
      gsutil_bin
  ] + disable_update + args
  return subprocess.call(cmd, shell=IS_WINDOWS)
@ -155,19 +163,13 @@ def parse_args():
  bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR)

  parser = argparse.ArgumentParser()
-
+  parser.add_argument('--force-version', default='4.30')
  parser.add_argument('--clean', action='store_true',
      help='Clear any existing gsutil package, forcing a new download.')
+  parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
  parser.add_argument('--target', default=bin_dir,
      help='The target directory to download/store a gsutil version in. '
           '(default is %(default)s).')
-
-  # These two args exist for backwards-compatibility but are no-ops.
-  parser.add_argument('--force-version', default=VERSION,
-                      help='(deprecated, this flag has no effect)')
-  parser.add_argument('--fallback',
-                      help='(deprecated, this flag has no effect)')
-
  parser.add_argument('args', nargs=argparse.REMAINDER)

  args, extras = parser.parse_known_args()
@ -180,7 +182,8 @@ def parse_args():

 def main():
  args = parse_args()
-  return run_gsutil(args.target, args.args, clean=args.clean)
+  return run_gsutil(args.force_version, args.fallback, args.target, args.args,
+                    clean=args.clean)


 if __name__ == '__main__':
--- a/gsutil.py.vpython3
+++ b/gsutil.py.vpython3
@ -1,19 +0,0 @@
-# vpython VirtualEnv protobuf for "gsutil".
-#
-# See:
-# https://chromium.googlesource.com/infra/luci/luci-go/+/HEAD/vpython/api/vpython/spec.proto
-#
-# This is a "vpython" VirtualEnv specification applied to invocations of "gsutil"
-# by the bootstrap wrapper, "gsutil.py". It ensures that any Python distribution
-# has the expected Python packages installed.
-#
-# This is specifically relevant on Google Compute Engine invocations of
-# "gsutil", where a stock system-deployed file in "/etc/" explicitly specifies
-# that the "google_compute_engine" and "boto" Python packages are available.
-
-python_version: "3.8"
-
-wheel: <
-  name: "infra/python/wheels/crcmod/${vpython_platform}"
-  version: "version:1.7"
->
--- a/gsutil.vpython
+++ b/gsutil.vpython
@ -0,0 +1,115 @@
+# vpython VirtualEnv protobuf for "gsutil".
+#
+# See:
+# https://chromium.googlesource.com/infra/luci/luci-go/+/HEAD/vpython/api/vpython/spec.proto
+#
+# This is a "vpython" VirtualEnv specification applied to invocations of "gsutil"
+# by the bootstrap wrapper, "gsutil.py". It ensures that any Python distribution
+# has the expected Python packages installed.
+#
+# This is specifically relevant on Google Compute Engine invocations of
+# "gsutil", where a stock system-deployed file in "/etc/" explicitly specifies
+# that the "google_compute_engine" and "boto" Python packages are available.
+
+python_version: "2.7"
+
+wheel <
+  name: "infra/python/wheels/google_compute_engine-py2_py3"
+  version: "version:2.6.2"
+>
+wheel <
+  name: "infra/python/wheels/boto-py2_py3"
+  version: "version:2.48.0"
+>
+
+# "gsutil" on non-GCE can require PyOpenSSL, which, in turn, requires
+# "cryptography".
+
+wheel: <
+  name: "infra/python/wheels/pyopenssl-py2_py3"
+  version: "version:17.2.0"
+>
+
+##
+# BEGIN "cryptography" dependencies.
+##
+
+wheel: <
+  name: "infra/python/wheels/cryptography/${vpython_platform}"
+  version: "version:2.9.2"
+>
+
+wheel: <
+  name: "infra/python/wheels/appdirs-py2_py3"
+  version: "version:1.4.3"
+>
+
+wheel: <
+  name: "infra/python/wheels/enum34-py2"
+  version: "version:1.1.6"
+>
+
+wheel: <
+  name: "infra/python/wheels/cffi/${vpython_platform}"
+  version: "version:1.14.5"
+>
+
+wheel: <
+  name: "infra/python/wheels/idna-py2_py3"
+  version: "version:2.5"
+>
+
+wheel: <
+  name: "infra/python/wheels/ipaddress-py2"
+  version: "version:1.0.18"
+>
+
+wheel: <
+  name: "infra/python/wheels/packaging-py2_py3"
+  version: "version:16.8"
+>
+
+wheel: <
+  name: "infra/python/wheels/pyasn1-py2_py3"
+  version: "version:0.2.3"
+>
+
+wheel: <
+  name: "infra/python/wheels/pycparser-py2_py3"
+  version: "version:2.17"
+>
+
+wheel: <
+  name: "infra/python/wheels/pyparsing-py2_py3"
+  version: "version:2.2.0"
+>
+
+wheel: <
+  name: "infra/python/wheels/setuptools-py2_py3"
+  version: "version:34.3.2"
+>
+
+wheel: <
+  name: "infra/python/wheels/six-py2_py3"
+  version: "version:1.10.0"
+>
+
+##
+# END "cryptography" dependencies.
+##
+
+wheel: <
+  name: "infra/python/wheels/crcmod/${vpython_platform}"
+  version: "version:1.7"
+  match_tag: <
+    abi: "cp27mu"
+    platform: "manylinux1_i686"
+  >
+  match_tag: <
+    abi: "cp27mu"
+    platform: "manylinux1_x86_64"
+  >
+  match_tag: <
+    platform: "macosx_10_6_intel"
+  >
+>
--- a/tests/gsutil_test.py
+++ b/tests/gsutil_test.py
@ -72,7 +72,7 @@ class GsutilUnitTests(unittest.TestCase):
    setattr(subprocess, 'call', self.old_call)

  def test_download_gsutil(self):
-    version = gsutil.VERSION
+    version = '4.2'
    filename = 'gsutil_%s.zip' % version
    full_filename = os.path.join(self.tempdir, filename)
    fake_file = b'This is gsutil.zip'
@ -113,7 +113,7 @@ class GsutilUnitTests(unittest.TestCase):
    self.assertEqual(self.fake.expectations, [])

  def test_ensure_gsutil_full(self):
-    version = gsutil.VERSION
+    version = '4.2'
    gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil')
    gsutil_bin = os.path.join(gsutil_dir, 'gsutil')
    gsutil_flag = os.path.join(gsutil_dir, 'install.flag')
@ -137,7 +137,7 @@ class GsutilUnitTests(unittest.TestCase):
    self.assertEqual(self.fake.expectations, [])

  def test_ensure_gsutil_short(self):
-    version = gsutil.VERSION
+    version = '4.2'
    gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil')
    gsutil_bin = os.path.join(gsutil_dir, 'gsutil')
    gsutil_flag = os.path.join(gsutil_dir, 'install.flag')
--- a/upload_to_google_storage.py
+++ b/upload_to_google_storage.py
@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.