diff --git a/download_from_google_storage.py b/download_from_google_storage.py
index 5dda6cf7e..a51b16a25 100755
--- a/download_from_google_storage.py
+++ b/download_from_google_storage.py
@@ -44,12 +44,13 @@ def GetNormalizedPlatform():
class Gsutil(object):
"""Call gsutil with some predefined settings. This is a convenience object,
and is also immutable."""
- def __init__(self, path, boto_path, timeout=None):
+ def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False):
if not os.path.exists(path):
raise FileNotFoundError('GSUtil not found in %s' % path)
self.path = path
self.timeout = timeout
self.boto_path = boto_path
+ self.bypass_prodaccess = bypass_prodaccess
def get_sub_env(self):
env = os.environ.copy()
@@ -68,13 +69,19 @@ class Gsutil(object):
return env
def call(self, *args):
- return subprocess2.call((sys.executable, self.path) + args,
- env=self.get_sub_env(),
- timeout=self.timeout)
+ cmd = [sys.executable, self.path]
+ if self.bypass_prodaccess:
+ cmd.append('--bypass_prodaccess')
+ cmd.extend(args)
+ return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)
def check_call(self, *args):
+ cmd = [sys.executable, self.path]
+ if self.bypass_prodaccess:
+ cmd.append('--bypass_prodaccess')
+ cmd.extend(args)
((out, err), code) = subprocess2.communicate(
- (sys.executable, self.path) + args,
+ cmd,
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE,
env=self.get_sub_env(),
@@ -343,17 +350,14 @@ def main(args):
if options.no_auth:
options.boto = os.devnull
- # Make sure we can find a working instance of gsutil.
+ # Make sure gsutil exists where we expect it to.
if os.path.exists(GSUTIL_DEFAULT_PATH):
- gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
+ gsutil = Gsutil(GSUTIL_DEFAULT_PATH,
+ boto_path=options.boto,
+ bypass_prodaccess=options.no_auth)
else:
- gsutil = None
- for path in os.environ["PATH"].split(os.pathsep):
- if os.path.exists(path) and 'gsutil' in os.listdir(path):
- gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
- if not gsutil:
- parser.error('gsutil not found in %s, bad depot_tools checkout?' %
- GSUTIL_DEFAULT_PATH)
+ parser.error('gsutil not found in %s, bad depot_tools checkout?' %
+ GSUTIL_DEFAULT_PATH)
# Passing in -g/--config will run our copy of GSUtil, then quit.
if options.config:
diff --git a/tests/download_from_google_storage_unittests.py b/tests/download_from_google_storage_unittests.py
index 8a9aa7c41..e5e2055f7 100755
--- a/tests/download_from_google_storage_unittests.py
+++ b/tests/download_from_google_storage_unittests.py
@@ -75,15 +75,6 @@ class GstoolsUnitTests(unittest.TestCase):
self.assertEqual(code, 0)
self.assertEqual(err, '')
- def test_gsutil_version(self):
- gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None)
- _, _, err = gsutil.check_call('version')
- err_lines = err.splitlines()
- self.assertEqual(err_lines[0], 'gsutil version 3.25')
- self.assertEqual(
- err_lines[1],
- 'checksum c9cffb512f467c0aa54880788b9ee6ca (OK)')
-
def test_get_sha1(self):
lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt')
self.assertEqual(
diff --git a/third_party/gsutil/README.chromium b/third_party/gsutil/README.chromium
index 73202b5f4..8f91e70c7 100644
--- a/third_party/gsutil/README.chromium
+++ b/third_party/gsutil/README.chromium
@@ -15,6 +15,9 @@ Modifications:
* Moved gsutil/third_party into our own third_party directory
* Append sys.path in gsutil/gsutil to find the moved third_party modules
* Updated checksum ce71ac982f1148315e7fa65cff2f83e8 -> c9cffb512f467c0aa54880788b9ee6ca
-* Remove code to remove http_proxy before boto.config invocation.
+* Removed code to remove http_proxy before boto.config invocation.
+* Added and imports gsutil/plugins/sso_auth.py to support prodaccess
+ based authentication.
+* Added flag to bypass prodaccess authentication.
Full license is in the COPYING file.
diff --git a/third_party/gsutil/gslib/command.py b/third_party/gsutil/gslib/command.py
index 076455baf..308b04839 100644
--- a/third_party/gsutil/gslib/command.py
+++ b/third_party/gsutil/gslib/command.py
@@ -169,7 +169,8 @@ class Command(object):
def __init__(self, command_runner, args, headers, debug, parallel_operations,
gsutil_bin_dir, boto_lib_dir, config_file_list, gsutil_ver,
- bucket_storage_uri_class, test_method=None):
+ bucket_storage_uri_class, test_method=None,
+ bypass_prodaccess=True):
"""
Args:
command_runner: CommandRunner (for commands built atop other commands).
@@ -186,6 +187,7 @@ class Command(object):
test_method: Optional general purpose method for testing purposes.
Application and semantics of this method will vary by
command and test type.
+ bypass_prodaccess: Boolean to ignore the existance of prodaccess.
Implementation note: subclasses shouldn't need to define an __init__
method, and instead depend on the shared initialization that happens
@@ -209,6 +211,7 @@ class Command(object):
self.exclude_symlinks = False
self.recursion_requested = False
self.all_versions = False
+ self.bypass_prodaccess = bypass_prodaccess
# Process sub-command instance specifications.
# First, ensure subclass implementation sets all required keys.
@@ -343,9 +346,9 @@ class Command(object):
if os.path.isfile(acl_arg):
acl_file = open(acl_arg, 'r')
acl_arg = acl_file.read()
-
+
# TODO: Remove this workaround when GCS allows
- # whitespace in the Permission element on the server-side
+ # whitespace in the Permission element on the server-side
acl_arg = re.sub(r'\s*(\S+)\s*',
r'\1', acl_arg)
@@ -642,7 +645,7 @@ class Command(object):
def _ConfigureNoOpAuthIfNeeded(self):
"""Sets up no-op auth handler if no boto credentials are configured."""
config = boto.config
- if not util.HasConfiguredCredentials():
+ if not util.HasConfiguredCredentials(self.bypass_prodaccess):
if self.config_file_list:
if (config.has_option('Credentials', 'gs_oauth2_refresh_token')
and not HAVE_OAUTH2):
diff --git a/third_party/gsutil/gslib/util.py b/third_party/gsutil/gslib/util.py
index 75daba8ab..c80578855 100644
--- a/third_party/gsutil/gslib/util.py
+++ b/third_party/gsutil/gslib/util.py
@@ -16,6 +16,7 @@
import math
import re
+import os
import sys
import time
@@ -61,7 +62,7 @@ class ListingStyle(object):
LONG_LONG = 'LONG_LONG'
-def HasConfiguredCredentials():
+def HasConfiguredCredentials(bypass_prodaccess):
"""Determines if boto credential/config file exists."""
config = boto.config
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
@@ -71,8 +72,10 @@ def HasConfiguredCredentials():
has_oauth_creds = (HAVE_OAUTH2 and
config.has_option('Credentials', 'gs_oauth2_refresh_token'))
has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
+ # Pretend prodaccess doesn't exist if --bypass_prodaccess is passed in.
+ has_prodaccess = HasExecutable('prodaccess') and not bypass_prodaccess
return (has_goog_creds or has_amzn_creds or has_oauth_creds
- or has_auth_plugins)
+ or has_auth_plugins or has_prodaccess)
def _RoundToNearestExponent(num):
@@ -153,3 +156,12 @@ def ExtractErrorDetail(e):
if detail_start != -1 and detail_end != -1:
return (exc_name, e.body[detail_start+9:detail_end])
return (exc_name, None)
+
+
+def HasExecutable(filename):
+ """Determines if an executable is available on the system."""
+ for path in os.environ['PATH'].split(os.pathsep):
+ exe_file = os.path.join(path, filename)
+ if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
+ return True
+ return False
diff --git a/third_party/gsutil/gsutil b/third_party/gsutil/gsutil
index aca0c7809..e6509edca 100755
--- a/third_party/gsutil/gsutil
+++ b/third_party/gsutil/gsutil
@@ -109,6 +109,7 @@ def main():
if sys.version_info[:3] < (2, 6):
raise CommandException('gsutil requires Python 2.6 or higher.')
+ bypass_prodaccess = False
config_file_list = _GetBotoConfigFileList()
command_runner = CommandRunner(gsutil_bin_dir, boto_lib_dir, config_file_list,
gsutil_ver)
@@ -128,9 +129,10 @@ def main():
boto.config.setbool('Boto', 'https_validate_certificates', True)
try:
- opts, args = getopt.getopt(sys.argv[1:], 'dDvh:m',
+ opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mb',
['debug', 'detailedDebug', 'version', 'help',
- 'header', 'multithreaded'])
+ 'header', 'multithreaded',
+ 'bypass_prodaccess'])
except getopt.GetoptError, e:
_HandleCommandException(CommandException(e.msg))
for o, a in opts:
@@ -155,6 +157,8 @@ def main():
headers[hdr_name] = hdr_val
if o in ('-m', '--multithreaded'):
parallel_operations = True
+ if o in ('-b', '--bypass_prodaccess'):
+ bypass_prodaccess = True
if debug > 1:
sys.stderr.write(
'***************************** WARNING *****************************\n'
@@ -186,9 +190,13 @@ def main():
else:
command_name = args[0]
+ if not bypass_prodaccess:
+ import plugins.sso_auth
+
return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
args[1:], headers, debug,
- parallel_operations)
+ parallel_operations,
+ bypass_prodaccess)
def _GetBotoConfigFileList():
@@ -244,7 +252,8 @@ def _HandleSigQuit(signal_num, cur_stack_frame):
def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
headers=None, debug=0,
- parallel_operations=False):
+ parallel_operations=False,
+ bypass_prodaccess=False):
try:
# Catch ^C so we can print a brief message instead of the normal Python
# stack trace.
@@ -283,7 +292,7 @@ def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
# config file (who might previously have been using gsutil only for
# accessing publicly readable buckets and objects).
if e.status == 403:
- if not HasConfiguredCredentials():
+ if not HasConfiguredCredentials(bypass_prodaccess):
_OutputAndExit(
'You are attempting to access protected data with no configured '
'credentials.\nPlease see '
diff --git a/third_party/gsutil/plugins/__init__.py b/third_party/gsutil/plugins/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/third_party/gsutil/plugins/sso_auth.py b/third_party/gsutil/plugins/sso_auth.py
new file mode 100644
index 000000000..71f0cc9c2
--- /dev/null
+++ b/third_party/gsutil/plugins/sso_auth.py
@@ -0,0 +1,105 @@
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""AuthHandler plugin for gsutil's boto to support LOAS based auth."""
+
+import getpass
+import json
+import os
+import re
+import subprocess
+import time
+import urllib2
+
+from boto.auth_handler import AuthHandler
+from boto.auth_handler import NotReadyToAuthenticate
+
+CMD = ['stubby', '--proto2', 'call', 'blade:sso', 'CorpLogin.Exchange']
+
+STUBBY_CMD = """target: {
+ scope: GAIA_USER
+ name: "%s"
+}
+target_credential: {
+ type: OAUTH2_TOKEN
+ oauth2_attributes: {
+ scope: 'https://www.googleapis.com/auth/devstorage.read_only'
+ }
+}"""
+
+COOKIE_LOCATION = os.path.expanduser('~/.devstore_token')
+
+TOKEN_EXPIRY = 300
+
+
+class SSOAuthError(Exception):
+ pass
+
+
+class SSOAuth(AuthHandler):
+ """SSO based auth handler."""
+
+ capability = ['google-oauth2', 's3']
+
+ def __init__(self, path, config, provider):
+ if provider.name == 'google' and self.has_prodaccess():
+ # If we don't have a loas token, then bypass this auth handler.
+ if subprocess.call('loas_check',
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE):
+ raise NotReadyToAuthenticate()
+ else:
+ raise NotReadyToAuthenticate()
+ self.token = None
+ self.expire = 0
+
+ def GetAccessToken(self):
+ """Returns a valid devstore access token.
+
+ This will return from an in-memory cache if the token is there already,
+ then try a filesystem cache, and then runs a stubby call if none of the
+ caches have a valid token.
+ """
+ if self.token and self.expire > time.time():
+ return self.token
+
+ # Try to retrieve token from filesystem cache.
+ if os.path.exists(COOKIE_LOCATION):
+ last_modified = os.path.getmtime(COOKIE_LOCATION)
+ if time.time() - last_modified < TOKEN_EXPIRY:
+ with open(COOKIE_LOCATION, 'rb') as f:
+ self.token = f.read()
+ self.expire = last_modified + TOKEN_EXPIRY
+ return self.token
+
+ # If the token is not in either caches, or has expired, then fetch token.
+ username = '%s@google.com' % getpass.getuser()
+ proc = subprocess.Popen(CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ out, err = proc.communicate(STUBBY_CMD % username)
+ if proc.returncode:
+ raise SSOAuthError('Stubby returned %d\n%s' % (proc.returncode, err))
+ token_match = re.search(r'oauth2_token: "(.*)"$', out)
+
+ if not token_match:
+ raise SSOAuthError('Oauth2 token not found in %s' % out)
+
+ token = token_match.group(1)
+ self.token = token
+ self.expire = time.time() + TOKEN_EXPIRY
+ with os.fdopen(os.open(COOKIE_LOCATION,
+ os.O_WRONLY | os.O_CREAT,
+ 0600), 'wb') as f:
+ f.write(token)
+ return token
+
+ def add_auth(self, http_request):
+ http_request.headers['Authorization'] = 'OAuth %s' % self.GetAccessToken()
+
+ @staticmethod
+ def has_prodaccess():
+ for path in os.environ['PATH'].split(os.pathsep):
+ exe_file = os.path.join(path, 'prodaccess')
+ if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
+ return True
+ return False
diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py
index e6cd2eeb5..35bc03936 100755
--- a/upload_to_google_storage.py
+++ b/upload_to_google_storage.py
@@ -234,7 +234,8 @@ def main(args):
# Make sure we can find a working instance of gsutil.
if os.path.exists(GSUTIL_DEFAULT_PATH):
- gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
+ gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto,
+ bypass_prodaccess=True)
else:
gsutil = None
for path in os.environ["PATH"].split(os.pathsep):