Adds SSO auth to gsutil

Code path:
1. plugins.sso_auth is imported, which adds the AuthHandler class to the global state.
2. HasConfiguredCredentials() in gslib/utils.py is called by gsutil, and will return true if "prodaccess" exists on the system, which tells the system that we don't want a no-op auth handler.
3. When a command is called, all the auth handlers are cycled through and sso_auth.SSOAuth is called, which calls a stubby command to emit a gaiamint'ed oauth2 access token, which is then used as the Authorization Header

if --bypass_prodaccess is passed in, then:
1. HasConfiguredCredentials() will bypass the check for prodaccess, as if it didn't exist.
2. plugins.sso_auth does not get imported.
Which will essentially cause gsutil to behave as if this patch never existed.

So the expected behavior is:
=.boto file does not exist, prodaccess exists, but unauthenticated=
Failure: No handler was ready to authenticate. 3 handlers were checked. ['OAuth2Auth', 'HmacAuthV1Handler', 'SSOAuth'] Check your credentials.

=.boto file exists, prodaccess exists, but unauthenticated=
sso_auth will raise NotReadyToAuthenticate, and the .boto file will be used instead

=.boto file exists, prodaccess exists, authenticated=
sso_auth will be run _after_ the default gsutil authenticator, which causes the sso_auth to be used over whatever the default authentication is.

bypass_prodaccess is passed in by default to upload_to_google_storage because we expect people who use upload_to_google_storage to not need prodaccess and have their own boto file already.  Also the sso_auth plugin will only request a readonlyi token, which will not work for uploading.

BUG=258152

Review URL: https://codereview.chromium.org/86123002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@240266 0039d316-1c4b-4281-b951-d872f2087c98
experimental/szager/collated-output
hinoka@chromium.org 12 years ago
parent aeab41a341
commit c6a2ee6930

@ -44,12 +44,13 @@ def GetNormalizedPlatform():
class Gsutil(object):
"""Call gsutil with some predefined settings. This is a convenience object,
and is also immutable."""
def __init__(self, path, boto_path, timeout=None):
def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False):
if not os.path.exists(path):
raise FileNotFoundError('GSUtil not found in %s' % path)
self.path = path
self.timeout = timeout
self.boto_path = boto_path
self.bypass_prodaccess = bypass_prodaccess
def get_sub_env(self):
env = os.environ.copy()
@ -68,13 +69,19 @@ class Gsutil(object):
return env
def call(self, *args):
return subprocess2.call((sys.executable, self.path) + args,
env=self.get_sub_env(),
timeout=self.timeout)
cmd = [sys.executable, self.path]
if self.bypass_prodaccess:
cmd.append('--bypass_prodaccess')
cmd.extend(args)
return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)
def check_call(self, *args):
cmd = [sys.executable, self.path]
if self.bypass_prodaccess:
cmd.append('--bypass_prodaccess')
cmd.extend(args)
((out, err), code) = subprocess2.communicate(
(sys.executable, self.path) + args,
cmd,
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE,
env=self.get_sub_env(),
@ -343,15 +350,12 @@ def main(args):
if options.no_auth:
options.boto = os.devnull
# Make sure we can find a working instance of gsutil.
# Make sure gsutil exists where we expect it to.
if os.path.exists(GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
gsutil = Gsutil(GSUTIL_DEFAULT_PATH,
boto_path=options.boto,
bypass_prodaccess=options.no_auth)
else:
gsutil = None
for path in os.environ["PATH"].split(os.pathsep):
if os.path.exists(path) and 'gsutil' in os.listdir(path):
gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
if not gsutil:
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
GSUTIL_DEFAULT_PATH)

@ -75,15 +75,6 @@ class GstoolsUnitTests(unittest.TestCase):
self.assertEqual(code, 0)
self.assertEqual(err, '')
def test_gsutil_version(self):
gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None)
_, _, err = gsutil.check_call('version')
err_lines = err.splitlines()
self.assertEqual(err_lines[0], 'gsutil version 3.25')
self.assertEqual(
err_lines[1],
'checksum c9cffb512f467c0aa54880788b9ee6ca (OK)')
def test_get_sha1(self):
lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt')
self.assertEqual(

@ -15,6 +15,9 @@ Modifications:
* Moved gsutil/third_party into our own third_party directory
* Append sys.path in gsutil/gsutil to find the moved third_party modules
* Updated checksum ce71ac982f1148315e7fa65cff2f83e8 -> c9cffb512f467c0aa54880788b9ee6ca
* Remove code to remove http_proxy before boto.config invocation.
* Removed code to remove http_proxy before boto.config invocation.
* Added and imports gsutil/plugins/sso_auth.py to support prodaccess
based authentication.
* Added flag to bypass prodaccess authentication.
Full license is in the COPYING file.

@ -169,7 +169,8 @@ class Command(object):
def __init__(self, command_runner, args, headers, debug, parallel_operations,
gsutil_bin_dir, boto_lib_dir, config_file_list, gsutil_ver,
bucket_storage_uri_class, test_method=None):
bucket_storage_uri_class, test_method=None,
bypass_prodaccess=True):
"""
Args:
command_runner: CommandRunner (for commands built atop other commands).
@ -186,6 +187,7 @@ class Command(object):
test_method: Optional general purpose method for testing purposes.
Application and semantics of this method will vary by
command and test type.
bypass_prodaccess: Boolean to ignore the existance of prodaccess.
Implementation note: subclasses shouldn't need to define an __init__
method, and instead depend on the shared initialization that happens
@ -209,6 +211,7 @@ class Command(object):
self.exclude_symlinks = False
self.recursion_requested = False
self.all_versions = False
self.bypass_prodaccess = bypass_prodaccess
# Process sub-command instance specifications.
# First, ensure subclass implementation sets all required keys.
@ -642,7 +645,7 @@ class Command(object):
def _ConfigureNoOpAuthIfNeeded(self):
"""Sets up no-op auth handler if no boto credentials are configured."""
config = boto.config
if not util.HasConfiguredCredentials():
if not util.HasConfiguredCredentials(self.bypass_prodaccess):
if self.config_file_list:
if (config.has_option('Credentials', 'gs_oauth2_refresh_token')
and not HAVE_OAUTH2):

@ -16,6 +16,7 @@
import math
import re
import os
import sys
import time
@ -61,7 +62,7 @@ class ListingStyle(object):
LONG_LONG = 'LONG_LONG'
def HasConfiguredCredentials():
def HasConfiguredCredentials(bypass_prodaccess):
"""Determines if boto credential/config file exists."""
config = boto.config
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
@ -71,8 +72,10 @@ def HasConfiguredCredentials():
has_oauth_creds = (HAVE_OAUTH2 and
config.has_option('Credentials', 'gs_oauth2_refresh_token'))
has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
# Pretend prodaccess doesn't exist if --bypass_prodaccess is passed in.
has_prodaccess = HasExecutable('prodaccess') and not bypass_prodaccess
return (has_goog_creds or has_amzn_creds or has_oauth_creds
or has_auth_plugins)
or has_auth_plugins or has_prodaccess)
def _RoundToNearestExponent(num):
@ -153,3 +156,12 @@ def ExtractErrorDetail(e):
if detail_start != -1 and detail_end != -1:
return (exc_name, e.body[detail_start+9:detail_end])
return (exc_name, None)
def HasExecutable(filename):
"""Determines if an executable is available on the system."""
for path in os.environ['PATH'].split(os.pathsep):
exe_file = os.path.join(path, filename)
if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
return True
return False

@ -109,6 +109,7 @@ def main():
if sys.version_info[:3] < (2, 6):
raise CommandException('gsutil requires Python 2.6 or higher.')
bypass_prodaccess = False
config_file_list = _GetBotoConfigFileList()
command_runner = CommandRunner(gsutil_bin_dir, boto_lib_dir, config_file_list,
gsutil_ver)
@ -128,9 +129,10 @@ def main():
boto.config.setbool('Boto', 'https_validate_certificates', True)
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvh:m',
opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mb',
['debug', 'detailedDebug', 'version', 'help',
'header', 'multithreaded'])
'header', 'multithreaded',
'bypass_prodaccess'])
except getopt.GetoptError, e:
_HandleCommandException(CommandException(e.msg))
for o, a in opts:
@ -155,6 +157,8 @@ def main():
headers[hdr_name] = hdr_val
if o in ('-m', '--multithreaded'):
parallel_operations = True
if o in ('-b', '--bypass_prodaccess'):
bypass_prodaccess = True
if debug > 1:
sys.stderr.write(
'***************************** WARNING *****************************\n'
@ -186,9 +190,13 @@ def main():
else:
command_name = args[0]
if not bypass_prodaccess:
import plugins.sso_auth
return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
args[1:], headers, debug,
parallel_operations)
parallel_operations,
bypass_prodaccess)
def _GetBotoConfigFileList():
@ -244,7 +252,8 @@ def _HandleSigQuit(signal_num, cur_stack_frame):
def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
headers=None, debug=0,
parallel_operations=False):
parallel_operations=False,
bypass_prodaccess=False):
try:
# Catch ^C so we can print a brief message instead of the normal Python
# stack trace.
@ -283,7 +292,7 @@ def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
# config file (who might previously have been using gsutil only for
# accessing publicly readable buckets and objects).
if e.status == 403:
if not HasConfiguredCredentials():
if not HasConfiguredCredentials(bypass_prodaccess):
_OutputAndExit(
'You are attempting to access protected data with no configured '
'credentials.\nPlease see '

@ -0,0 +1,105 @@
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""AuthHandler plugin for gsutil's boto to support LOAS based auth."""
import getpass
import json
import os
import re
import subprocess
import time
import urllib2
from boto.auth_handler import AuthHandler
from boto.auth_handler import NotReadyToAuthenticate
CMD = ['stubby', '--proto2', 'call', 'blade:sso', 'CorpLogin.Exchange']
STUBBY_CMD = """target: {
scope: GAIA_USER
name: "%s"
}
target_credential: {
type: OAUTH2_TOKEN
oauth2_attributes: {
scope: 'https://www.googleapis.com/auth/devstorage.read_only'
}
}"""
COOKIE_LOCATION = os.path.expanduser('~/.devstore_token')
TOKEN_EXPIRY = 300
class SSOAuthError(Exception):
pass
class SSOAuth(AuthHandler):
"""SSO based auth handler."""
capability = ['google-oauth2', 's3']
def __init__(self, path, config, provider):
if provider.name == 'google' and self.has_prodaccess():
# If we don't have a loas token, then bypass this auth handler.
if subprocess.call('loas_check',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE):
raise NotReadyToAuthenticate()
else:
raise NotReadyToAuthenticate()
self.token = None
self.expire = 0
def GetAccessToken(self):
"""Returns a valid devstore access token.
This will return from an in-memory cache if the token is there already,
then try a filesystem cache, and then runs a stubby call if none of the
caches have a valid token.
"""
if self.token and self.expire > time.time():
return self.token
# Try to retrieve token from filesystem cache.
if os.path.exists(COOKIE_LOCATION):
last_modified = os.path.getmtime(COOKIE_LOCATION)
if time.time() - last_modified < TOKEN_EXPIRY:
with open(COOKIE_LOCATION, 'rb') as f:
self.token = f.read()
self.expire = last_modified + TOKEN_EXPIRY
return self.token
# If the token is not in either caches, or has expired, then fetch token.
username = '%s@google.com' % getpass.getuser()
proc = subprocess.Popen(CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out, err = proc.communicate(STUBBY_CMD % username)
if proc.returncode:
raise SSOAuthError('Stubby returned %d\n%s' % (proc.returncode, err))
token_match = re.search(r'oauth2_token: "(.*)"$', out)
if not token_match:
raise SSOAuthError('Oauth2 token not found in %s' % out)
token = token_match.group(1)
self.token = token
self.expire = time.time() + TOKEN_EXPIRY
with os.fdopen(os.open(COOKIE_LOCATION,
os.O_WRONLY | os.O_CREAT,
0600), 'wb') as f:
f.write(token)
return token
def add_auth(self, http_request):
http_request.headers['Authorization'] = 'OAuth %s' % self.GetAccessToken()
@staticmethod
def has_prodaccess():
for path in os.environ['PATH'].split(os.pathsep):
exe_file = os.path.join(path, 'prodaccess')
if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
return True
return False

@ -234,7 +234,8 @@ def main(args):
# Make sure we can find a working instance of gsutil.
if os.path.exists(GSUTIL_DEFAULT_PATH):
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto,
bypass_prodaccess=True)
else:
gsutil = None
for path in os.environ["PATH"].split(os.pathsep):

Loading…
Cancel
Save