Added gsutil/ to depot_tools/third_party/
This is needed for https://chromiumcodereview.appspot.com/12042069/ Which uses gsutil to download objects from Google Storage based on SHA1 sums Continuation of: https://chromiumcodereview.appspot.com/12317103/ Rietveld didn't like a giant CL with all of gsutil (kept crashing on upload), The CL is being split into three parts Related: https://chromiumcodereview.appspot.com/12755026/ (gsutil/boto) https://chromiumcodereview.appspot.com/12685010/ (gsutil/gslib) BUG= Review URL: https://codereview.chromium.org/12685009 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@188896 0039d316-1c4b-4281-b951-d872f2087c98experimental/szager/collated-output
hinoka@google.com
12 years ago
parent
8efca395c0
commit
8d2f67235a
@ -0,0 +1,21 @@
|
||||
The fancy_urllib library was obtained from
|
||||
http://googleappengine.googlecode.com/svn/trunk/python/lib/fancy_urllib/fancy_urllib/__init__.py
|
||||
under the following license (http://googleappengine.googlecode.com/svn/trunk/python/LICENSE):
|
||||
|
||||
|
||||
GOOGLE APP ENGINE SDK
|
||||
=====================
|
||||
Copyright 2008 Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -0,0 +1,398 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software
|
||||
# Foundation; All Rights Reserved
|
||||
|
||||
"""A HTTPSConnection/Handler with additional proxy and cert validation features.
|
||||
|
||||
In particular, monkey patches in Python r74203 to provide support for CONNECT
|
||||
proxies and adds SSL cert validation if the ssl module is present.
|
||||
"""
|
||||
|
||||
__author__ = "{frew,nick.johnson}@google.com (Fred Wulff and Nick Johnson)"
|
||||
|
||||
import base64
|
||||
import httplib
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
import urllib2
|
||||
|
||||
from urllib import splittype
|
||||
from urllib import splituser
|
||||
from urllib import splitpasswd
|
||||
|
||||
class InvalidCertificateException(httplib.HTTPException):
|
||||
"""Raised when a certificate is provided with an invalid hostname."""
|
||||
|
||||
def __init__(self, host, cert, reason):
|
||||
"""Constructor.
|
||||
|
||||
Args:
|
||||
host: The hostname the connection was made to.
|
||||
cert: The SSL certificate (as a dictionary) the host returned.
|
||||
"""
|
||||
httplib.HTTPException.__init__(self)
|
||||
self.host = host
|
||||
self.cert = cert
|
||||
self.reason = reason
|
||||
|
||||
def __str__(self):
|
||||
return ('Host %s returned an invalid certificate (%s): %s\n'
|
||||
'To learn more, see '
|
||||
'http://code.google.com/appengine/kb/general.html#rpcssl' %
|
||||
(self.host, self.reason, self.cert))
|
||||
|
||||
def can_validate_certs():
|
||||
"""Return True if we have the SSL package and can validate certificates."""
|
||||
try:
|
||||
import ssl
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
def _create_fancy_connection(tunnel_host=None, key_file=None,
|
||||
cert_file=None, ca_certs=None):
|
||||
# This abomination brought to you by the fact that
|
||||
# the HTTPHandler creates the connection instance in the middle
|
||||
# of do_open so we need to add the tunnel host to the class.
|
||||
|
||||
class PresetProxyHTTPSConnection(httplib.HTTPSConnection):
|
||||
"""An HTTPS connection that uses a proxy defined by the enclosing scope."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
|
||||
|
||||
self._tunnel_host = tunnel_host
|
||||
if tunnel_host:
|
||||
logging.debug("Creating preset proxy https conn: %s", tunnel_host)
|
||||
|
||||
self.key_file = key_file
|
||||
self.cert_file = cert_file
|
||||
self.ca_certs = ca_certs
|
||||
try:
|
||||
import ssl
|
||||
if self.ca_certs:
|
||||
self.cert_reqs = ssl.CERT_REQUIRED
|
||||
else:
|
||||
self.cert_reqs = ssl.CERT_NONE
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
def _tunnel(self):
|
||||
self._set_hostport(self._tunnel_host, None)
|
||||
logging.info("Connecting through tunnel to: %s:%d",
|
||||
self.host, self.port)
|
||||
self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port))
|
||||
response = self.response_class(self.sock, strict=self.strict,
|
||||
method=self._method)
|
||||
(_, code, message) = response._read_status()
|
||||
|
||||
if code != 200:
|
||||
self.close()
|
||||
raise socket.error, "Tunnel connection failed: %d %s" % (
|
||||
code, message.strip())
|
||||
|
||||
while True:
|
||||
line = response.fp.readline()
|
||||
if line == "\r\n":
|
||||
break
|
||||
|
||||
def _get_valid_hosts_for_cert(self, cert):
|
||||
"""Returns a list of valid host globs for an SSL certificate.
|
||||
|
||||
Args:
|
||||
cert: A dictionary representing an SSL certificate.
|
||||
Returns:
|
||||
list: A list of valid host globs.
|
||||
"""
|
||||
if 'subjectAltName' in cert:
|
||||
return [x[1] for x in cert['subjectAltName'] if x[0].lower() == 'dns']
|
||||
else:
|
||||
# Return a list of commonName fields
|
||||
return [x[0][1] for x in cert['subject']
|
||||
if x[0][0].lower() == 'commonname']
|
||||
|
||||
def _validate_certificate_hostname(self, cert, hostname):
|
||||
"""Validates that a given hostname is valid for an SSL certificate.
|
||||
|
||||
Args:
|
||||
cert: A dictionary representing an SSL certificate.
|
||||
hostname: The hostname to test.
|
||||
Returns:
|
||||
bool: Whether or not the hostname is valid for this certificate.
|
||||
"""
|
||||
hosts = self._get_valid_hosts_for_cert(cert)
|
||||
for host in hosts:
|
||||
# Convert the glob-style hostname expression (eg, '*.google.com') into a
|
||||
# valid regular expression.
|
||||
host_re = host.replace('.', '\.').replace('*', '[^.]*')
|
||||
if re.search('^%s$' % (host_re,), hostname, re.I):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def connect(self):
|
||||
# TODO(frew): When we drop support for <2.6 (in the far distant future),
|
||||
# change this to socket.create_connection.
|
||||
self.sock = _create_connection((self.host, self.port))
|
||||
|
||||
if self._tunnel_host:
|
||||
self._tunnel()
|
||||
|
||||
# ssl and FakeSocket got deprecated. Try for the new hotness of wrap_ssl,
|
||||
# with fallback.
|
||||
try:
|
||||
import ssl
|
||||
self.sock = ssl.wrap_socket(self.sock,
|
||||
keyfile=self.key_file,
|
||||
certfile=self.cert_file,
|
||||
ca_certs=self.ca_certs,
|
||||
cert_reqs=self.cert_reqs)
|
||||
|
||||
if self.cert_reqs & ssl.CERT_REQUIRED:
|
||||
cert = self.sock.getpeercert()
|
||||
hostname = self.host.split(':', 0)[0]
|
||||
if not self._validate_certificate_hostname(cert, hostname):
|
||||
raise InvalidCertificateException(hostname, cert,
|
||||
'hostname mismatch')
|
||||
except ImportError:
|
||||
ssl = socket.ssl(self.sock,
|
||||
keyfile=self.key_file,
|
||||
certfile=self.cert_file)
|
||||
self.sock = httplib.FakeSocket(self.sock, ssl)
|
||||
|
||||
return PresetProxyHTTPSConnection
|
||||
|
||||
|
||||
# Here to end of _create_connection copied wholesale from Python 2.6"s socket.py
|
||||
_GLOBAL_DEFAULT_TIMEOUT = object()
|
||||
|
||||
|
||||
def _create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT):
|
||||
"""Connect to *address* and return the socket object.
|
||||
|
||||
Convenience function. Connect to *address* (a 2-tuple ``(host,
|
||||
port)``) and return the socket object. Passing the optional
|
||||
*timeout* parameter will set the timeout on the socket instance
|
||||
before attempting to connect. If no *timeout* is supplied, the
|
||||
global default timeout setting returned by :func:`getdefaulttimeout`
|
||||
is used.
|
||||
"""
|
||||
|
||||
msg = "getaddrinfo returns an empty list"
|
||||
host, port = address
|
||||
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
sock = None
|
||||
try:
|
||||
sock = socket.socket(af, socktype, proto)
|
||||
if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
|
||||
sock.settimeout(timeout)
|
||||
sock.connect(sa)
|
||||
return sock
|
||||
|
||||
except socket.error, msg:
|
||||
if sock is not None:
|
||||
sock.close()
|
||||
|
||||
raise socket.error, msg
|
||||
|
||||
|
||||
class FancyRequest(urllib2.Request):
|
||||
"""A request that allows the use of a CONNECT proxy."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
urllib2.Request.__init__(self, *args, **kwargs)
|
||||
self._tunnel_host = None
|
||||
self._key_file = None
|
||||
self._cert_file = None
|
||||
self._ca_certs = None
|
||||
|
||||
def set_proxy(self, host, type):
|
||||
saved_type = None
|
||||
|
||||
if self.get_type() == "https" and not self._tunnel_host:
|
||||
self._tunnel_host = self.get_host()
|
||||
saved_type = self.get_type()
|
||||
urllib2.Request.set_proxy(self, host, type)
|
||||
|
||||
if saved_type:
|
||||
# Don't set self.type, we want to preserve the
|
||||
# type for tunneling.
|
||||
self.type = saved_type
|
||||
|
||||
def set_ssl_info(self, key_file=None, cert_file=None, ca_certs=None):
|
||||
self._key_file = key_file
|
||||
self._cert_file = cert_file
|
||||
self._ca_certs = ca_certs
|
||||
|
||||
|
||||
class FancyProxyHandler(urllib2.ProxyHandler):
|
||||
"""A ProxyHandler that works with CONNECT-enabled proxies."""
|
||||
|
||||
# Taken verbatim from /usr/lib/python2.5/urllib2.py
|
||||
def _parse_proxy(self, proxy):
|
||||
"""Return (scheme, user, password, host/port) given a URL or an authority.
|
||||
|
||||
If a URL is supplied, it must have an authority (host:port) component.
|
||||
According to RFC 3986, having an authority component means the URL must
|
||||
have two slashes after the scheme:
|
||||
|
||||
>>> _parse_proxy('file:/ftp.example.com/')
|
||||
Traceback (most recent call last):
|
||||
ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
|
||||
|
||||
The first three items of the returned tuple may be None.
|
||||
|
||||
Examples of authority parsing:
|
||||
|
||||
>>> _parse_proxy('proxy.example.com')
|
||||
(None, None, None, 'proxy.example.com')
|
||||
>>> _parse_proxy('proxy.example.com:3128')
|
||||
(None, None, None, 'proxy.example.com:3128')
|
||||
|
||||
The authority component may optionally include userinfo (assumed to be
|
||||
username:password):
|
||||
|
||||
>>> _parse_proxy('joe:password@proxy.example.com')
|
||||
(None, 'joe', 'password', 'proxy.example.com')
|
||||
>>> _parse_proxy('joe:password@proxy.example.com:3128')
|
||||
(None, 'joe', 'password', 'proxy.example.com:3128')
|
||||
|
||||
Same examples, but with URLs instead:
|
||||
|
||||
>>> _parse_proxy('http://proxy.example.com/')
|
||||
('http', None, None, 'proxy.example.com')
|
||||
>>> _parse_proxy('http://proxy.example.com:3128/')
|
||||
('http', None, None, 'proxy.example.com:3128')
|
||||
>>> _parse_proxy('http://joe:password@proxy.example.com/')
|
||||
('http', 'joe', 'password', 'proxy.example.com')
|
||||
>>> _parse_proxy('http://joe:password@proxy.example.com:3128')
|
||||
('http', 'joe', 'password', 'proxy.example.com:3128')
|
||||
|
||||
Everything after the authority is ignored:
|
||||
|
||||
>>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')
|
||||
('ftp', 'joe', 'password', 'proxy.example.com')
|
||||
|
||||
Test for no trailing '/' case:
|
||||
|
||||
>>> _parse_proxy('http://joe:password@proxy.example.com')
|
||||
('http', 'joe', 'password', 'proxy.example.com')
|
||||
|
||||
"""
|
||||
scheme, r_scheme = splittype(proxy)
|
||||
if not r_scheme.startswith("/"):
|
||||
# authority
|
||||
scheme = None
|
||||
authority = proxy
|
||||
else:
|
||||
# URL
|
||||
if not r_scheme.startswith("//"):
|
||||
raise ValueError("proxy URL with no authority: %r" % proxy)
|
||||
# We have an authority, so for RFC 3986-compliant URLs (by ss 3.
|
||||
# and 3.3.), path is empty or starts with '/'
|
||||
end = r_scheme.find("/", 2)
|
||||
if end == -1:
|
||||
end = None
|
||||
authority = r_scheme[2:end]
|
||||
userinfo, hostport = splituser(authority)
|
||||
if userinfo is not None:
|
||||
user, password = splitpasswd(userinfo)
|
||||
else:
|
||||
user = password = None
|
||||
return scheme, user, password, hostport
|
||||
|
||||
def proxy_open(self, req, proxy, type):
|
||||
# This block is copied wholesale from Python2.6 urllib2.
|
||||
# It is idempotent, so the superclass method call executes as normal
|
||||
# if invoked.
|
||||
orig_type = req.get_type()
|
||||
proxy_type, user, password, hostport = self._parse_proxy(proxy)
|
||||
if proxy_type is None:
|
||||
proxy_type = orig_type
|
||||
if user and password:
|
||||
user_pass = "%s:%s" % (urllib2.unquote(user), urllib2.unquote(password))
|
||||
creds = base64.b64encode(user_pass).strip()
|
||||
# Later calls overwrite earlier calls for the same header
|
||||
req.add_header("Proxy-authorization", "Basic " + creds)
|
||||
hostport = urllib2.unquote(hostport)
|
||||
req.set_proxy(hostport, proxy_type)
|
||||
# This condition is the change
|
||||
if orig_type == "https":
|
||||
return None
|
||||
|
||||
return urllib2.ProxyHandler.proxy_open(self, req, proxy, type)
|
||||
|
||||
|
||||
class FancyHTTPSHandler(urllib2.HTTPSHandler):
|
||||
"""An HTTPSHandler that works with CONNECT-enabled proxies."""
|
||||
|
||||
def do_open(self, http_class, req):
|
||||
# Intentionally very specific so as to opt for false negatives
|
||||
# rather than false positives.
|
||||
try:
|
||||
return urllib2.HTTPSHandler.do_open(
|
||||
self,
|
||||
_create_fancy_connection(req._tunnel_host,
|
||||
req._key_file,
|
||||
req._cert_file,
|
||||
req._ca_certs),
|
||||
req)
|
||||
except urllib2.URLError, url_error:
|
||||
try:
|
||||
import ssl
|
||||
if (type(url_error.reason) == ssl.SSLError and
|
||||
url_error.reason.args[0] == 1):
|
||||
# Display the reason to the user. Need to use args for python2.5
|
||||
# compat.
|
||||
raise InvalidCertificateException(req.host, '',
|
||||
url_error.reason.args[1])
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
raise url_error
|
||||
|
||||
|
||||
# We have to implement this so that we persist the tunneling behavior
|
||||
# through redirects.
|
||||
class FancyRedirectHandler(urllib2.HTTPRedirectHandler):
|
||||
"""A redirect handler that persists CONNECT-enabled proxy information."""
|
||||
|
||||
def redirect_request(self, req, *args, **kwargs):
|
||||
new_req = urllib2.HTTPRedirectHandler.redirect_request(
|
||||
self, req, *args, **kwargs)
|
||||
# Same thing as in our set_proxy implementation, but in this case
|
||||
# we"ve only got a Request to work with, so it was this or copy
|
||||
# everything over piecemeal.
|
||||
#
|
||||
# Note that we do not persist tunneling behavior from an http request
|
||||
# to an https request, because an http request does not set _tunnel_host.
|
||||
#
|
||||
# Also note that in Python < 2.6, you will get an error in
|
||||
# FancyHTTPSHandler.do_open() on an https urllib2.Request that uses an http
|
||||
# proxy, since the proxy type will be set to http instead of https.
|
||||
# (FancyRequest, and urllib2.Request in Python >= 2.6 set the proxy type to
|
||||
# https.) Such an urllib2.Request could result from this redirect
|
||||
# if you are redirecting from an http request (since an an http request
|
||||
# does not have _tunnel_host set, and thus you will not set the proxy
|
||||
# in the code below), and if you have defined a proxy for https in, say,
|
||||
# FancyProxyHandler, and that proxy has type http.
|
||||
if hasattr(req, "_tunnel_host") and isinstance(new_req, urllib2.Request):
|
||||
if new_req.get_type() == "https":
|
||||
if req._tunnel_host:
|
||||
# req is proxied, so copy the proxy info.
|
||||
new_req._tunnel_host = new_req.get_host()
|
||||
new_req.set_proxy(req.host, "https")
|
||||
else:
|
||||
# req is not proxied, so just make sure _tunnel_host is defined.
|
||||
new_req._tunnel_host = None
|
||||
new_req.type = "https"
|
||||
if hasattr(req, "_key_file") and isinstance(new_req, urllib2.Request):
|
||||
# Copy the auxiliary data in case this or any further redirect is https
|
||||
new_req._key_file = req._key_file
|
||||
new_req._cert_file = req._cert_file
|
||||
new_req._ca_certs = req._ca_certs
|
||||
|
||||
return new_req
|
||||
@ -0,0 +1 @@
|
||||
010822c61d38d70ac23600bc955fccf5
|
||||
@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -0,0 +1,295 @@
|
||||
GOOGLE APP ENGINE SDK
|
||||
**************************************************************************
|
||||
Copyright 2008 Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
|
||||
fancy_urllib
|
||||
**************************************************************************
|
||||
|
||||
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software
|
||||
# Foundation; All Rights Reserved
|
||||
|
||||
|
||||
A. HISTORY OF THE SOFTWARE
|
||||
==========================
|
||||
|
||||
Python was created in the early 1990s by Guido van Rossum at Stichting
|
||||
Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
|
||||
as a successor of a language called ABC. Guido remains Python's
|
||||
principal author, although it includes many contributions from others.
|
||||
|
||||
In 1995, Guido continued his work on Python at the Corporation for
|
||||
National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
|
||||
in Reston, Virginia where he released several versions of the
|
||||
software.
|
||||
|
||||
In May 2000, Guido and the Python core development team moved to
|
||||
BeOpen.com to form the BeOpen PythonLabs team. In October of the same
|
||||
year, the PythonLabs team moved to Digital Creations (now Zope
|
||||
Corporation, see http://www.zope.com). In 2001, the Python Software
|
||||
Foundation (PSF, see http://www.python.org/psf/) was formed, a
|
||||
non-profit organization created specifically to own Python-related
|
||||
Intellectual Property. Zope Corporation is a sponsoring member of
|
||||
the PSF.
|
||||
|
||||
All Python releases are Open Source (see http://www.opensource.org for
|
||||
the Open Source Definition). Historically, most, but not all, Python
|
||||
releases have also been GPL-compatible; the table below summarizes
|
||||
the various releases.
|
||||
|
||||
Release Derived Year Owner GPL-
|
||||
from compatible? (1)
|
||||
|
||||
0.9.0 thru 1.2 1991-1995 CWI yes
|
||||
1.3 thru 1.5.2 1.2 1995-1999 CNRI yes
|
||||
1.6 1.5.2 2000 CNRI no
|
||||
2.0 1.6 2000 BeOpen.com no
|
||||
1.6.1 1.6 2001 CNRI yes (2)
|
||||
2.1 2.0+1.6.1 2001 PSF no
|
||||
2.0.1 2.0+1.6.1 2001 PSF yes
|
||||
2.1.1 2.1+2.0.1 2001 PSF yes
|
||||
2.2 2.1.1 2001 PSF yes
|
||||
2.1.2 2.1.1 2002 PSF yes
|
||||
2.1.3 2.1.2 2002 PSF yes
|
||||
2.2.1 2.2 2002 PSF yes
|
||||
2.2.2 2.2.1 2002 PSF yes
|
||||
2.2.3 2.2.2 2003 PSF yes
|
||||
2.3 2.2.2 2002-2003 PSF yes
|
||||
2.3.1 2.3 2002-2003 PSF yes
|
||||
2.3.2 2.3.1 2002-2003 PSF yes
|
||||
2.3.3 2.3.2 2002-2003 PSF yes
|
||||
2.3.4 2.3.3 2004 PSF yes
|
||||
2.3.5 2.3.4 2005 PSF yes
|
||||
2.4 2.3 2004 PSF yes
|
||||
2.4.1 2.4 2005 PSF yes
|
||||
2.4.2 2.4.1 2005 PSF yes
|
||||
2.4.3 2.4.2 2006 PSF yes
|
||||
2.5 2.4 2006 PSF yes
|
||||
2.5.1 2.5 2007 PSF yes
|
||||
|
||||
Footnotes:
|
||||
|
||||
(1) GPL-compatible doesn't mean that we're distributing Python under
|
||||
the GPL. All Python licenses, unlike the GPL, let you distribute
|
||||
a modified version without making your changes open source. The
|
||||
GPL-compatible licenses make it possible to combine Python with
|
||||
other software that is released under the GPL; the others don't.
|
||||
|
||||
(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
|
||||
because its license has a choice of law clause. According to
|
||||
CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
|
||||
is "not incompatible" with the GPL.
|
||||
|
||||
Thanks to the many outside volunteers who have worked under Guido's
|
||||
direction to make these releases possible.
|
||||
|
||||
|
||||
B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
|
||||
===============================================================
|
||||
|
||||
PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
||||
--------------------------------------------
|
||||
|
||||
1. This LICENSE AGREEMENT is between the Python Software Foundation
|
||||
("PSF"), and the Individual or Organization ("Licensee") accessing and
|
||||
otherwise using this software ("Python") in source or binary form and
|
||||
its associated documentation.
|
||||
|
||||
2. Subject to the terms and conditions of this License Agreement, PSF
|
||||
hereby grants Licensee a nonexclusive, royalty-free, world-wide
|
||||
license to reproduce, analyze, test, perform and/or display publicly,
|
||||
prepare derivative works, distribute, and otherwise use Python
|
||||
alone or in any derivative version, provided, however, that PSF's
|
||||
License Agreement and PSF's notice of copyright, i.e., "Copyright (c)
|
||||
2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software Foundation;
|
||||
All Rights Reserved" are retained in Python alone or in any derivative
|
||||
version prepared by Licensee.
|
||||
|
||||
3. In the event Licensee prepares a derivative work that is based on
|
||||
or incorporates Python or any part thereof, and wants to make
|
||||
the derivative work available to others as provided herein, then
|
||||
Licensee hereby agrees to include in any such work a brief summary of
|
||||
the changes made to Python.
|
||||
|
||||
4. PSF is making Python available to Licensee on an "AS IS"
|
||||
basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
|
||||
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
|
||||
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
|
||||
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
|
||||
INFRINGE ANY THIRD PARTY RIGHTS.
|
||||
|
||||
5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
|
||||
FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
|
||||
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
|
||||
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
|
||||
|
||||
6. This License Agreement will automatically terminate upon a material
|
||||
breach of its terms and conditions.
|
||||
|
||||
7. Nothing in this License Agreement shall be deemed to create any
|
||||
relationship of agency, partnership, or joint venture between PSF and
|
||||
Licensee. This License Agreement does not grant permission to use PSF
|
||||
trademarks or trade name in a trademark sense to endorse or promote
|
||||
products or services of Licensee, or any third party.
|
||||
|
||||
8. By copying, installing or otherwise using Python, Licensee
|
||||
agrees to be bound by the terms and conditions of this License
|
||||
Agreement.
|
||||
|
||||
|
||||
BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
|
||||
-------------------------------------------
|
||||
|
||||
BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
|
||||
|
||||
1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
|
||||
office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
|
||||
Individual or Organization ("Licensee") accessing and otherwise using
|
||||
this software in source or binary form and its associated
|
||||
documentation ("the Software").
|
||||
|
||||
2. Subject to the terms and conditions of this BeOpen Python License
|
||||
Agreement, BeOpen hereby grants Licensee a non-exclusive,
|
||||
royalty-free, world-wide license to reproduce, analyze, test, perform
|
||||
and/or display publicly, prepare derivative works, distribute, and
|
||||
otherwise use the Software alone or in any derivative version,
|
||||
provided, however, that the BeOpen Python License is retained in the
|
||||
Software, alone or in any derivative version prepared by Licensee.
|
||||
|
||||
3. BeOpen is making the Software available to Licensee on an "AS IS"
|
||||
basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
|
||||
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
|
||||
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
|
||||
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
|
||||
INFRINGE ANY THIRD PARTY RIGHTS.
|
||||
|
||||
4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
|
||||
SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
|
||||
AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
|
||||
DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
|
||||
|
||||
5. This License Agreement will automatically terminate upon a material
|
||||
breach of its terms and conditions.
|
||||
|
||||
6. This License Agreement shall be governed by and interpreted in all
|
||||
respects by the law of the State of California, excluding conflict of
|
||||
law provisions. Nothing in this License Agreement shall be deemed to
|
||||
create any relationship of agency, partnership, or joint venture
|
||||
between BeOpen and Licensee. This License Agreement does not grant
|
||||
permission to use BeOpen trademarks or trade names in a trademark
|
||||
sense to endorse or promote products or services of Licensee, or any
|
||||
third party. As an exception, the "BeOpen Python" logos available at
|
||||
http://www.pythonlabs.com/logos.html may be used according to the
|
||||
permissions granted on that web page.
|
||||
|
||||
7. By copying, installing or otherwise using the software, Licensee
|
||||
agrees to be bound by the terms and conditions of this License
|
||||
Agreement.
|
||||
|
||||
|
||||
CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
|
||||
---------------------------------------
|
||||
|
||||
1. This LICENSE AGREEMENT is between the Corporation for National
|
||||
Research Initiatives, having an office at 1895 Preston White Drive,
|
||||
Reston, VA 20191 ("CNRI"), and the Individual or Organization
|
||||
("Licensee") accessing and otherwise using Python 1.6.1 software in
|
||||
source or binary form and its associated documentation.
|
||||
|
||||
2. Subject to the terms and conditions of this License Agreement, CNRI
|
||||
hereby grants Licensee a nonexclusive, royalty-free, world-wide
|
||||
license to reproduce, analyze, test, perform and/or display publicly,
|
||||
prepare derivative works, distribute, and otherwise use Python 1.6.1
|
||||
alone or in any derivative version, provided, however, that CNRI's
|
||||
License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
|
||||
1995-2001 Corporation for National Research Initiatives; All Rights
|
||||
Reserved" are retained in Python 1.6.1 alone or in any derivative
|
||||
version prepared by Licensee. Alternately, in lieu of CNRI's License
|
||||
Agreement, Licensee may substitute the following text (omitting the
|
||||
quotes): "Python 1.6.1 is made available subject to the terms and
|
||||
conditions in CNRI's License Agreement. This Agreement together with
|
||||
Python 1.6.1 may be located on the Internet using the following
|
||||
unique, persistent identifier (known as a handle): 1895.22/1013. This
|
||||
Agreement may also be obtained from a proxy server on the Internet
|
||||
using the following URL: http://hdl.handle.net/1895.22/1013".
|
||||
|
||||
3. In the event Licensee prepares a derivative work that is based on
|
||||
or incorporates Python 1.6.1 or any part thereof, and wants to make
|
||||
the derivative work available to others as provided herein, then
|
||||
Licensee hereby agrees to include in any such work a brief summary of
|
||||
the changes made to Python 1.6.1.
|
||||
|
||||
4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
|
||||
basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
|
||||
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
|
||||
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
|
||||
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
|
||||
INFRINGE ANY THIRD PARTY RIGHTS.
|
||||
|
||||
5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
|
||||
1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
|
||||
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
|
||||
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
|
||||
|
||||
6. This License Agreement will automatically terminate upon a material
|
||||
breach of its terms and conditions.
|
||||
|
||||
7. This License Agreement shall be governed by the federal
|
||||
intellectual property law of the United States, including without
|
||||
limitation the federal copyright law, and, to the extent such
|
||||
U.S. federal law does not apply, by the law of the Commonwealth of
|
||||
Virginia, excluding Virginia's conflict of law provisions.
|
||||
Notwithstanding the foregoing, with regard to derivative works based
|
||||
on Python 1.6.1 that incorporate non-separable material that was
|
||||
previously distributed under the GNU General Public License (GPL), the
|
||||
law of the Commonwealth of Virginia shall govern this License
|
||||
Agreement only as to issues arising under or with respect to
|
||||
Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this
|
||||
License Agreement shall be deemed to create any relationship of
|
||||
agency, partnership, or joint venture between CNRI and Licensee. This
|
||||
License Agreement does not grant permission to use CNRI trademarks or
|
||||
trade name in a trademark sense to endorse or promote products or
|
||||
services of Licensee, or any third party.
|
||||
|
||||
8. By clicking on the "ACCEPT" button where indicated, or by copying,
|
||||
installing or otherwise using Python 1.6.1, Licensee agrees to be
|
||||
bound by the terms and conditions of this License Agreement.
|
||||
|
||||
ACCEPT
|
||||
|
||||
|
||||
CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
|
||||
--------------------------------------------------
|
||||
|
||||
Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
|
||||
The Netherlands. All rights reserved.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software and its
|
||||
documentation for any purpose and without fee is hereby granted,
|
||||
provided that the above copyright notice appear in all copies and that
|
||||
both that copyright notice and this permission notice appear in
|
||||
supporting documentation, and that the name of Stichting Mathematisch
|
||||
Centrum or CWI not be used in advertising or publicity pertaining to
|
||||
distribution of the software without specific, written prior
|
||||
permission.
|
||||
|
||||
STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
|
||||
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
|
||||
FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
|
||||
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
@ -0,0 +1,5 @@
|
||||
include gsutil COPYING VERSION LICENSE.third_party README setup.py pkg_util.py
|
||||
recursive-include gslib *
|
||||
recursive-include oauth2_plugin *
|
||||
recursive-include third_party *
|
||||
recursive-include boto *
|
||||
@ -0,0 +1,38 @@
|
||||
This directory contains the Python command line tool gsutil, which Google
|
||||
has released as open source, to demonstrate the Google Storage API and to
|
||||
provide a tool for manipulating data in the system.
|
||||
|
||||
Prerequisites:
|
||||
|
||||
Gsutil requires Python 2.6 or later.
|
||||
|
||||
To install gsutil take the following steps:
|
||||
|
||||
1. Pick a place where you want to install the software. You can
|
||||
install the code wherever you prefer; for brevity the instructions below
|
||||
assume you want to install in $HOME/gsutil.
|
||||
|
||||
2. To install gsutil on Linux/Unix or MacOS, open a shell window, change
|
||||
directories to where you downloaded the gsutil.tar.gz file, and do this:
|
||||
% tar xfz gsutil.tar.gz -C $HOME
|
||||
|
||||
Then add the following line to your $HOME/.bashrc shell initialization
|
||||
file:
|
||||
export PATH=${PATH}:$HOME/gsutil
|
||||
|
||||
The next time you start a shell you should be able to run gsutil from
|
||||
the command line.
|
||||
|
||||
3. To install gsutil on Windows, install cygwin (http://www.cygwin.com/),
|
||||
with at least version 2.6.5 of Python. Once you have that, start a shell
|
||||
and follow the Linux instructions above for unpacking and installing gsutil.
|
||||
|
||||
4. The first time you try to run gsutil, it will detect that you have no
|
||||
configuration file containing your credentials, interactively prompt you,
|
||||
and create the file.
|
||||
|
||||
After this you can use the tool. Running gsutil with with no arguments
|
||||
will print a help summary.
|
||||
|
||||
For more information on installing and using gsutil, see
|
||||
<http://code.google.com/apis/storage/docs/gsutil.html>.
|
||||
@ -0,0 +1,19 @@
|
||||
Name: gsutil
|
||||
URL: https://github.com/GoogleCloudPlatform/gsutil
|
||||
Version: 3.25
|
||||
License: Apache 2.0
|
||||
|
||||
Description:
|
||||
Set of tools to allow querying, uploading, and downloading objects from
|
||||
Google Storage.
|
||||
|
||||
Modifications:
|
||||
* Removed gsutil/gslib/commands/test.py
|
||||
* Removed gsutil/pkg_gen.sh
|
||||
* Removed gsutil/gslib/tests/
|
||||
* Moved gsutil/boto as a depot_tools third_party lib
|
||||
* Moved gsutil/third_party into our own third_party directory
|
||||
* Append sys.path in gsutil/gsutil to find the moved third_party modules
|
||||
* Updated checksum ce71ac982f1148315e7fa65cff2f83e8 -> bf29190007bc7557c33806367ee3ce9e
|
||||
|
||||
Full license is in the COPYING file.
|
||||
@ -0,0 +1,49 @@
|
||||
Package Generation Notes for gsutil
|
||||
|
||||
Gsutil can be distributed in one of three ways:
|
||||
|
||||
1. legacy mode - User unpacks archive file into a private directory tree
|
||||
and maintains his/her own private copy of gsutil, boto, etc. This is the
|
||||
only supported installation mode for Windows users.
|
||||
|
||||
2. enterprise mode - User unpacks the gsutil archive file and runs
|
||||
'python setup.py install' (as root), which installs everything into
|
||||
a shared directory tree (/usr/share/gsutil) with a symlink from
|
||||
/usr/bin/gsutil to /usr/share/gsutil/gsutil to provide easy access to
|
||||
the shared gsutil command. In enterprise mode, the software gets installed
|
||||
in one shared location, which makes it easier to install, update and
|
||||
manage gsutil for a community of users.
|
||||
|
||||
NOTE: Enterprise mode (installing gsutil via setup.py) is no longer
|
||||
officially supported - unpacking the zip file into a directory is the
|
||||
preferred method for installing gsutil for both shared and private
|
||||
configurations.
|
||||
|
||||
3. rpm mode - User installs the gsutil rpm package file on a Red Hat
|
||||
Linux system using the rpm command. The resulting installation image
|
||||
looks precisely the same as the results of installing with enterprise
|
||||
mode, i.e. a shared directory tree (/usr/share/gsutil) with a symlink
|
||||
from /usr/bin/gsutil. rpm mode is intended for enterprises that want
|
||||
a stable release that does not necessarily contain the latest changes.
|
||||
|
||||
All three modes derive their inventory from a common text file called
|
||||
MANIFEST.in. If you want to add one or more new files or directories,
|
||||
you only need to edit that one file and all three installation modes
|
||||
will automatically inherit the change(s).
|
||||
|
||||
GENERATING PACKAGE FILES
|
||||
|
||||
First update the VERSION file and the gsutil.spec files to reflect the
|
||||
new version number.
|
||||
|
||||
Legacy mode and enterprise mode are both embodied in the same gsutil
|
||||
archive file, the only difference being that the latter entails running
|
||||
one additional command after unpacking the gsutil archive file. So the
|
||||
same archive file we've always distributed for gsutil will be used for
|
||||
both legacy and enterprise installation modes.
|
||||
|
||||
For rpm mode, there's a new tool call pkg_gen.sh, which when run with no
|
||||
arguments creates an rpm file at this location:
|
||||
|
||||
$HOME/rpmbuild/RPMS/noarch/gsutil-2.0-1.noarch.rpm
|
||||
|
||||
@ -0,0 +1,825 @@
|
||||
Release 3.25 (release-date: 2013-02-21)
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed two version-specific URI bugs:
|
||||
|
||||
1. gsutil cp -r gs://bucket1 gs://bucket2 would create objects in bucket2
|
||||
with names corresponding to version-specific URIs in bucket1 (e.g.,
|
||||
gs://bucket2/obj#1361417568482000, where the "#1361417568482000" part was
|
||||
part of the object name, not the object's generation).
|
||||
|
||||
This problem similarly caused gsutil cp -r gs://bucket1 ./dir to create
|
||||
files names corresponding to version-specific URIs in bucket1.
|
||||
|
||||
2. gsutil rm -a gs://bucket/obj would attempt to delete the same object
|
||||
twice, getting a NoSuchKey error on the second attempt.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.24 (release-date: 2013-02-19)
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug that caused attempt to dupe-encode a unicode filename.
|
||||
|
||||
|
||||
Other Changes
|
||||
---------
|
||||
|
||||
- Refactored retry logic from setmeta and chacl to use @Retry decorator.
|
||||
|
||||
- Moved @Retry decorator to third_party.
|
||||
|
||||
- Fixed flaky tests.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.23 (release-date: 2013-02-16)
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Make version-specific URI parsing more robust. This fixes a bug where
|
||||
listing buckets in certain cases would result in the error
|
||||
'BucketStorageUri' object has no attribute 'version_specific_uri'
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.22 (release-date: 2013-02-15)
|
||||
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Implemented new chacl command, which makes it easy to add and remove bucket
|
||||
and object ACL grants without having to edit XML (like the older setacl
|
||||
command).
|
||||
|
||||
- Implemented new "daisy-chain" copying mode, which allows cross-provider
|
||||
copies to run without buffering to local disk, and to use resumable uploads.
|
||||
This copying mode also allows copying between locations and between storage
|
||||
classes, using the new gsutil cp -D option. (Daisy-chain copying is the
|
||||
default when copying between providers, but must be explicitly requested for
|
||||
the other cases to keep costs and performance expectations clear.)
|
||||
|
||||
- Implemented new perfdiag command to run a diagnostic test against
|
||||
a bucket, collect system information, and report results. Useful
|
||||
when working with Google Cloud Storage team to resolve questions
|
||||
about performance.
|
||||
|
||||
- Added SIGQUIT (^\) handler, to allow breakpointing a running gsutil.
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug where gsutil setwebcfg signature didn't match with
|
||||
HMAC authentication.
|
||||
|
||||
- Fixed ASCII codec decode error when constructing tracker filename
|
||||
from non-7bit ASCII intput filename.
|
||||
|
||||
- Changed boto auth plugin framework to allow multiple plugins
|
||||
supporting requested capability, which fixes gsutil exception
|
||||
that used to happen where a GCE user had a service account
|
||||
configured and then ran gsutil config.
|
||||
|
||||
- Changed Command.Apply method to be resilient to name expansion
|
||||
exceptions. Before this change, if an exception was raised
|
||||
during iteration of NameExpansionResult, the parent process
|
||||
would immediately stop execution, causing the
|
||||
_EOF_NAME_EXPANSION_RESULT to never be sent to child processes.
|
||||
This resulted in the process hanging forever.
|
||||
|
||||
- Fixed various bugs for gsutil running on Windows:
|
||||
- Fixed various places from a hard-coded '/' to os.sep.
|
||||
- Fixed a bug in the cp command where it was using the destination
|
||||
URI's .delim property instead of the source URI.
|
||||
- Fixed a bug in the cp command's _SrcDstSame function by
|
||||
simplifying it to use os.path.normpath.
|
||||
- Fixed windows bug in tests/util.py _NormalizeURI function.
|
||||
- Fixed ZeroDivisionError sometimes happening during unit tests
|
||||
on Windows.
|
||||
|
||||
- Fixed gsutil rm bug that caused exit status 1 when encountered
|
||||
non-existent URI.
|
||||
|
||||
- Fixed support for gsutil cp file -.
|
||||
|
||||
- Added preconditions and retry logic to setmeta command, to
|
||||
enforce concurrency control.
|
||||
|
||||
- Fixed bug in copying subdirs to subdirs.
|
||||
|
||||
- Fixed cases where boto debug_level caused too much or too little
|
||||
logging:
|
||||
- resumable and one-shot uploads weren't showing response headers
|
||||
when connection.debug > 0.
|
||||
- payload was showing up in debug output when connection.debug
|
||||
< 4 for streaming uploads.
|
||||
|
||||
- Removed XML parsing from setacl. The previous implementation
|
||||
relied on loose XML handling, which could truncate what it sends
|
||||
to the service, allowing invalid XML to be specified by the
|
||||
user. Instead now the ACL XML is passed verbatim and we rely
|
||||
on server-side schema enforcement.
|
||||
|
||||
- Added user-agent header to resumable uploads.
|
||||
|
||||
- Fixed reporting bits/s when it was really bytes/s.
|
||||
|
||||
- Changed so we now pass headers with API version & project ID
|
||||
to create_bucket().
|
||||
|
||||
- Made "gsutil rm -r gs://bucket/folder" remove xyz_$folder$ object
|
||||
(which is created by various GUI tools).
|
||||
|
||||
- Fixed bug where gsutil binary was shipped with protection 750
|
||||
instead of 755.
|
||||
|
||||
|
||||
Other Changes
|
||||
---------
|
||||
|
||||
- Reworked versioned object handling:
|
||||
- Removed need for commands to specify -v option to parse
|
||||
versions. Versioned URIs are now uniformly handled by all
|
||||
commands.
|
||||
- Refactored StorageUri parsing that had been split across
|
||||
storage_uri and conveience; made versioned URIs render with
|
||||
version string so StorageUri is round-trippable (boto change).
|
||||
- Implemented gsutil cp -v option for printing the version-specific
|
||||
URI that was just created.
|
||||
- Added error detail for attempt to delete non-empty versioned
|
||||
bucket. Also added versioning state to ls -L -b gs://bucket
|
||||
output.
|
||||
- Changed URI parsing to use pre-compiled regex's.
|
||||
- Other bug fixes.
|
||||
|
||||
- Rewrote/deepened/improved various parts of built-in help:
|
||||
- Updated 'gsutil help dev'.
|
||||
- Fixed help command handling when terminal does not have the
|
||||
number of rows set.
|
||||
- Rewrote versioning help.
|
||||
- Added gsutil help text for common 403 AccountProblem error.
|
||||
- Added text to 'gsutil help dev' about legal agreement needed
|
||||
with code submissions.
|
||||
- Fixed various other typos.
|
||||
- Updated doc for cp command regarding metadata not being
|
||||
preserved when copying between providers.
|
||||
- Fixed gsutil ls command documentation typo for the -L option.
|
||||
- Added HTTP scheme to doc/examples for gsutil setcors command.
|
||||
- Changed minimum version in documentation from 2.5 to 2.6 since
|
||||
gsutil no longer works in Python 2.5.
|
||||
- Cleaned up/clarify/deepen various other parts of gsutil
|
||||
built-in documentation.
|
||||
|
||||
- Numerous improvements to testing infrastructure:
|
||||
- Completely refactored infrastructure, allowing deeper testing
|
||||
and more readable test code, and enabling better debugging
|
||||
output when tests fail.
|
||||
- Moved gslib/test_*.py unit tests to gslib/tests module.
|
||||
- Made all tests (unit and integration, per-command and modules
|
||||
(like naming) run from single gsutil test command.
|
||||
- Moved TempDir functions from GsUtilIntegrationTestCase to
|
||||
GsUtilTestCase.
|
||||
- Made test runner message show the test function being run.
|
||||
- Added file path support to ObjectToURI function.
|
||||
- Disabled the test command if running on Python 2.6 and unittest2
|
||||
is not available instead of breaking all of gsutil.
|
||||
- Changed to pass GCS V2 API and project_id from boto config
|
||||
if necessary in integration_testcase#CreateBucket().
|
||||
- Fixed unit tests by using a GS-specific mocking class to
|
||||
override the S3 provider.
|
||||
- Added friendlier error message if test path munging fails.
|
||||
- Fixed bug where gsutil test only cleaned up first few test files.
|
||||
- Implemented setacl integration tests.
|
||||
- Implemented StorageUri parsing unit tests.
|
||||
- Implemented test for gsutil cp -D.
|
||||
- Implemented setacl integration tests.
|
||||
- Implemented tests for reading and seeking past end of file.
|
||||
- Implemented and tests for it in new tests module.
|
||||
- Changed cp tests that don't specify a Content-Type to check
|
||||
for new binary/octet-stream default instead of server-detected
|
||||
mime type.
|
||||
|
||||
- Changed gsutil mv to allow moving local files/dirs to the cloud.
|
||||
Previously this was disallowed in the belief we should be
|
||||
conservative about deleting data from local disk but there are
|
||||
legitimate use cases for moving data from a local dir to the
|
||||
cloud, it's clear to the user this would remove data from the
|
||||
local disk, and allowing it makes the tool behavior more
|
||||
consistent with what users would expect.
|
||||
|
||||
- Changed gsutil update command to insist on is_secure and
|
||||
https_validate_certificates.
|
||||
|
||||
- Fixed release no longer to include extraneous boto dirs in
|
||||
top-level of gsutil distribution (like bin/ and docs/).
|
||||
|
||||
- Changed resumable upload threshold from 1 MB to 2 MB.
|
||||
|
||||
- Removed leftover cloudauth and cloudreader dirs. Sample code
|
||||
now lives at https://github.com/GoogleCloudPlatform.
|
||||
|
||||
- Updated copyright notice on code files.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.21 (release-date: 2012-12-10)
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Added the ability for the cp command to continue even if there is an
|
||||
error. This can be activated with the -c flag.
|
||||
|
||||
- Added support for specifying src args for gsutil cp on stdin (-I option)
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed gsutil test cp, which assumed it was run from gsutil install dir.
|
||||
|
||||
- Mods so we send generation subresource only when user requested
|
||||
version parsing (-v option for cp and cat commands).
|
||||
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Updated docs about using setmeta with versioning enabled.
|
||||
|
||||
- Changed GCS endpoint in boto to storage.googleapis.com.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.20 (release-date: 2012-11-30)
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Added a noclobber (-n) setting for the cp command. Existing objects/files
|
||||
will not be overwritten when using this setting.
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed off-by-one error when reporting bytes transferred.
|
||||
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Improved versioning support for the remove command.
|
||||
|
||||
- Improved test runner support.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.19 (release-date: 2012-11-26)
|
||||
|
||||
New Features
|
||||
------------
|
||||
- Added support for object versions.
|
||||
|
||||
- Added support for storage classes (including Durable Reduced Availability).
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
- Fixed problem where cp -q prevented resumable uploads from being performed.
|
||||
|
||||
- Made setwebcfg and setcors tests robust wrt XML formatting variation.
|
||||
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Incorporated vapier@ mods to make version command not fail if CHECKSUM file
|
||||
missing.
|
||||
|
||||
- Refactored gsutil such that most functionality exists in boto.
|
||||
|
||||
- Updated gsutil help dev instructions for how to check out source.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.18 (release-date: 2012-09-19)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed resumable upload boundary condition when handling POST request
|
||||
when server already has complete file, which resulted in an infinite
|
||||
loop that consumed 100% of the CPU.
|
||||
|
||||
- Fixed one more place that outputted progress info when gsutil cp -q
|
||||
specified (during streaming uploads).
|
||||
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Updated help text for "gsutil help setmeta" and "gsutil help metadata", to
|
||||
clarify and deepen parts of the documentation.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.17 (release-date: 2012-08-17)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed race condition when multiple threads attempt to get an OAuth2 refresh
|
||||
token concurrently.
|
||||
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Implemented simplified syntax for setmeta command. The old syntax still
|
||||
works but is now deprecated.
|
||||
|
||||
- Added help to gsutil cp -z option, to describe how to change where temp
|
||||
files are written.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.16 (release-date: 2012-08-13)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Added info to built-in help for setmeta command, to explain the syntax
|
||||
needed when running from Windows.
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.15 (release-date: 2012-08-12)
|
||||
|
||||
New Features
|
||||
------------
|
||||
- Implemented gsutil setmeta command.
|
||||
|
||||
- Made gsutil understand bucket subdir conventions used by various tools
|
||||
(like GCS Manager and CloudBerry) so if you cp or mv to a subdir you
|
||||
created with one of those tools it will work as expected.
|
||||
|
||||
- Added support for Windows drive letter-prefaced paths when using Storage
|
||||
URIs.
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed performance bug when downloading a large object with Content-
|
||||
Encoding:gzip, where decompression attempted to load the entire object
|
||||
in memory. Also added "Uncompressing" log output if file is larger than
|
||||
50M, to make it clear the download hasn't stalled.
|
||||
|
||||
- Fixed naming bug when performing gsutil mv from a bucket subdir to
|
||||
and existing bucket subdir.
|
||||
|
||||
- Fixed bug that caused cross-provider copies into Google Cloud Storage to
|
||||
fail.
|
||||
|
||||
- Made change needed to make resumable transfer progress messages not print
|
||||
when running gsutil cp -q.
|
||||
|
||||
- Fixed copy/paste error in config file documentation for
|
||||
https_validate_certificates option.
|
||||
|
||||
- Various typo fixes.
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Changed gsutil to unset http_proxy environment variable if it's set,
|
||||
because it confuses boto. (Proxies should instead be configured via the
|
||||
boto config file.)
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.14 (release-date: 2012-07-28)
|
||||
|
||||
New Features
|
||||
------------
|
||||
- Added cp -q option, to support quiet operation from cron jobs.
|
||||
|
||||
- Made config command restore backed up file if there was a failure or user
|
||||
hits ^C.
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug where gsutil cp -R from a source directory didn't generate
|
||||
correct destination path.
|
||||
|
||||
- Fixed file handle leak in gsutil cp -z
|
||||
|
||||
- Fixed bug that caused cp -a option not to work when copying in the cloud.
|
||||
|
||||
- Fixed bug that caused '/-' to be appended to object name for streaming
|
||||
uploads.
|
||||
|
||||
- Revert incorrect line I changed in previous CL, that attempted to
|
||||
get fp from src_key object. The real fix that's needed is described in
|
||||
http://code.google.com/p/gsutil/issues/detail?id=73.
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- Changed logging to print "Copying..." and Content-Type on same line;
|
||||
refactored content type and log handling.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.13 (release-date: 2012-07-19)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Included the fix to make 'gsutil config' honor BOTO_CONFIG environment
|
||||
variable (which was intended to be included in Release 3.12)
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.11 (release-date: 2012-06-28)
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Added support for configuring website buckets.
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug that caused simultaneous resumable downloads of the same source
|
||||
object to use the same tracker file.
|
||||
|
||||
- Changed language code spec pointer from Wikipedia to loc.gov (for
|
||||
Content-Language header).
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.10 (release-date: 2012-06-19)
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Added support for setting and listing Content-Language header.
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug that caused getacl/setacl commands to get a character encoding
|
||||
exception when ACL content contained content not representable in ISO-8859-1
|
||||
character set.
|
||||
|
||||
- Fixed gsutil update not to fail under Windows exclusive file locking.
|
||||
|
||||
- Fixed gsutil ls -L to continue past 403 errors.
|
||||
|
||||
- Updated gsutil tests and also help dev with instructions on how to run
|
||||
boto tests, based on recent test refactoring done to in boto library.
|
||||
|
||||
- Cleaned up parts of cp help text.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.9 (release-date: 2012-05-24)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug that caused extra "file:/" to be included in pathnames with
|
||||
doing gsutil cp -R on Windows.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.8 (release-date: 2012-05-20)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed problem with non-ASCII filename characters not setting encoding before
|
||||
attempting to hash for generating resumable transfer filename.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.7 (release-date: 2012-05-11)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed handling of HTTPS tunneling through a proxy.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.6 (release-date: 2012-05-09)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug that caused wildcards spanning directories not to work.
|
||||
- Fixed bug that gsutil cp -z not to find available tmp space correctly
|
||||
under Windows.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.5 (release-date: 2012-04-30)
|
||||
|
||||
Performance Improvement
|
||||
-----------------------
|
||||
|
||||
- Change by Evan Worley to calculate MD5s incrementally during uploads and
|
||||
downloads. This reduces overall transfer time substantially for large
|
||||
objects.
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed bug where uploading and moving multiple files to a bucket subdirectory
|
||||
didn't work as intended.
|
||||
(http://code.google.com/p/gsutil/issues/detail?id=93).
|
||||
- Fixed bug where gsutil cp -r sourcedir didn't copy to specified subdir
|
||||
if there is only one file in sourcedir.
|
||||
- Fixed bug where tracker file included a timestamp that caused it not to
|
||||
be recognized across sessions.
|
||||
- Fixed bug where gs://bucket/*/dir wildcard matches too many objects.
|
||||
- Fixed documentation errors in help associated with ACLs and projects.
|
||||
- Changed GCS ACL parsing to be case-insensitive.
|
||||
- Changed ls to print error and exit with non-0 status when wildcard matches
|
||||
nothing, to be more consistent with UNIX shell behavior.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.4 (release-date: 2012-04-06)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed problem where resumable uploads/downloads of objects with very long
|
||||
names would generate tracking files with names that exceeded local file
|
||||
system limits, making it impossible to complete resumable transfers for
|
||||
those objects. Solution was to build the tracking file name from a fixed
|
||||
prefix, SHA1 hash of the long filename, epoch timestamp and last 16
|
||||
chars of the long filename, which is guarantee to be a predicable and
|
||||
reasonable length.
|
||||
|
||||
- Fixed minor bug in output from 'gsutil help dev' which advised executing
|
||||
an inconsequential test script (test_util.py).
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.3 (release-date: 2012-04-03)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed problem where gsutil ver and debug flags crashed when used
|
||||
with newly generated boto config files.
|
||||
|
||||
- Fixed gsutil bug in windows path handling, and make checksumming work
|
||||
across platforms.
|
||||
|
||||
- Fixed enablelogging to translate -b URI param to plain bucket name in REST
|
||||
API request.
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.2 (release-date: 2012-03-30)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Fixed problem where gsutil didn't convert between OS-specific directory
|
||||
separators when copying individually-named files (issue 87).
|
||||
|
||||
- Fixed problem where gsutil ls -R didn't work right if there was a key
|
||||
with a leading path (like /foo/bar/baz)
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.1 (release-date: 2012-03-20)
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- Removed erroneous setting of Content-Encoding when a gzip file is uploaded
|
||||
(vs running gsutil cp -z, when Content-Encoding should be set). This
|
||||
error caused users to get gsutil.tar.gz file uncompressed by the user
|
||||
agent (like wget) while downloading, making the file appear to be of the
|
||||
wrong size/content.
|
||||
|
||||
- Fixed handling of gsutil help for Windows (previous code depended on
|
||||
termios and fcntl libs, which are Linux/MacOS-specific).
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
|
||||
Release 3.0 (release-date: 2012-03-20)
|
||||
|
||||
|
||||
Important Notes
|
||||
---------------
|
||||
|
||||
- Backwards-incompatible wildcard change
|
||||
|
||||
The '*' wildcard now only matches objects within a bucket directory. If
|
||||
you have scripts that depend on being able to match spanning multiple
|
||||
directories you need to use '**' instead. For example, the command:
|
||||
|
||||
gsutil cp gs://bucket/*.txt
|
||||
|
||||
will now only match .txt files in the top-level directory.
|
||||
|
||||
gsutil cp gs://bucket/**.txt
|
||||
|
||||
will match across all directories.
|
||||
|
||||
- gsutil ls now lists one directory at a time. If you want to list all objects
|
||||
in a bucket, you can use:
|
||||
|
||||
gsutil ls gs://bucket/**
|
||||
|
||||
or:
|
||||
|
||||
gsutil ls -R gs://bucket
|
||||
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Built-in help for all commands and many additional topics. Try
|
||||
"gsutil help" for a list of available commands and topics.
|
||||
|
||||
- A new hierarchical file tree abstraction layer, which makes the flat bucket
|
||||
name space look like a hierarchical file tree. This makes several things
|
||||
possible:
|
||||
- copying data to/from bucket sub-directories (see “gsutil help cp”).
|
||||
- distributing large uploads/downloads across many machines
|
||||
(see “gsutil help cp”)
|
||||
- renaming bucket sub-directories (see “gsutil help mv”).
|
||||
- listing individual bucket sub-directories and for listing directories
|
||||
recursively (see “gsutil help ls”).
|
||||
- setting ACLs for objects in a sub-directory (see “gsutil help setacl”).
|
||||
|
||||
- Support for per-directory (*) and recursive (**) wildcards. Essentially,
|
||||
** works the way * did in previous gsutil releases, and * now behaves
|
||||
consistently with how it works in command interpreters (like bash). The
|
||||
ability to specify directory-only wildcards also enables a number of use
|
||||
cases, such as distributing large uploads/downloads by wildcarded name. See
|
||||
"gsutil help wildcards" for details.
|
||||
|
||||
- Support for Cross-Origin Resource Sharing (CORS) configuration. See "gsutil
|
||||
help cors" for details.
|
||||
|
||||
- Support for multi-threading and recursive operation for setacl command
|
||||
(see “gsutil help setacl”).
|
||||
|
||||
- Ability to use the UNIX 'file' command to do content type recognition as
|
||||
an alternative to filename extensions.
|
||||
|
||||
- Introduction of new end-to-end test suite.
|
||||
|
||||
- The gsutil version command now computes a checksum of the code, to detect
|
||||
corruption and local modification when assisting with technical support.
|
||||
|
||||
- The gsutil update command is no longer beta/experimental, and now also
|
||||
supports updating from named URIs (for early/test releases).
|
||||
|
||||
- Changed gsutil ls -L to also print Content-Disposition header.
|
||||
|
||||
|
||||
Bug Fixes
|
||||
---------
|
||||
|
||||
- The gsutil cp -t option previously didn't work as documented, and instead
|
||||
Content-Type was always detected based on filename extension. Content-Type
|
||||
detection is now the default, the -t option is deprecated (to be removed in
|
||||
the future), and specifying a -h Content-Type header now correctly overrides
|
||||
the filename extension based handling. For details see "gsutil help
|
||||
metadata".
|
||||
|
||||
- Fixed bug that caused multi-threaded mv command not to percolate failures
|
||||
during the cp phase to the rm phase, which could under some circumstances
|
||||
cause data that was not copied to be deleted.
|
||||
|
||||
- Fixed bug that caused gsutil to use GET for ls -L requests. It now uses HEAD
|
||||
for ls -L requests, which is more efficient and faster.
|
||||
|
||||
- Fixed bug that caused gsutil not to preserve metadata during
|
||||
copy-in-the-cloud.
|
||||
|
||||
- Fixed bug that prevented setacl command from allowing DisplayName's in ACLs.
|
||||
|
||||
- Fixed bug that caused gsutil/boto to suppress consecutive slashes in path
|
||||
names.
|
||||
|
||||
- Fixed spec-non-compliant URI construction for resumable uploads.
|
||||
|
||||
- Fixed bug that caused rm -f not to work.
|
||||
|
||||
- Fixed UnicodeEncodeError that happened when redirecting gsutil ls output
|
||||
to a file with non-ASCII object names.
|
||||
|
||||
|
||||
Other Changes
|
||||
-------------
|
||||
|
||||
- UserAgent sent in HTTP requests now includes gsutil version number and OS
|
||||
name.
|
||||
|
||||
- Starting with this release users are able to get individual named releases
|
||||
from version-named objects: gs://pub/gsutil_<version>.tar.gz
|
||||
and gs://pub/gsutil_<version>.zip. The version-less counterparts
|
||||
(gs://pub/gsutil.tar.gz and gs://pub/gsutil.zip) will contain the latest
|
||||
release. Also, the gs://pub bucket is now publicly readable (so, anyone
|
||||
can list its contents).
|
||||
|
||||
|
||||
================================================================================
|
||||
|
||||
Release 2.0 (release-date: 2012-01-13)
|
||||
|
||||
|
||||
New Features
|
||||
------------
|
||||
|
||||
- Support for for two new installation modes: enterprise and RPM.
|
||||
Customers can now install gsutil one of three ways:
|
||||
|
||||
- Individual user mode (previously the only available mode): unpacking from
|
||||
a gzipped tarball (gs://pub/gsutil.tar.gz) or zip file
|
||||
(gs://pub/gsutil.zip) and running the gsutil command in place in the
|
||||
unpacked gsutil directory.
|
||||
|
||||
- Enterprise mode (new): unpacking as above, and then running the setup.py
|
||||
script in the unpacked gsutil directory. This allows a systems
|
||||
administrator to install gsutil in a central location, using the Python
|
||||
distutils facility. This mode is supported only on Linux and MacOS.
|
||||
|
||||
- RPM mode (new). A RedHat RPM can be built from the gsutil.spec.in file
|
||||
in the unpacked gsutil directory, allowing it to be installed as part of
|
||||
a RedHat build.
|
||||
|
||||
- Note: v2.0 is the first numbered gsutil release. Previous releases
|
||||
were given timestamps for versions. Numbered releases enable downstream
|
||||
package builds (like RPMs) to define dependencies more easily.
|
||||
This is also the first version where we began including release notes.
|
||||
@ -0,0 +1 @@
|
||||
3.25
|
||||
@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf8
|
||||
# Copyright 2010 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Main module for Google Cloud Storage command line tool."""
|
||||
|
||||
|
||||
import ConfigParser
|
||||
import errno
|
||||
import getopt
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
third_party_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, os.path.dirname(third_party_dir))
|
||||
sys.path.insert(0, third_party_dir)
|
||||
|
||||
|
||||
def _OutputAndExit(message):
|
||||
global debug
|
||||
if debug == 4:
|
||||
stack_trace = traceback.format_exc()
|
||||
sys.stderr.write('DEBUG: Exception stack trace:\n %s\n' %
|
||||
re.sub('\\n', '\n ', stack_trace))
|
||||
else:
|
||||
sys.stderr.write('%s\n' % message)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _OutputUsageAndExit(command_runner):
|
||||
command_runner.RunNamedCommand('help')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
debug = 0
|
||||
# Before importing boto, find where gsutil is installed and include its
|
||||
# boto sub-directory at the start of the PYTHONPATH, to ensure the versions of
|
||||
# gsutil and boto stay in sync after software updates. This also allows gsutil
|
||||
# to be used without explicitly adding it to the PYTHONPATH.
|
||||
# We use realpath() below to unwind symlinks if any were used in the gsutil
|
||||
# installation.
|
||||
gsutil_bin_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
|
||||
if not gsutil_bin_dir:
|
||||
_OutputAndExit('Unable to determine where gsutil is installed. Sorry, '
|
||||
'cannot run correctly without this.\n')
|
||||
boto_lib_dir = os.path.join(gsutil_bin_dir, '..', 'boto')
|
||||
if not os.path.isdir(boto_lib_dir):
|
||||
_OutputAndExit('There is no boto library under the gsutil install directory '
|
||||
'(%s).\nThe gsutil command cannot work properly when installed '
|
||||
'this way.\nPlease re-install gsutil per the installation '
|
||||
'instructions.' % gsutil_bin_dir)
|
||||
# sys.path.insert(0, boto_lib_dir)
|
||||
import boto
|
||||
from boto.exception import BotoClientError
|
||||
from boto.exception import InvalidAclError
|
||||
from boto.exception import InvalidUriError
|
||||
from boto.exception import ResumableUploadException
|
||||
from boto.exception import StorageResponseError
|
||||
from gslib.command_runner import CommandRunner
|
||||
from gslib.exception import CommandException
|
||||
from gslib.exception import ProjectIdException
|
||||
from gslib import util
|
||||
from gslib.util import ExtractErrorDetail
|
||||
from gslib.util import HasConfiguredCredentials
|
||||
from gslib.wildcard_iterator import WildcardException
|
||||
|
||||
# Load the gsutil version number and append it to boto.UserAgent so the value
|
||||
# is set before anything instantiates boto. (If parts of boto were instantiated
|
||||
# first those parts would have the old value of boto.UserAgent, so we wouldn't
|
||||
# be guaranteed that all code paths send the correct user agent.)
|
||||
ver_file_path = os.path.join(gsutil_bin_dir, 'VERSION')
|
||||
if not os.path.isfile(ver_file_path):
|
||||
raise CommandException(
|
||||
'%s not found. Please reinstall gsutil from scratch' % ver_file_path)
|
||||
ver_file = open(ver_file_path, 'r')
|
||||
gsutil_ver = ver_file.read().rstrip()
|
||||
ver_file.close()
|
||||
boto.UserAgent += ' gsutil/%s (%s)' % (gsutil_ver, sys.platform)
|
||||
|
||||
# We don't use the oauth2 authentication plugin directly; importing it here
|
||||
# ensures that it's loaded and available by default when an operation requiring
|
||||
# authentication is performed.
|
||||
try:
|
||||
from oauth2_plugin import oauth2_plugin
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
global debug
|
||||
|
||||
if sys.version_info[:3] < (2, 6):
|
||||
raise CommandException('gsutil requires Python 2.6 or higher.')
|
||||
|
||||
config_file_list = _GetBotoConfigFileList()
|
||||
command_runner = CommandRunner(gsutil_bin_dir, boto_lib_dir, config_file_list,
|
||||
gsutil_ver)
|
||||
headers = {}
|
||||
parallel_operations = False
|
||||
debug = 0
|
||||
|
||||
# If user enters no commands just print the usage info.
|
||||
if len(sys.argv) == 1:
|
||||
sys.argv.append('help')
|
||||
|
||||
# Change the default of the 'https_validate_certificates' boto option to
|
||||
# True (it is currently False in boto).
|
||||
if not boto.config.has_option('Boto', 'https_validate_certificates'):
|
||||
if not boto.config.has_section('Boto'):
|
||||
boto.config.add_section('Boto')
|
||||
boto.config.setbool('Boto', 'https_validate_certificates', True)
|
||||
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'dDvh:m',
|
||||
['debug', 'detailedDebug', 'version', 'help',
|
||||
'header', 'multithreaded'])
|
||||
except getopt.GetoptError, e:
|
||||
_HandleCommandException(CommandException(e.msg))
|
||||
for o, a in opts:
|
||||
if o in ('-d', '--debug'):
|
||||
# Passing debug=2 causes boto to include httplib header output.
|
||||
debug = 2
|
||||
if o in ('-D', '--detailedDebug'):
|
||||
# We use debug level 3 to ask gsutil code to output more detailed
|
||||
# debug output. This is a bit of a hack since it overloads the same
|
||||
# flag that was originally implemented for boto use. And we use -DD
|
||||
# to ask for really detailed debugging (i.e., including HTTP payload).
|
||||
if debug == 3:
|
||||
debug = 4
|
||||
else:
|
||||
debug = 3
|
||||
if o in ('-?', '--help'):
|
||||
_OutputUsageAndExit(command_runner)
|
||||
if o in ('-h', '--header'):
|
||||
(hdr_name, unused_ptn, hdr_val) = a.partition(':')
|
||||
if not hdr_name:
|
||||
_OutputUsageAndExit(command_runner)
|
||||
headers[hdr_name] = hdr_val
|
||||
if o in ('-m', '--multithreaded'):
|
||||
parallel_operations = True
|
||||
if debug > 1:
|
||||
sys.stderr.write(
|
||||
'***************************** WARNING *****************************\n'
|
||||
'*** You are running gsutil with debug output enabled.\n'
|
||||
'*** Be aware that debug output includes authentication '
|
||||
'credentials.\n'
|
||||
'*** Do not share (e.g., post to support forums) debug output\n'
|
||||
'*** unless you have sanitized authentication tokens in the\n'
|
||||
'*** output, or have revoked your credentials.\n'
|
||||
'***************************** WARNING *****************************\n')
|
||||
if debug == 2:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
elif debug > 2:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
command_runner.RunNamedCommand('ver')
|
||||
config_items = []
|
||||
try:
|
||||
config_items.extend(boto.config.items('Boto'))
|
||||
config_items.extend(boto.config.items('GSUtil'))
|
||||
except ConfigParser.NoSectionError:
|
||||
pass
|
||||
sys.stderr.write('config_file_list: %s\n' % config_file_list)
|
||||
sys.stderr.write('config: %s\n' % str(config_items))
|
||||
else:
|
||||
logging.basicConfig()
|
||||
|
||||
if not args:
|
||||
command_name = 'help'
|
||||
else:
|
||||
command_name = args[0]
|
||||
|
||||
# Unset http_proxy environment variable if it's set, because it confuses
|
||||
# boto. (Proxies should instead be configured via the boto config file.)
|
||||
if 'http_proxy' in os.environ:
|
||||
if debug > 1:
|
||||
sys.stderr.write(
|
||||
'Unsetting http_proxy environment variable within gsutil run.\n')
|
||||
del os.environ['http_proxy']
|
||||
|
||||
return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
|
||||
args[1:], headers, debug,
|
||||
parallel_operations)
|
||||
|
||||
|
||||
def _GetBotoConfigFileList():
|
||||
"""Returns list of boto config files that exist."""
|
||||
config_paths = boto.pyami.config.BotoConfigLocations
|
||||
if 'AWS_CREDENTIAL_FILE' in os.environ:
|
||||
config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
|
||||
config_files = {}
|
||||
for config_path in config_paths:
|
||||
if os.path.exists(config_path):
|
||||
config_files[config_path] = 1
|
||||
cf_list = []
|
||||
for config_file in config_files:
|
||||
cf_list.append(config_file)
|
||||
return cf_list
|
||||
|
||||
|
||||
def _HandleUnknownFailure(e):
|
||||
global debug
|
||||
# Called if we fall through all known/handled exceptions. Allows us to
|
||||
# print a stacktrace if -D option used.
|
||||
if debug > 2:
|
||||
stack_trace = traceback.format_exc()
|
||||
sys.stderr.write('DEBUG: Exception stack trace:\n %s\n' %
|
||||
re.sub('\\n', '\n ', stack_trace))
|
||||
else:
|
||||
_OutputAndExit('Failure: %s.' % e)
|
||||
|
||||
|
||||
def _HandleCommandException(e):
|
||||
if e.informational:
|
||||
_OutputAndExit(e.reason)
|
||||
else:
|
||||
_OutputAndExit('CommandException: %s' % e.reason)
|
||||
|
||||
|
||||
def _HandleControlC(signal_num, cur_stack_frame):
|
||||
"""Called when user hits ^C so we can print a brief message instead of
|
||||
the normal Python stack trace (unless -D option is used)."""
|
||||
global debug
|
||||
if debug > 2:
|
||||
stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
|
||||
_OutputAndExit('DEBUG: Caught signal %d - Exception stack trace:\n'
|
||||
' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace)))
|
||||
else:
|
||||
_OutputAndExit('Caught signal %d - exiting' % signal_num)
|
||||
|
||||
|
||||
def _HandleSigQuit(signal_num, cur_stack_frame):
|
||||
"""Called when user hits ^\, so we can force breakpoint a running gsutil."""
|
||||
import pdb; pdb.set_trace()
|
||||
|
||||
|
||||
def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
|
||||
headers=None, debug=0,
|
||||
parallel_operations=False):
|
||||
try:
|
||||
# Catch ^C so we can print a brief message instead of the normal Python
|
||||
# stack trace.
|
||||
signal.signal(signal.SIGINT, _HandleControlC)
|
||||
# Catch ^\ so we can force a breakpoint in a running gsutil.
|
||||
if not util.IS_WINDOWS:
|
||||
signal.signal(signal.SIGQUIT, _HandleSigQuit)
|
||||
return command_runner.RunNamedCommand(command_name, args, headers, debug,
|
||||
parallel_operations)
|
||||
except AttributeError, e:
|
||||
if str(e).find('secret_access_key') != -1:
|
||||
_OutputAndExit('Missing credentials for the given URI(s). Does your '
|
||||
'boto config file contain all needed credentials?')
|
||||
else:
|
||||
_OutputAndExit(str(e))
|
||||
except BotoClientError, e:
|
||||
_OutputAndExit('BotoClientError: %s.' % e.reason)
|
||||
except CommandException, e:
|
||||
_HandleCommandException(e)
|
||||
except getopt.GetoptError, e:
|
||||
_HandleCommandException(CommandException(e.msg))
|
||||
except InvalidAclError, e:
|
||||
_OutputAndExit('InvalidAclError: %s.' % str(e))
|
||||
except InvalidUriError, e:
|
||||
_OutputAndExit('InvalidUriError: %s.' % e.message)
|
||||
except ProjectIdException, e:
|
||||
_OutputAndExit('ProjectIdException: %s.' % e.reason)
|
||||
except boto.auth_handler.NotReadyToAuthenticate:
|
||||
_OutputAndExit('NotReadyToAuthenticate')
|
||||
except OSError, e:
|
||||
_OutputAndExit('OSError: %s.' % e.strerror)
|
||||
except WildcardException, e:
|
||||
_OutputAndExit(e.reason)
|
||||
except StorageResponseError, e:
|
||||
# Check for access denied, and provide detail to users who have no boto
|
||||
# config file (who might previously have been using gsutil only for
|
||||
# accessing publicly readable buckets and objects).
|
||||
if e.status == 403:
|
||||
if not HasConfiguredCredentials():
|
||||
_OutputAndExit(
|
||||
'You are attempting to access protected data with no configured '
|
||||
'credentials.\nPlease see '
|
||||
'http://code.google.com/apis/storage/docs/signup.html for\ndetails '
|
||||
'about activating the Google Cloud Storage service and then run '
|
||||
'the\n"gsutil config" command to configure gsutil to use these '
|
||||
'credentials.')
|
||||
elif (e.error_code == 'AccountProblem'
|
||||
and ','.join(args).find('gs://') != -1):
|
||||
default_project_id = boto.config.get_value('GSUtil',
|
||||
'default_project_id')
|
||||
acct_help_part_1 = (
|
||||
"""Your request resulted in an AccountProblem (403) error. Usually this happens
|
||||
if you attempt to create a bucket or upload an object without having first
|
||||
enabled billing for the project you are using. To remedy this problem, please do
|
||||
the following:
|
||||
|
||||
1. Navigate to the Google APIs console (https://code.google.com/apis/console),
|
||||
and ensure the drop-down selector beneath "Google APIs" shows the project
|
||||
you're attempting to use.
|
||||
|
||||
""")
|
||||
acct_help_part_2 = '\n'
|
||||
if default_project_id:
|
||||
acct_help_part_2 = (
|
||||
"""2. Click "Google Cloud Storage" on the left hand pane, and then check that
|
||||
the value listed for "x-goog-project-id" on this page matches the project ID
|
||||
(%s) from your boto config file.
|
||||
|
||||
""" % default_project_id)
|
||||
acct_help_part_3 = (
|
||||
"""Check whether there's an "!" next to Billing. If so, click Billing and then
|
||||
enable billing for this project. Note that it can take up to one hour after
|
||||
enabling billing for the project to become activated for creating buckets and
|
||||
uploading objects.
|
||||
|
||||
If the above doesn't resolve your AccountProblem, please send mail to
|
||||
gs-team@google.com requesting assistance, noting the exact command you ran, the
|
||||
fact that you received a 403 AccountProblem error, and your project ID. Please
|
||||
do not post your project ID on the public discussion forum (gs-discussion) or on
|
||||
StackOverflow.
|
||||
|
||||
Note: It's possible to use Google Cloud Storage without enabling billing if
|
||||
you're only listing or reading objects for which you're authorized, or if
|
||||
you're uploading objects to a bucket billed to a project that has billing
|
||||
enabled. But if you're attempting to create buckets or upload objects to a
|
||||
bucket owned by your own project, you must first enable billing for that
|
||||
project.""")
|
||||
if default_project_id:
|
||||
_OutputAndExit(acct_help_part_1 + acct_help_part_2 + '3. '
|
||||
+ acct_help_part_3)
|
||||
else:
|
||||
_OutputAndExit(acct_help_part_1 + '2. ' + acct_help_part_3)
|
||||
|
||||
if not e.body:
|
||||
e.body = ''
|
||||
exc_name, error_detail = ExtractErrorDetail(e)
|
||||
if error_detail:
|
||||
_OutputAndExit('%s: status=%d, code=%s, reason=%s, detail=%s.' %
|
||||
(exc_name, e.status, e.code, e.reason, error_detail))
|
||||
else:
|
||||
_OutputAndExit('%s: status=%d, code=%s, reason=%s.' %
|
||||
(exc_name, e.status, e.code, e.reason))
|
||||
except ResumableUploadException, e:
|
||||
_OutputAndExit('ResumableUploadException: %s.' % e.message)
|
||||
except socket.error, e:
|
||||
if e.args[0] == errno.EPIPE:
|
||||
# Retrying with a smaller file (per suggestion below) works because
|
||||
# the library code send loop (in boto/s3/key.py) can get through the
|
||||
# entire file and then request the HTTP response before the socket
|
||||
# gets closed and the response lost.
|
||||
message = (
|
||||
"""
|
||||
Got a "Broken pipe" error. This can happen to clients using Python 2.x,
|
||||
when the server sends an error response and then closes the socket (see
|
||||
http://bugs.python.org/issue5542). If you are trying to upload a large
|
||||
object you might retry with a small (say 200k) object, and see if you get
|
||||
a more specific error code.
|
||||
""")
|
||||
_OutputAndExit(message)
|
||||
else:
|
||||
_HandleUnknownFailure(e)
|
||||
except Exception, e:
|
||||
_HandleUnknownFailure(e)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,75 @@
|
||||
#
|
||||
# gsutil.spec - RPM specification file for Google Cloud Storage command
|
||||
# line utility (gsutil).
|
||||
#
|
||||
# Copyright 2011 Google Inc.
|
||||
#
|
||||
|
||||
Name: gsutil
|
||||
Version: 2.0
|
||||
Release: 1%{?dist}
|
||||
Summary: gsutil command line utility for Google Cloud Storage
|
||||
License: ASL 2.0
|
||||
Group: Development/Libraries
|
||||
Url: http://code.google.com/apis/storage/docs/gsutil.html
|
||||
Source0: http://gsutil.googlecode.com/files/%{name}-%{version}.zip
|
||||
BuildArch: noarch
|
||||
BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
|
||||
# Dependency on boto commented out for now because initially we plan to
|
||||
# bundle boto with this package, however, when we're ready to depend on
|
||||
# a separate boto rpm package, this line should be uncommented.
|
||||
#Requires: python-boto
|
||||
|
||||
%description
|
||||
|
||||
GSUtil is a Python application that facilitates access to Google Cloud Storage
|
||||
from the command line. You can use GSUtil to do a wide range of bucket and
|
||||
object management tasks, including:
|
||||
|
||||
- Creating and deleting buckets.
|
||||
- Uploading, downloading, and deleting objects.
|
||||
- Listing buckets and objects.
|
||||
- Moving, copying, and renaming objects.
|
||||
- Setting object and bucket ACLs.
|
||||
|
||||
%prep
|
||||
%setup -q
|
||||
|
||||
%build
|
||||
python setup.py build
|
||||
|
||||
%install
|
||||
python setup.py install --skip-build --root=%{buildroot}
|
||||
# Make all files and dirs in build area readable by other
|
||||
# and make all directories executable by other. These steps
|
||||
# are performed in support of the rpm installation mode,
|
||||
# in which users with different user/group than the
|
||||
# installation user/group must be able to run gsutil.
|
||||
chmod -R o+r %{buildroot}/usr/share/gsutil
|
||||
find %{buildroot}/usr/share/gsutil -type d | xargs chmod o+x
|
||||
# Make main gsutil script readable and executable by other.
|
||||
chmod o+rx %{buildroot}/usr/share/gsutil/gsutil
|
||||
# Remove Python egg file, which we don't use (but setup.py insists on
|
||||
# building) so we remove it here.
|
||||
rm %{buildroot}/usr/local/lib/python2.6/dist-packages/gsutil-2.0.egg-info
|
||||
# Remove update command, which shouldn't be used when gsutil is managed by RPM.
|
||||
rm %{buildroot}/usr/share/gsutil/gslib/commands/update.py
|
||||
# Create /usr/bin under buildroot and symlink gsutil so users don't
|
||||
# need to add a custom directory to their PATH.
|
||||
mkdir -p %{buildroot}%{_bindir}
|
||||
cd %{buildroot}%{_bindir}
|
||||
ln -s ../share/gsutil/gsutil gsutil
|
||||
|
||||
%clean
|
||||
rm -rf %{buildroot}
|
||||
|
||||
%files
|
||||
%defattr(-,root,root,-)
|
||||
# Lines ending with a slash cause recursive enumeration of directory contents.
|
||||
%{_bindir}/%{name}
|
||||
###FILES_GO_HERE###
|
||||
|
||||
%changelog
|
||||
* Tue Dec 10 2011 Marc Cohen <gs-team@google.com> 2.0-1
|
||||
- initial version of rpm spec file for gsutil for inclusion in RHEL
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the
|
||||
# "Software"), to deal in the Software without restriction, including
|
||||
# without limitation the rights to use, copy, modify, merge, publish, dis-
|
||||
# tribute, sublicense, and/or sell copies of the Software, and to permit
|
||||
# persons to whom the Software is furnished to do so, subject to the fol-
|
||||
# lowing conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included
|
||||
# in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
||||
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||||
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
# IN THE SOFTWARE.
|
||||
|
||||
"""Package marker file."""
|
||||
@ -0,0 +1,642 @@
|
||||
# Copyright 2010 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""An OAuth2 client library.
|
||||
|
||||
This library provides a client implementation of the OAuth2 protocol (see
|
||||
http://code.google.com/apis/accounts/docs/OAuth2.html).
|
||||
|
||||
**** Experimental API ****
|
||||
|
||||
This module is experimental and is subject to modification or removal without
|
||||
notice.
|
||||
"""
|
||||
|
||||
# This implementation is inspired by the implementation in
|
||||
# http://code.google.com/p/google-api-python-client/source/browse/oauth2client/,
|
||||
# with the following main differences:
|
||||
# - This library uses the fancy_urllib monkey patch for urllib to correctly
|
||||
# implement SSL certificate validation.
|
||||
# - This library does not assume that client code is using the httplib2 library
|
||||
# to make HTTP requests.
|
||||
# - This library implements caching of access tokens independent of refresh
|
||||
# tokens (in the python API client oauth2client, there is a single class that
|
||||
# encapsulates both refresh and access tokens).
|
||||
|
||||
|
||||
import cgi
|
||||
import datetime
|
||||
import errno
|
||||
from hashlib import sha1
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import urllib
|
||||
import urllib2
|
||||
import urlparse
|
||||
|
||||
from boto import cacerts
|
||||
from third_party import fancy_urllib
|
||||
|
||||
try:
|
||||
import json
|
||||
except ImportError:
|
||||
try:
|
||||
# Try to import from django, should work on App Engine
|
||||
from django.utils import simplejson as json
|
||||
except ImportError:
|
||||
# Try for simplejson
|
||||
import simplejson as json
|
||||
|
||||
LOG = logging.getLogger('oauth2_client')
|
||||
# Lock used for checking/exchanging refresh token, so multithreaded
|
||||
# operation doesn't attempt concurrent refreshes.
|
||||
token_exchange_lock = threading.Lock()
|
||||
|
||||
# SHA1 sum of the CA certificates file imported from boto.
|
||||
CACERTS_FILE_SHA1SUM = 'ed024a78d9327f8669b3b117d9eac9e3c9460e9b'
|
||||
|
||||
class Error(Exception):
|
||||
"""Base exception for the OAuth2 module."""
|
||||
pass
|
||||
|
||||
|
||||
class AccessTokenRefreshError(Error):
|
||||
"""Error trying to exchange a refresh token into an access token."""
|
||||
pass
|
||||
|
||||
|
||||
class AuthorizationCodeExchangeError(Error):
|
||||
"""Error trying to exchange an authorization code into a refresh token."""
|
||||
pass
|
||||
|
||||
|
||||
class TokenCache(object):
|
||||
"""Interface for OAuth2 token caches."""
|
||||
|
||||
def PutToken(self, key, value):
|
||||
raise NotImplementedError
|
||||
|
||||
def GetToken(self, key):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class NoopTokenCache(TokenCache):
|
||||
"""A stub implementation of TokenCache that does nothing."""
|
||||
|
||||
def PutToken(self, key, value):
|
||||
pass
|
||||
|
||||
def GetToken(self, key):
|
||||
return None
|
||||
|
||||
|
||||
class InMemoryTokenCache(TokenCache):
|
||||
"""An in-memory token cache.
|
||||
|
||||
The cache is implemented by a python dict, and inherits the thread-safety
|
||||
properties of dict.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(InMemoryTokenCache, self).__init__()
|
||||
self.cache = dict()
|
||||
|
||||
def PutToken(self, key, value):
|
||||
LOG.info('InMemoryTokenCache.PutToken: key=%s', key)
|
||||
self.cache[key] = value
|
||||
|
||||
def GetToken(self, key):
|
||||
value = self.cache.get(key, None)
|
||||
LOG.info('InMemoryTokenCache.GetToken: key=%s%s present',
|
||||
key, ' not' if value is None else '')
|
||||
return value
|
||||
|
||||
|
||||
class FileSystemTokenCache(TokenCache):
|
||||
"""An implementation of a token cache that persists tokens on disk.
|
||||
|
||||
Each token object in the cache is stored in serialized form in a separate
|
||||
file. The cache file's name can be configured via a path pattern that is
|
||||
parameterized by the key under which a value is cached and optionally the
|
||||
current processes uid as obtained by os.getuid().
|
||||
|
||||
Since file names are generally publicly visible in the system, it is important
|
||||
that the cache key does not leak information about the token's value. If
|
||||
client code computes cache keys from token values, a cryptographically strong
|
||||
one-way function must be used.
|
||||
"""
|
||||
|
||||
def __init__(self, path_pattern=None):
|
||||
"""Creates a FileSystemTokenCache.
|
||||
|
||||
Args:
|
||||
path_pattern: Optional string argument to specify the path pattern for
|
||||
cache files. The argument should be a path with format placeholders
|
||||
'%(key)s' and optionally '%(uid)s'. If the argument is omitted, the
|
||||
default pattern
|
||||
<tmpdir>/oauth2client-tokencache.%(uid)s.%(key)s
|
||||
is used, where <tmpdir> is replaced with the system temp dir as
|
||||
obtained from tempfile.gettempdir().
|
||||
"""
|
||||
super(FileSystemTokenCache, self).__init__()
|
||||
self.path_pattern = path_pattern
|
||||
if not path_pattern:
|
||||
self.path_pattern = os.path.join(
|
||||
tempfile.gettempdir(), 'oauth2_client-tokencache.%(uid)s.%(key)s')
|
||||
|
||||
def CacheFileName(self, key):
|
||||
uid = '_'
|
||||
try:
|
||||
# os.getuid() doesn't seem to work in Windows
|
||||
uid = str(os.getuid())
|
||||
except:
|
||||
pass
|
||||
return self.path_pattern % {'key': key, 'uid': uid}
|
||||
|
||||
def PutToken(self, key, value):
|
||||
"""Serializes the value to the key's filename.
|
||||
|
||||
To ensure that written tokens aren't leaked to a different users, we
|
||||
a) unlink an existing cache file, if any (to ensure we don't fall victim
|
||||
to symlink attacks and the like),
|
||||
b) create a new file with O_CREAT | O_EXCL (to ensure nobody is trying to
|
||||
race us)
|
||||
If either of these steps fail, we simply give up (but log a warning). Not
|
||||
caching access tokens is not catastrophic, and failure to create a file
|
||||
can happen for either of the following reasons:
|
||||
- someone is attacking us as above, in which case we want to default to
|
||||
safe operation (not write the token);
|
||||
- another legitimate process is racing us; in this case one of the two
|
||||
will win and write the access token, which is fine;
|
||||
- we don't have permission to remove the old file or write to the
|
||||
specified directory, in which case we can't recover
|
||||
|
||||
Args:
|
||||
key: the refresh_token hash key to store.
|
||||
value: the access_token value to serialize.
|
||||
"""
|
||||
|
||||
cache_file = self.CacheFileName(key)
|
||||
LOG.info('FileSystemTokenCache.PutToken: key=%s, cache_file=%s',
|
||||
key, cache_file)
|
||||
try:
|
||||
os.unlink(cache_file)
|
||||
except:
|
||||
# Ignore failure to unlink the file; if the file exists and can't be
|
||||
# unlinked, the subsequent open with O_CREAT | O_EXCL will fail.
|
||||
pass
|
||||
|
||||
flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
|
||||
|
||||
# Accommodate Windows; stolen from python2.6/tempfile.py.
|
||||
if hasattr(os, 'O_NOINHERIT'):
|
||||
flags |= os.O_NOINHERIT
|
||||
if hasattr(os, 'O_BINARY'):
|
||||
flags |= os.O_BINARY
|
||||
|
||||
try:
|
||||
fd = os.open(cache_file, flags, 0600)
|
||||
except (OSError, IOError), e:
|
||||
LOG.warning('FileSystemTokenCache.PutToken: '
|
||||
'Failed to create cache file %s: %s', cache_file, e)
|
||||
return
|
||||
f = os.fdopen(fd, 'w+b')
|
||||
f.write(value.Serialize())
|
||||
f.close()
|
||||
|
||||
def GetToken(self, key):
|
||||
"""Returns a deserialized access token from the key's filename."""
|
||||
value = None
|
||||
cache_file = self.CacheFileName(key)
|
||||
try:
|
||||
f = open(cache_file)
|
||||
value = AccessToken.UnSerialize(f.read())
|
||||
f.close()
|
||||
except (IOError, OSError), e:
|
||||
if e.errno != errno.ENOENT:
|
||||
LOG.warning('FileSystemTokenCache.GetToken: '
|
||||
'Failed to read cache file %s: %s', cache_file, e)
|
||||
except Exception, e:
|
||||
LOG.warning('FileSystemTokenCache.GetToken: '
|
||||
'Failed to read cache file %s (possibly corrupted): %s',
|
||||
cache_file, e)
|
||||
|
||||
LOG.info('FileSystemTokenCache.GetToken: key=%s%s present (cache_file=%s)',
|
||||
key, ' not' if value is None else '', cache_file)
|
||||
return value
|
||||
|
||||
|
||||
class OAuth2Provider(object):
|
||||
"""Encapsulates information about an OAuth2 provider."""
|
||||
|
||||
def __init__(self, label, authorization_uri, token_uri):
|
||||
"""Creates an OAuth2Provider.
|
||||
|
||||
Args:
|
||||
label: A string identifying this oauth2 provider, e.g. "Google".
|
||||
authorization_uri: The provider's authorization URI.
|
||||
token_uri: The provider's token endpoint URI.
|
||||
"""
|
||||
self.label = label
|
||||
self.authorization_uri = authorization_uri
|
||||
self.token_uri = token_uri
|
||||
|
||||
|
||||
class OAuth2Client(object):
|
||||
"""An OAuth2 client."""
|
||||
|
||||
def __init__(self, provider, client_id, client_secret,
|
||||
url_opener=None,
|
||||
proxy=None,
|
||||
access_token_cache=None,
|
||||
datetime_strategy=datetime.datetime):
|
||||
"""Creates an OAuth2Client.
|
||||
|
||||
Args:
|
||||
provider: The OAuth2Provider provider this client will authenticate
|
||||
against.
|
||||
client_id: The OAuth2 client ID of this client.
|
||||
client_secret: The OAuth2 client secret of this client.
|
||||
url_opener: An optinal urllib2.OpenerDirector to use for making HTTP
|
||||
requests to the OAuth2 provider's token endpoint. The provided
|
||||
url_opener *must* be configured to validate server SSL certificates
|
||||
for requests to https connections, and to correctly handle proxying of
|
||||
https requests. If this argument is omitted or None, a suitable
|
||||
opener based on fancy_urllib is used.
|
||||
proxy: An optional string specifying a HTTP proxy to be used, in the form
|
||||
'<proxy>:<port>'. This option is only effective if the url_opener has
|
||||
been configured with a fancy_urllib.FancyProxyHandler (this is the
|
||||
case for the default url_opener).
|
||||
access_token_cache: An optional instance of a TokenCache. If omitted or
|
||||
None, an InMemoryTokenCache is used.
|
||||
datetime_strategy: datetime module strategy to use.
|
||||
"""
|
||||
self.provider = provider
|
||||
self.client_id = client_id
|
||||
self.client_secret = client_secret
|
||||
# datetime_strategy is used to invoke utcnow() on; it is injected into the
|
||||
# constructor for unit testing purposes.
|
||||
self.datetime_strategy = datetime_strategy
|
||||
self._proxy = proxy
|
||||
|
||||
self.access_token_cache = access_token_cache or InMemoryTokenCache()
|
||||
|
||||
self.ca_certs_file = os.path.join(
|
||||
os.path.dirname(os.path.abspath(cacerts.__file__)), 'cacerts.txt')
|
||||
|
||||
if url_opener is None:
|
||||
# Check that the cert file distributed with boto has not been tampered
|
||||
# with.
|
||||
h = sha1()
|
||||
h.update(file(self.ca_certs_file).read())
|
||||
actual_sha1 = h.hexdigest()
|
||||
if actual_sha1 != CACERTS_FILE_SHA1SUM:
|
||||
raise Error(
|
||||
'CA certificates file does not have expected SHA1 sum; '
|
||||
'expected: %s, actual: %s' % (CACERTS_FILE_SHA1SUM, actual_sha1))
|
||||
# TODO(Google): set user agent?
|
||||
url_opener = urllib2.build_opener(
|
||||
fancy_urllib.FancyProxyHandler(),
|
||||
fancy_urllib.FancyRedirectHandler(),
|
||||
fancy_urllib.FancyHTTPSHandler())
|
||||
self.url_opener = url_opener
|
||||
|
||||
def _TokenRequest(self, request):
|
||||
"""Make a requst to this client's provider's token endpoint.
|
||||
|
||||
Args:
|
||||
request: A dict with the request parameteres.
|
||||
Returns:
|
||||
A tuple (response, error) where,
|
||||
- response is the parsed JSON response received from the token endpoint,
|
||||
or None if no parseable response was received, and
|
||||
- error is None if the request succeeded or
|
||||
an Exception if an error occurred.
|
||||
"""
|
||||
|
||||
body = urllib.urlencode(request)
|
||||
LOG.debug('_TokenRequest request: %s', body)
|
||||
response = None
|
||||
try:
|
||||
request = fancy_urllib.FancyRequest(
|
||||
self.provider.token_uri, data=body)
|
||||
if self._proxy:
|
||||
request.set_proxy(self._proxy, 'http')
|
||||
|
||||
request.set_ssl_info(ca_certs=self.ca_certs_file)
|
||||
result = self.url_opener.open(request)
|
||||
resp_body = result.read()
|
||||
LOG.debug('_TokenRequest response: %s', resp_body)
|
||||
except urllib2.HTTPError, e:
|
||||
try:
|
||||
response = json.loads(e.read())
|
||||
except:
|
||||
pass
|
||||
return (response, e)
|
||||
|
||||
try:
|
||||
response = json.loads(resp_body)
|
||||
except ValueError, e:
|
||||
return (None, e)
|
||||
|
||||
return (response, None)
|
||||
|
||||
def GetAccessToken(self, refresh_token):
|
||||
"""Given a RefreshToken, obtains a corresponding access token.
|
||||
|
||||
First, this client's access token cache is checked for an existing,
|
||||
not-yet-expired access token for the provided refresh token. If none is
|
||||
found, the client obtains a fresh access token for the provided refresh
|
||||
token from the OAuth2 provider's token endpoint.
|
||||
|
||||
Args:
|
||||
refresh_token: The RefreshToken object which to get an access token for.
|
||||
Returns:
|
||||
The cached or freshly obtained AccessToken.
|
||||
Raises:
|
||||
AccessTokenRefreshError if an error occurs.
|
||||
"""
|
||||
# Ensure only one thread at a time attempts to get (and possibly refresh)
|
||||
# the access token. This doesn't prevent concurrent refresh attempts across
|
||||
# multiple gsutil instances, but at least protects against multiple threads
|
||||
# simultaneously attempting to refresh when gsutil -m is used.
|
||||
token_exchange_lock.acquire()
|
||||
try:
|
||||
cache_key = refresh_token.CacheKey()
|
||||
LOG.info('GetAccessToken: checking cache for key %s', cache_key)
|
||||
access_token = self.access_token_cache.GetToken(cache_key)
|
||||
LOG.debug('GetAccessToken: token from cache: %s', access_token)
|
||||
if access_token is None or access_token.ShouldRefresh():
|
||||
LOG.info('GetAccessToken: fetching fresh access token...')
|
||||
access_token = self.FetchAccessToken(refresh_token)
|
||||
LOG.debug('GetAccessToken: fresh access token: %s', access_token)
|
||||
self.access_token_cache.PutToken(cache_key, access_token)
|
||||
return access_token
|
||||
finally:
|
||||
token_exchange_lock.release()
|
||||
|
||||
def FetchAccessToken(self, refresh_token):
|
||||
"""Fetches an access token from the provider's token endpoint.
|
||||
|
||||
Given a RefreshToken, fetches an access token from this client's OAuth2
|
||||
provider's token endpoint.
|
||||
|
||||
Args:
|
||||
refresh_token: The RefreshToken object which to get an access token for.
|
||||
Returns:
|
||||
The fetched AccessToken.
|
||||
Raises:
|
||||
AccessTokenRefreshError: if an error occurs.
|
||||
"""
|
||||
request = {
|
||||
'grant_type': 'refresh_token',
|
||||
'client_id': self.client_id,
|
||||
'client_secret': self.client_secret,
|
||||
'refresh_token': refresh_token.refresh_token,
|
||||
}
|
||||
LOG.debug('FetchAccessToken request: %s', request)
|
||||
|
||||
response, error = self._TokenRequest(request)
|
||||
LOG.debug(
|
||||
'FetchAccessToken response (error = %s): %s', error, response)
|
||||
|
||||
if error:
|
||||
oauth2_error = ''
|
||||
if response and response['error']:
|
||||
oauth2_error = '; OAuth2 error: %s' % response['error']
|
||||
raise AccessTokenRefreshError(
|
||||
'Failed to exchange refresh token into access token; '
|
||||
'request failed: %s%s' % (error, oauth2_error))
|
||||
|
||||
if 'access_token' not in response:
|
||||
raise AccessTokenRefreshError(
|
||||
'Failed to exchange refresh token into access token; response: %s' %
|
||||
response)
|
||||
|
||||
token_expiry = None
|
||||
if 'expires_in' in response:
|
||||
token_expiry = (
|
||||
self.datetime_strategy.utcnow() +
|
||||
datetime.timedelta(seconds=int(response['expires_in'])))
|
||||
|
||||
return AccessToken(response['access_token'], token_expiry,
|
||||
datetime_strategy=self.datetime_strategy)
|
||||
|
||||
def GetAuthorizationUri(self, redirect_uri, scopes, extra_params=None):
|
||||
"""Gets the OAuth2 authorization URI and the specified scope(s).
|
||||
|
||||
Applications should navigate/redirect the user's user agent to this URI. The
|
||||
user will be shown an approval UI requesting the user to approve access of
|
||||
this client to the requested scopes under the identity of the authenticated
|
||||
end user.
|
||||
|
||||
The application should expect the user agent to be redirected to the
|
||||
specified redirect_uri after the user's approval/disapproval.
|
||||
|
||||
Installed applications may use the special redirect_uri
|
||||
'urn:ietf:wg:oauth:2.0:oob' to indicate that instead of redirecting the
|
||||
browser, the user be shown a confirmation page with a verification code.
|
||||
The application should query the user for this code.
|
||||
|
||||
Args:
|
||||
redirect_uri: Either the string 'urn:ietf:wg:oauth:2.0:oob' for a
|
||||
non-web-based application, or a URI that handles the callback from the
|
||||
authorization server.
|
||||
scopes: A list of strings specifying the OAuth scopes the application
|
||||
requests access to.
|
||||
extra_params: Optional dictionary of additional parameters to be passed to
|
||||
the OAuth2 authorization URI.
|
||||
Returns:
|
||||
The authorization URI for the specified scopes as a string.
|
||||
"""
|
||||
|
||||
request = {
|
||||
'response_type': 'code',
|
||||
'client_id': self.client_id,
|
||||
'redirect_uri': redirect_uri,
|
||||
'scope': ' '.join(scopes),
|
||||
}
|
||||
|
||||
if extra_params:
|
||||
request.update(extra_params)
|
||||
url_parts = list(urlparse.urlparse(self.provider.authorization_uri))
|
||||
# 4 is the index of the query part
|
||||
request.update(dict(cgi.parse_qsl(url_parts[4])))
|
||||
url_parts[4] = urllib.urlencode(request)
|
||||
return urlparse.urlunparse(url_parts)
|
||||
|
||||
def ExchangeAuthorizationCode(self, code, redirect_uri, scopes):
|
||||
"""Exchanges an authorization code for a refresh token.
|
||||
|
||||
Invokes this client's OAuth2 provider's token endpoint to exchange an
|
||||
authorization code into a refresh token.
|
||||
|
||||
Args:
|
||||
code: the authrorization code.
|
||||
redirect_uri: Either the string 'urn:ietf:wg:oauth:2.0:oob' for a
|
||||
non-web-based application, or a URI that handles the callback from the
|
||||
authorization server.
|
||||
scopes: A list of strings specifying the OAuth scopes the application
|
||||
requests access to.
|
||||
Returns:
|
||||
A tuple consting of the resulting RefreshToken and AccessToken.
|
||||
Raises:
|
||||
AuthorizationCodeExchangeError: if an error occurs.
|
||||
"""
|
||||
request = {
|
||||
'grant_type': 'authorization_code',
|
||||
'client_id': self.client_id,
|
||||
'client_secret': self.client_secret,
|
||||
'code': code,
|
||||
'redirect_uri': redirect_uri,
|
||||
'scope': ' '.join(scopes),
|
||||
}
|
||||
LOG.debug('ExchangeAuthorizationCode request: %s', request)
|
||||
|
||||
response, error = self._TokenRequest(request)
|
||||
LOG.debug(
|
||||
'ExchangeAuthorizationCode response (error = %s): %s',
|
||||
error, response)
|
||||
|
||||
if error:
|
||||
oauth2_error = ''
|
||||
if response and response['error']:
|
||||
oauth2_error = '; OAuth2 error: %s' % response['error']
|
||||
raise AuthorizationCodeExchangeError(
|
||||
'Failed to exchange refresh token into access token; '
|
||||
'request failed: %s%s' % (str(error), oauth2_error))
|
||||
|
||||
if not 'access_token' in response:
|
||||
raise AuthorizationCodeExchangeError(
|
||||
'Failed to exchange authorization code into access token; '
|
||||
'response: %s' % response)
|
||||
|
||||
token_expiry = None
|
||||
if 'expires_in' in response:
|
||||
token_expiry = (
|
||||
self.datetime_strategy.utcnow() +
|
||||
datetime.timedelta(seconds=int(response['expires_in'])))
|
||||
|
||||
access_token = AccessToken(response['access_token'], token_expiry,
|
||||
datetime_strategy=self.datetime_strategy)
|
||||
|
||||
refresh_token = None
|
||||
refresh_token_string = response.get('refresh_token', None)
|
||||
|
||||
token_exchange_lock.acquire()
|
||||
try:
|
||||
if refresh_token_string:
|
||||
refresh_token = RefreshToken(self, refresh_token_string)
|
||||
self.access_token_cache.PutToken(refresh_token.CacheKey(), access_token)
|
||||
finally:
|
||||
token_exchange_lock.release()
|
||||
|
||||
return (refresh_token, access_token)
|
||||
|
||||
|
||||
class AccessToken(object):
|
||||
"""Encapsulates an OAuth2 access token."""
|
||||
|
||||
def __init__(self, token, expiry, datetime_strategy=datetime.datetime):
|
||||
self.token = token
|
||||
self.expiry = expiry
|
||||
self.datetime_strategy = datetime_strategy
|
||||
|
||||
@staticmethod
|
||||
def UnSerialize(query):
|
||||
"""Creates an AccessToken object from its serialized form."""
|
||||
|
||||
def GetValue(d, key):
|
||||
return (d.get(key, [None]))[0]
|
||||
kv = cgi.parse_qs(query)
|
||||
if not kv['token']:
|
||||
return None
|
||||
expiry = None
|
||||
expiry_tuple = GetValue(kv, 'expiry')
|
||||
if expiry_tuple:
|
||||
try:
|
||||
expiry = datetime.datetime(
|
||||
*[int(n) for n in expiry_tuple.split(',')])
|
||||
except:
|
||||
return None
|
||||
return AccessToken(GetValue(kv, 'token'), expiry)
|
||||
|
||||
def Serialize(self):
|
||||
"""Serializes this object as URI-encoded key-value pairs."""
|
||||
# There's got to be a better way to serialize a datetime. Unfortunately,
|
||||
# there is no reliable way to convert into a unix epoch.
|
||||
kv = {'token': self.token}
|
||||
if self.expiry:
|
||||
t = self.expiry
|
||||
tupl = (t.year, t.month, t.day, t.hour, t.minute, t.second, t.microsecond)
|
||||
kv['expiry'] = ','.join([str(i) for i in tupl])
|
||||
return urllib.urlencode(kv)
|
||||
|
||||
def ShouldRefresh(self, time_delta=300):
|
||||
"""Whether the access token needs to be refreshed.
|
||||
|
||||
Args:
|
||||
time_delta: refresh access token when it expires within time_delta secs.
|
||||
|
||||
Returns:
|
||||
True if the token is expired or about to expire, False if the
|
||||
token should be expected to work. Note that the token may still
|
||||
be rejected, e.g. if it has been revoked server-side.
|
||||
"""
|
||||
if self.expiry is None:
|
||||
return False
|
||||
return (self.datetime_strategy.utcnow()
|
||||
+ datetime.timedelta(seconds=time_delta) > self.expiry)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.token == other.token and self.expiry == other.expiry
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __str__(self):
|
||||
return 'AccessToken(token=%s, expiry=%sZ)' % (self.token, self.expiry)
|
||||
|
||||
|
||||
class RefreshToken(object):
|
||||
"""Encapsulates an OAuth2 refresh token."""
|
||||
|
||||
def __init__(self, oauth2_client, refresh_token):
|
||||
self.oauth2_client = oauth2_client
|
||||
self.refresh_token = refresh_token
|
||||
|
||||
def CacheKey(self):
|
||||
"""Computes a cache key for this refresh token.
|
||||
|
||||
The cache key is computed as the SHA1 hash of the token, and as such
|
||||
satisfies the FileSystemTokenCache requirement that cache keys do not leak
|
||||
information about token values.
|
||||
|
||||
Returns:
|
||||
A hash key for this refresh token.
|
||||
"""
|
||||
h = sha1()
|
||||
h.update(self.refresh_token)
|
||||
return h.hexdigest()
|
||||
|
||||
def GetAuthorizationHeader(self):
|
||||
"""Gets the access token HTTP authorication header value.
|
||||
|
||||
Returns:
|
||||
The value of an Authorization HTTP header that authenticates
|
||||
requests with an OAuth2 access token based on this refresh token.
|
||||
"""
|
||||
return 'Bearer %s' % self.oauth2_client.GetAccessToken(self).token
|
||||
@ -0,0 +1,374 @@
|
||||
# Copyright 2010 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for oauth2_client."""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import urllib2
|
||||
import urlparse
|
||||
from stat import S_IMODE
|
||||
from StringIO import StringIO
|
||||
|
||||
test_bin_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
|
||||
|
||||
lib_dir = os.path.join(test_bin_dir, '..')
|
||||
sys.path.insert(0, lib_dir)
|
||||
|
||||
# Needed for boto.cacerts
|
||||
boto_lib_dir = os.path.join(test_bin_dir, '..', 'boto')
|
||||
sys.path.insert(0, boto_lib_dir)
|
||||
|
||||
import oauth2_client
|
||||
|
||||
LOG = logging.getLogger('oauth2_client_test')
|
||||
|
||||
class MockOpener:
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.open_error = None
|
||||
self.open_result = None
|
||||
self.open_capture_url = None
|
||||
self.open_capture_data = None
|
||||
|
||||
def open(self, req, data=None):
|
||||
self.open_capture_url = req.get_full_url()
|
||||
self.open_capture_data = req.get_data()
|
||||
if self.open_error is not None:
|
||||
raise self.open_error
|
||||
else:
|
||||
return StringIO(self.open_result)
|
||||
|
||||
|
||||
class MockDateTime:
|
||||
def __init__(self):
|
||||
self.mock_now = None
|
||||
|
||||
def utcnow(self):
|
||||
return self.mock_now
|
||||
|
||||
|
||||
class OAuth2ClientTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.opener = MockOpener()
|
||||
self.mock_datetime = MockDateTime()
|
||||
self.start_time = datetime.datetime(2011, 3, 1, 10, 25, 13, 300826)
|
||||
self.mock_datetime.mock_now = self.start_time
|
||||
self.client = oauth2_client.OAuth2Client(
|
||||
oauth2_client.OAuth2Provider(
|
||||
'Sample OAuth Provider',
|
||||
'https://provider.example.com/oauth/provider?mode=authorize',
|
||||
'https://provider.example.com/oauth/provider?mode=token'),
|
||||
'clid', 'clsecret',
|
||||
url_opener=self.opener, datetime_strategy=self.mock_datetime)
|
||||
|
||||
def testFetchAccessToken(self):
|
||||
refresh_token = '1/ZaBrxdPl77Bi4jbsO7x-NmATiaQZnWPB51nTvo8n9Sw'
|
||||
access_token = '1/aalskfja-asjwerwj'
|
||||
self.opener.open_result = (
|
||||
'{"access_token":"%s","expires_in":3600}' % access_token)
|
||||
cred = oauth2_client.RefreshToken(self.client, refresh_token)
|
||||
token = self.client.FetchAccessToken(cred)
|
||||
|
||||
self.assertEquals(
|
||||
self.opener.open_capture_url,
|
||||
'https://provider.example.com/oauth/provider?mode=token')
|
||||
self.assertEquals({
|
||||
'grant_type': ['refresh_token'],
|
||||
'client_id': ['clid'],
|
||||
'client_secret': ['clsecret'],
|
||||
'refresh_token': [refresh_token]},
|
||||
urlparse.parse_qs(self.opener.open_capture_data, keep_blank_values=True,
|
||||
strict_parsing=True))
|
||||
self.assertEquals(access_token, token.token)
|
||||
self.assertEquals(
|
||||
datetime.datetime(2011, 3, 1, 11, 25, 13, 300826),
|
||||
token.expiry)
|
||||
|
||||
def testFetchAccessTokenFailsForBadJsonResponse(self):
|
||||
self.opener.open_result = 'blah'
|
||||
cred = oauth2_client.RefreshToken(self.client, 'abc123')
|
||||
self.assertRaises(
|
||||
oauth2_client.AccessTokenRefreshError, self.client.FetchAccessToken, cred)
|
||||
|
||||
def testFetchAccessTokenFailsForErrorResponse(self):
|
||||
self.opener.open_error = urllib2.HTTPError(
|
||||
None, 400, 'Bad Request', None, StringIO('{"error": "invalid token"}'))
|
||||
cred = oauth2_client.RefreshToken(self.client, 'abc123')
|
||||
self.assertRaises(
|
||||
oauth2_client.AccessTokenRefreshError, self.client.FetchAccessToken, cred)
|
||||
|
||||
def testFetchAccessTokenFailsForHttpError(self):
|
||||
self.opener.open_result = urllib2.HTTPError(
|
||||
'foo', 400, 'Bad Request', None, None)
|
||||
cred = oauth2_client.RefreshToken(self.client, 'abc123')
|
||||
self.assertRaises(
|
||||
oauth2_client.AccessTokenRefreshError, self.client.FetchAccessToken, cred)
|
||||
|
||||
def testGetAccessToken(self):
|
||||
refresh_token = 'ref_token'
|
||||
access_token_1 = 'abc123'
|
||||
self.opener.open_result = (
|
||||
'{"access_token":"%s",' '"expires_in":3600}' % access_token_1)
|
||||
cred = oauth2_client.RefreshToken(self.client, refresh_token)
|
||||
|
||||
token_1 = self.client.GetAccessToken(cred)
|
||||
|
||||
# There's no access token in the cache; verify that we fetched a fresh
|
||||
# token.
|
||||
self.assertEquals({
|
||||
'grant_type': ['refresh_token'],
|
||||
'client_id': ['clid'],
|
||||
'client_secret': ['clsecret'],
|
||||
'refresh_token': [refresh_token]},
|
||||
urlparse.parse_qs(self.opener.open_capture_data, keep_blank_values=True,
|
||||
strict_parsing=True))
|
||||
self.assertEquals(access_token_1, token_1.token)
|
||||
self.assertEquals(self.start_time + datetime.timedelta(minutes=60),
|
||||
token_1.expiry)
|
||||
|
||||
# Advance time by less than expiry time, and fetch another token.
|
||||
self.opener.reset()
|
||||
self.mock_datetime.mock_now = (
|
||||
self.start_time + datetime.timedelta(minutes=55))
|
||||
token_2 = self.client.GetAccessToken(cred)
|
||||
|
||||
# Since the access token wasn't expired, we get the cache token, and there
|
||||
# was no refresh request.
|
||||
self.assertEquals(token_1, token_2)
|
||||
self.assertEquals(access_token_1, token_2.token)
|
||||
self.assertEquals(None, self.opener.open_capture_url)
|
||||
self.assertEquals(None, self.opener.open_capture_data)
|
||||
|
||||
# Advance time past expiry time, and fetch another token.
|
||||
self.opener.reset()
|
||||
self.mock_datetime.mock_now = (
|
||||
self.start_time + datetime.timedelta(minutes=55, seconds=1))
|
||||
access_token_2 = 'zyx456'
|
||||
self.opener.open_result = (
|
||||
'{"access_token":"%s",' '"expires_in":3600}' % access_token_2)
|
||||
token_3 = self.client.GetAccessToken(cred)
|
||||
|
||||
# This should have resulted in a refresh request and a fresh access token.
|
||||
self.assertEquals({
|
||||
'grant_type': ['refresh_token'],
|
||||
'client_id': ['clid'],
|
||||
'client_secret': ['clsecret'],
|
||||
'refresh_token': [refresh_token]},
|
||||
urlparse.parse_qs(self.opener.open_capture_data, keep_blank_values=True,
|
||||
strict_parsing=True))
|
||||
self.assertEquals(access_token_2, token_3.token)
|
||||
self.assertEquals(self.mock_datetime.mock_now + datetime.timedelta(minutes=60),
|
||||
token_3.expiry)
|
||||
|
||||
def testGetAuthorizationUri(self):
|
||||
authn_uri = self.client.GetAuthorizationUri(
|
||||
'https://www.example.com/oauth/redir?mode=approve%20me',
|
||||
('scope_foo', 'scope_bar'),
|
||||
{'state': 'this and that & sundry'})
|
||||
|
||||
uri_parts = urlparse.urlsplit(authn_uri)
|
||||
self.assertEquals(('https', 'provider.example.com', '/oauth/provider'),
|
||||
uri_parts[:3])
|
||||
|
||||
self.assertEquals({
|
||||
'response_type': ['code'],
|
||||
'client_id': ['clid'],
|
||||
'redirect_uri':
|
||||
['https://www.example.com/oauth/redir?mode=approve%20me'],
|
||||
'scope': ['scope_foo scope_bar'],
|
||||
'state': ['this and that & sundry'],
|
||||
'mode': ['authorize']},
|
||||
urlparse.parse_qs(uri_parts[3]))
|
||||
|
||||
def testExchangeAuthorizationCode(self):
|
||||
code = 'codeABQ1234'
|
||||
exp_refresh_token = 'ref_token42'
|
||||
exp_access_token = 'access_tokenXY123'
|
||||
self.opener.open_result = (
|
||||
'{"access_token":"%s","expires_in":3600,"refresh_token":"%s"}'
|
||||
% (exp_access_token, exp_refresh_token))
|
||||
|
||||
refresh_token, access_token = self.client.ExchangeAuthorizationCode(
|
||||
code, 'urn:ietf:wg:oauth:2.0:oob', ('scope1', 'scope2'))
|
||||
|
||||
self.assertEquals({
|
||||
'grant_type': ['authorization_code'],
|
||||
'client_id': ['clid'],
|
||||
'client_secret': ['clsecret'],
|
||||
'code': [code],
|
||||
'redirect_uri': ['urn:ietf:wg:oauth:2.0:oob'],
|
||||
'scope': ['scope1 scope2'] },
|
||||
urlparse.parse_qs(self.opener.open_capture_data, keep_blank_values=True,
|
||||
strict_parsing=True))
|
||||
self.assertEquals(exp_access_token, access_token.token)
|
||||
self.assertEquals(self.start_time + datetime.timedelta(minutes=60),
|
||||
access_token.expiry)
|
||||
|
||||
self.assertEquals(self.client, refresh_token.oauth2_client)
|
||||
self.assertEquals(exp_refresh_token, refresh_token.refresh_token)
|
||||
|
||||
# Check that the access token was put in the cache.
|
||||
cached_token = self.client.access_token_cache.GetToken(
|
||||
refresh_token.CacheKey())
|
||||
self.assertEquals(access_token, cached_token)
|
||||
|
||||
|
||||
class AccessTokenTest(unittest.TestCase):
|
||||
|
||||
def testShouldRefresh(self):
|
||||
mock_datetime = MockDateTime()
|
||||
start = datetime.datetime(2011, 3, 1, 11, 25, 13, 300826)
|
||||
expiry = start + datetime.timedelta(minutes=60)
|
||||
token = oauth2_client.AccessToken(
|
||||
'foo', expiry, datetime_strategy=mock_datetime)
|
||||
|
||||
mock_datetime.mock_now = start
|
||||
self.assertFalse(token.ShouldRefresh())
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(minutes=54)
|
||||
self.assertFalse(token.ShouldRefresh())
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(minutes=55)
|
||||
self.assertFalse(token.ShouldRefresh())
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(
|
||||
minutes=55, seconds=1)
|
||||
self.assertTrue(token.ShouldRefresh())
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(
|
||||
minutes=61)
|
||||
self.assertTrue(token.ShouldRefresh())
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(minutes=58)
|
||||
self.assertFalse(token.ShouldRefresh(time_delta=120))
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(
|
||||
minutes=58, seconds=1)
|
||||
self.assertTrue(token.ShouldRefresh(time_delta=120))
|
||||
|
||||
def testShouldRefreshNoExpiry(self):
|
||||
mock_datetime = MockDateTime()
|
||||
start = datetime.datetime(2011, 3, 1, 11, 25, 13, 300826)
|
||||
token = oauth2_client.AccessToken(
|
||||
'foo', None, datetime_strategy=mock_datetime)
|
||||
|
||||
mock_datetime.mock_now = start
|
||||
self.assertFalse(token.ShouldRefresh())
|
||||
|
||||
mock_datetime.mock_now = start + datetime.timedelta(
|
||||
minutes=472)
|
||||
self.assertFalse(token.ShouldRefresh())
|
||||
|
||||
def testSerialization(self):
|
||||
expiry = datetime.datetime(2011, 3, 1, 11, 25, 13, 300826)
|
||||
token = oauth2_client.AccessToken('foo', expiry)
|
||||
serialized_token = token.Serialize()
|
||||
LOG.debug('testSerialization: serialized_token=%s' % serialized_token)
|
||||
|
||||
token2 = oauth2_client.AccessToken.UnSerialize(serialized_token)
|
||||
self.assertEquals(token, token2)
|
||||
|
||||
|
||||
class RefreshTokenTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.opener = MockOpener()
|
||||
self.mock_datetime = MockDateTime()
|
||||
self.start_time = datetime.datetime(2011, 3, 1, 10, 25, 13, 300826)
|
||||
self.mock_datetime.mock_now = self.start_time
|
||||
self.client = oauth2_client.OAuth2Client(
|
||||
oauth2_client.OAuth2Provider(
|
||||
'Sample OAuth Provider',
|
||||
'https://provider.example.com/oauth/provider?mode=authorize',
|
||||
'https://provider.example.com/oauth/provider?mode=token'),
|
||||
'clid', 'clsecret',
|
||||
url_opener=self.opener, datetime_strategy=self.mock_datetime)
|
||||
|
||||
self.cred = oauth2_client.RefreshToken(self.client, 'ref_token_abc123')
|
||||
|
||||
def testUniqeId(self):
|
||||
cred_id = self.cred.CacheKey()
|
||||
self.assertEquals('0720afed6871f12761fbea3271f451e6ba184bf5', cred_id)
|
||||
|
||||
def testGetAuthorizationHeader(self):
|
||||
access_token = 'access_123'
|
||||
self.opener.open_result = (
|
||||
'{"access_token":"%s","expires_in":3600}' % access_token)
|
||||
|
||||
self.assertEquals('Bearer %s' % access_token,
|
||||
self.cred.GetAuthorizationHeader())
|
||||
|
||||
|
||||
class FileSystemTokenCacheTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.cache = oauth2_client.FileSystemTokenCache()
|
||||
self.start_time = datetime.datetime(2011, 3, 1, 10, 25, 13, 300826)
|
||||
self.token_1 = oauth2_client.AccessToken('token1', self.start_time)
|
||||
self.token_2 = oauth2_client.AccessToken(
|
||||
'token2', self.start_time + datetime.timedelta(seconds=492))
|
||||
self.key = 'token1key'
|
||||
|
||||
def tearDown(self):
|
||||
try:
|
||||
os.unlink(self.cache.CacheFileName(self.key))
|
||||
except:
|
||||
pass
|
||||
|
||||
def testPut(self):
|
||||
self.cache.PutToken(self.key, self.token_1)
|
||||
# Assert that the cache file exists and has correct permissions.
|
||||
self.assertEquals(
|
||||
0600, S_IMODE(os.stat(self.cache.CacheFileName(self.key)).st_mode))
|
||||
|
||||
def testPutGet(self):
|
||||
# No cache file present.
|
||||
self.assertEquals(None, self.cache.GetToken(self.key))
|
||||
|
||||
# Put a token
|
||||
self.cache.PutToken(self.key, self.token_1)
|
||||
cached_token = self.cache.GetToken(self.key)
|
||||
self.assertEquals(self.token_1, cached_token)
|
||||
|
||||
# Put a different token
|
||||
self.cache.PutToken(self.key, self.token_2)
|
||||
cached_token = self.cache.GetToken(self.key)
|
||||
self.assertEquals(self.token_2, cached_token)
|
||||
|
||||
def testGetBadFile(self):
|
||||
f = open(self.cache.CacheFileName(self.key), 'w')
|
||||
f.write('blah')
|
||||
f.close()
|
||||
self.assertEquals(None, self.cache.GetToken(self.key))
|
||||
|
||||
def testCacheFileName(self):
|
||||
cache = oauth2_client.FileSystemTokenCache(
|
||||
path_pattern='/var/run/ccache/token.%(uid)s.%(key)s')
|
||||
self.assertEquals('/var/run/ccache/token.%d.abc123' % os.getuid(),
|
||||
cache.CacheFileName('abc123'))
|
||||
|
||||
cache = oauth2_client.FileSystemTokenCache(
|
||||
path_pattern='/var/run/ccache/token.%(key)s')
|
||||
self.assertEquals('/var/run/ccache/token.abc123',
|
||||
cache.CacheFileName('abc123'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
unittest.main()
|
||||
@ -0,0 +1,110 @@
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Helper routines to facilitate use of oauth2_client in gsutil."""
|
||||
|
||||
import sys
|
||||
import time
|
||||
import webbrowser
|
||||
|
||||
import oauth2_client
|
||||
|
||||
GSUTIL_CLIENT_ID = '909320924072.apps.googleusercontent.com'
|
||||
# Google OAuth2 clients always have a secret, even if the client is an installed
|
||||
# application/utility such as gsutil. Of course, in such cases the "secret" is
|
||||
# actually publicly known; security depends entirly on the secrecy of refresh
|
||||
# tokens, which effectively become bearer tokens.
|
||||
GSUTIL_CLIENT_NOTSOSECRET = 'p3RlpR10xMFh9ZXBS/ZNLYUu'
|
||||
|
||||
GOOGLE_OAUTH2_PROVIDER_LABEL = 'Google'
|
||||
GOOGLE_OAUTH2_PROVIDER_AUTHORIZATION_URI = (
|
||||
'https://accounts.google.com/o/oauth2/auth')
|
||||
GOOGLE_OAUTH2_PROVIDER_TOKEN_URI = (
|
||||
'https://accounts.google.com/o/oauth2/token')
|
||||
|
||||
OOB_REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
|
||||
|
||||
def OAuth2ClientFromBotoConfig(config):
|
||||
token_cache = None
|
||||
token_cache_type = config.get('OAuth2', 'token_cache', 'file_system')
|
||||
|
||||
if token_cache_type == 'file_system':
|
||||
if config.has_option('OAuth2', 'token_cache_path_pattern'):
|
||||
token_cache = oauth2_client.FileSystemTokenCache(
|
||||
path_pattern=config.get('OAuth2', 'token_cache_path_pattern'))
|
||||
else:
|
||||
token_cache = oauth2_client.FileSystemTokenCache()
|
||||
elif token_cache_type == 'in_memory':
|
||||
token_cache = oauth2_client.InMemoryTokenCache()
|
||||
else:
|
||||
raise Exception(
|
||||
"Invalid value for config option OAuth2/token_cache: %s" %
|
||||
token_cache_type)
|
||||
|
||||
proxy = None
|
||||
if (config.has_option('Boto', 'proxy')
|
||||
and config.has_option('Boto', 'proxy_port')):
|
||||
proxy = "%s:%s" % (config.get('Boto', 'proxy'),
|
||||
config.get('Boto', 'proxy_port'))
|
||||
|
||||
provider_label = config.get(
|
||||
'OAuth2', 'provider_label', GOOGLE_OAUTH2_PROVIDER_LABEL)
|
||||
provider_authorization_uri = config.get(
|
||||
'OAuth2', 'provider_authorization_uri',
|
||||
GOOGLE_OAUTH2_PROVIDER_AUTHORIZATION_URI)
|
||||
provider_token_uri = config.get(
|
||||
'OAuth2', 'provider_token_uri', GOOGLE_OAUTH2_PROVIDER_TOKEN_URI)
|
||||
|
||||
client_id = config.get('OAuth2', 'client_id', GSUTIL_CLIENT_ID)
|
||||
client_secret = config.get(
|
||||
'OAuth2', 'client_secret', GSUTIL_CLIENT_NOTSOSECRET)
|
||||
|
||||
return oauth2_client.OAuth2Client(
|
||||
oauth2_client.OAuth2Provider(
|
||||
provider_label, provider_authorization_uri, provider_token_uri),
|
||||
client_id, client_secret,
|
||||
proxy=proxy, access_token_cache=token_cache)
|
||||
|
||||
def OAuth2ApprovalFlow(oauth2_client, scopes, launch_browser=False):
|
||||
approval_url = oauth2_client.GetAuthorizationUri(OOB_REDIRECT_URI, scopes)
|
||||
if launch_browser:
|
||||
sys.stdout.write(
|
||||
'Attempting to launch a browser with the OAuth2 approval dialog at '
|
||||
'URL: %s\n\n'
|
||||
'[Note: due to a Python bug, you may see a spurious error message "object is not\n'
|
||||
'callable [...] in [...] Popen.__del__" which can be ignored.]\n\n' % approval_url)
|
||||
else:
|
||||
sys.stdout.write(
|
||||
'Please navigate your browser to the following URL:\n%s\n' %
|
||||
approval_url)
|
||||
|
||||
sys.stdout.write(
|
||||
'In your browser you should see a page that requests you to authorize '
|
||||
'gsutil to access\nGoogle Cloud Storage on your behalf. After you '
|
||||
'approve, an authorization code will be displayed.\n\n')
|
||||
if (launch_browser and
|
||||
not webbrowser.open(approval_url, new=1, autoraise=True)):
|
||||
sys.stdout.write(
|
||||
'Launching browser appears to have failed; please navigate a browser '
|
||||
'to the following URL:\n%s\n' % approval_url)
|
||||
# Short delay; webbrowser.open on linux insists on printing out a message
|
||||
# which we don't want to run into the prompt for the auth code.
|
||||
time.sleep(2)
|
||||
code = raw_input('Enter the authorization code: ')
|
||||
|
||||
refresh_token, access_token = oauth2_client.ExchangeAuthorizationCode(
|
||||
code, OOB_REDIRECT_URI, scopes)
|
||||
|
||||
return refresh_token
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
from boto.auth_handler import AuthHandler
|
||||
from boto.auth_handler import NotReadyToAuthenticate
|
||||
import oauth2_client
|
||||
import oauth2_helper
|
||||
|
||||
class OAuth2Auth(AuthHandler):
|
||||
|
||||
capability = ['google-oauth2', 's3']
|
||||
|
||||
def __init__(self, path, config, provider):
|
||||
if (provider.name == 'google'
|
||||
and config.has_option('Credentials', 'gs_oauth2_refresh_token')):
|
||||
|
||||
self.oauth2_client = oauth2_helper.OAuth2ClientFromBotoConfig(config)
|
||||
|
||||
self.refresh_token = oauth2_client.RefreshToken(
|
||||
self.oauth2_client,
|
||||
config.get('Credentials', 'gs_oauth2_refresh_token'))
|
||||
else:
|
||||
raise NotReadyToAuthenticate()
|
||||
|
||||
def add_auth(self, http_request):
|
||||
http_request.headers['Authorization'] = \
|
||||
self.refresh_token.GetAuthorizationHeader()
|
||||
@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Utilities to facilitate maintaining one master list of package contents
|
||||
# in MANIFEST.in and allow us to import that list into various packaging
|
||||
# tools (e.g. rpmbuid and setup.py).
|
||||
|
||||
# Define the file in which we maintain package contents. Rather than
|
||||
# hard-coding our package contents, to ease maintenance we read the
|
||||
# manifest file to obtain the list of files and directories to include.
|
||||
MANIFEST_IN = 'MANIFEST.in'
|
||||
|
||||
# Define input and output files for customizing the rpm package spec.
|
||||
SPEC_IN = 'gsutil.spec.in'
|
||||
SPEC_OUT = 'gsutil.spec'
|
||||
|
||||
# Root of rpmbuild tree for file enumeration in gsutil.spec file.
|
||||
RPM_ROOT = '%{_datadir}/%{name}/'
|
||||
|
||||
def parse_manifest(files, dirs):
|
||||
'''Parse contents of manifest file and append results to passed lists
|
||||
of files and directories.
|
||||
'''
|
||||
f = open(MANIFEST_IN, 'r')
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
# Skip empty or comment lines.
|
||||
if (len(line) <= 0) or (line[0] == '#'):
|
||||
continue
|
||||
tokens = line.split()
|
||||
if len(tokens) >= 0:
|
||||
if tokens[0] == 'include':
|
||||
files.extend(tokens[1:])
|
||||
elif tokens[0] == 'recursive-include' and tokens[2] == '*':
|
||||
dirs.append(tokens[1])
|
||||
else:
|
||||
err = 'Unsupported type ' + tokens[0] + ' in ' + MANIFEST_IN + ' file.'
|
||||
raise Exception(err)
|
||||
f.close()
|
||||
|
||||
# When executed as a separate script, create a dynamically generated rpm
|
||||
# spec file. Otherwise, when loaded as a module by another script, no
|
||||
# specific actions are taken, other than making utility functions available
|
||||
# to the loading script.
|
||||
if __name__ == '__main__':
|
||||
# Running as main so generate a new rpm spec file.
|
||||
files = []
|
||||
dirs = []
|
||||
parse_manifest(files, dirs)
|
||||
fin = open(SPEC_IN, 'r')
|
||||
fout = open(SPEC_OUT, 'w')
|
||||
for line in fin:
|
||||
if line.strip() == '###FILES_GO_HERE###':
|
||||
for file in files:
|
||||
fout.write(RPM_ROOT + file + '\n')
|
||||
for dir in dirs:
|
||||
fout.write(RPM_ROOT + dir + '/\n')
|
||||
else:
|
||||
fout.write(line)
|
||||
fout.close()
|
||||
fin.close()
|
||||
@ -0,0 +1,30 @@
|
||||
Copyright (c) 2013, SaltyCrane
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
* Neither the name of the SaltyCrane nor the names of its
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@ -0,0 +1,45 @@
|
||||
import time
|
||||
from functools import wraps
|
||||
|
||||
|
||||
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
|
||||
"""Retry calling the decorated function using an exponential backoff.
|
||||
|
||||
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
|
||||
original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
|
||||
|
||||
:param ExceptionToCheck: the exception to check. may be a tuple of
|
||||
exceptions to check
|
||||
:type ExceptionToCheck: Exception or tuple
|
||||
:param tries: number of times to try (not retry) before giving up
|
||||
:type tries: int
|
||||
:param delay: initial delay between retries in seconds
|
||||
:type delay: int
|
||||
:param backoff: backoff multiplier e.g. value of 2 will double the delay
|
||||
each retry
|
||||
:type backoff: int
|
||||
:param logger: logger to use. If None, print
|
||||
:type logger: logging.Logger instance
|
||||
"""
|
||||
def deco_retry(f):
|
||||
|
||||
@wraps(f)
|
||||
def f_retry(*args, **kwargs):
|
||||
mtries, mdelay = tries, delay
|
||||
while mtries > 1:
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
except ExceptionToCheck, e:
|
||||
msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
|
||||
if logger:
|
||||
logger.warning(msg)
|
||||
else:
|
||||
print msg
|
||||
time.sleep(mdelay)
|
||||
mtries -= 1
|
||||
mdelay *= backoff
|
||||
return f(*args, **kwargs)
|
||||
|
||||
return f_retry # true decorator
|
||||
|
||||
return deco_retry
|
||||
Loading…
Reference in New Issue