You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1082 lines
43 KiB
Python
1082 lines
43 KiB
Python
# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
|
|
# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.
|
|
# Copyright (c) 2010 Google
|
|
# Copyright (c) 2008 rPath, Inc.
|
|
# Copyright (c) 2009 The Echo Nest Corporation
|
|
# Copyright (c) 2010, Eucalyptus Systems, Inc.
|
|
# Copyright (c) 2011, Nexenta Systems Inc.
|
|
# All rights reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
# copy of this software and associated documentation files (the
|
|
# "Software"), to deal in the Software without restriction, including
|
|
# without limitation the rights to use, copy, modify, merge, publish, dis-
|
|
# tribute, sublicense, and/or sell copies of the Software, and to permit
|
|
# persons to whom the Software is furnished to do so, subject to the fol-
|
|
# lowing conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included
|
|
# in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
|
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
|
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
# IN THE SOFTWARE.
|
|
|
|
#
|
|
# Parts of this code were copied or derived from sample code supplied by AWS.
|
|
# The following notice applies to that code.
|
|
#
|
|
# This software code is made available "AS IS" without warranties of any
|
|
# kind. You may copy, display, modify and redistribute the software
|
|
# code either by itself or as incorporated into your code; provided that
|
|
# you do not remove any proprietary notices. Your use of this software
|
|
# code is at your own risk and you waive any claim against Amazon
|
|
# Digital Services, Inc. or its affiliates with respect to your use of
|
|
# this software code. (c) 2006 Amazon Digital Services, Inc. or its
|
|
# affiliates.
|
|
|
|
"""
|
|
Handles basic connections to AWS
|
|
"""
|
|
|
|
from __future__ import with_statement
|
|
import base64
|
|
import errno
|
|
import httplib
|
|
import os
|
|
import Queue
|
|
import random
|
|
import re
|
|
import socket
|
|
import sys
|
|
import time
|
|
import urllib
|
|
import urlparse
|
|
import xml.sax
|
|
import copy
|
|
|
|
import auth
|
|
import auth_handler
|
|
import boto
|
|
import boto.utils
|
|
import boto.handler
|
|
import boto.cacerts
|
|
|
|
from boto import config, UserAgent
|
|
from boto.exception import AWSConnectionError, BotoClientError
|
|
from boto.exception import BotoServerError
|
|
from boto.provider import Provider
|
|
from boto.resultset import ResultSet
|
|
|
|
HAVE_HTTPS_CONNECTION = False
|
|
try:
|
|
import ssl
|
|
from boto import https_connection
|
|
# Google App Engine runs on Python 2.5 so doesn't have ssl.SSLError.
|
|
if hasattr(ssl, 'SSLError'):
|
|
HAVE_HTTPS_CONNECTION = True
|
|
except ImportError:
|
|
pass
|
|
|
|
try:
|
|
import threading
|
|
except ImportError:
|
|
import dummy_threading as threading
|
|
|
|
ON_APP_ENGINE = all(key in os.environ for key in (
|
|
'USER_IS_ADMIN', 'CURRENT_VERSION_ID', 'APPLICATION_ID'))
|
|
|
|
PORTS_BY_SECURITY = {True: 443,
|
|
False: 80}
|
|
|
|
DEFAULT_CA_CERTS_FILE = os.path.join(os.path.dirname(os.path.abspath(boto.cacerts.__file__ )), "cacerts.txt")
|
|
|
|
|
|
class HostConnectionPool(object):
|
|
|
|
"""
|
|
A pool of connections for one remote (host,is_secure).
|
|
|
|
When connections are added to the pool, they are put into a
|
|
pending queue. The _mexe method returns connections to the pool
|
|
before the response body has been read, so they connections aren't
|
|
ready to send another request yet. They stay in the pending queue
|
|
until they are ready for another request, at which point they are
|
|
returned to the pool of ready connections.
|
|
|
|
The pool of ready connections is an ordered list of
|
|
(connection,time) pairs, where the time is the time the connection
|
|
was returned from _mexe. After a certain period of time,
|
|
connections are considered stale, and discarded rather than being
|
|
reused. This saves having to wait for the connection to time out
|
|
if AWS has decided to close it on the other end because of
|
|
inactivity.
|
|
|
|
Thread Safety:
|
|
|
|
This class is used only fram ConnectionPool while it's mutex
|
|
is held.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.queue = []
|
|
|
|
def size(self):
|
|
"""
|
|
Returns the number of connections in the pool for this host.
|
|
Some of the connections may still be in use, and may not be
|
|
ready to be returned by get().
|
|
"""
|
|
return len(self.queue)
|
|
|
|
def put(self, conn):
|
|
"""
|
|
Adds a connection to the pool, along with the time it was
|
|
added.
|
|
"""
|
|
self.queue.append((conn, time.time()))
|
|
|
|
def get(self):
|
|
"""
|
|
Returns the next connection in this pool that is ready to be
|
|
reused. Returns None of there aren't any.
|
|
"""
|
|
# Discard ready connections that are too old.
|
|
self.clean()
|
|
|
|
# Return the first connection that is ready, and remove it
|
|
# from the queue. Connections that aren't ready are returned
|
|
# to the end of the queue with an updated time, on the
|
|
# assumption that somebody is actively reading the response.
|
|
for _ in range(len(self.queue)):
|
|
(conn, _) = self.queue.pop(0)
|
|
if self._conn_ready(conn):
|
|
return conn
|
|
else:
|
|
self.put(conn)
|
|
return None
|
|
|
|
def _conn_ready(self, conn):
|
|
"""
|
|
There is a nice state diagram at the top of httplib.py. It
|
|
indicates that once the response headers have been read (which
|
|
_mexe does before adding the connection to the pool), a
|
|
response is attached to the connection, and it stays there
|
|
until it's done reading. This isn't entirely true: even after
|
|
the client is done reading, the response may be closed, but
|
|
not removed from the connection yet.
|
|
|
|
This is ugly, reading a private instance variable, but the
|
|
state we care about isn't available in any public methods.
|
|
"""
|
|
if ON_APP_ENGINE:
|
|
# Google AppEngine implementation of HTTPConnection doesn't contain
|
|
# _HTTPConnection__response attribute. Moreover, it's not possible
|
|
# to determine if given connection is ready. Reusing connections
|
|
# simply doesn't make sense with App Engine urlfetch service.
|
|
return False
|
|
else:
|
|
response = getattr(conn, '_HTTPConnection__response', None)
|
|
return (response is None) or response.isclosed()
|
|
|
|
def clean(self):
|
|
"""
|
|
Get rid of stale connections.
|
|
"""
|
|
# Note that we do not close the connection here -- somebody
|
|
# may still be reading from it.
|
|
while len(self.queue) > 0 and self._pair_stale(self.queue[0]):
|
|
self.queue.pop(0)
|
|
|
|
def _pair_stale(self, pair):
|
|
"""
|
|
Returns true of the (connection,time) pair is too old to be
|
|
used.
|
|
"""
|
|
(_conn, return_time) = pair
|
|
now = time.time()
|
|
return return_time + ConnectionPool.STALE_DURATION < now
|
|
|
|
|
|
class ConnectionPool(object):
|
|
|
|
"""
|
|
A connection pool that expires connections after a fixed period of
|
|
time. This saves time spent waiting for a connection that AWS has
|
|
timed out on the other end.
|
|
|
|
This class is thread-safe.
|
|
"""
|
|
|
|
#
|
|
# The amout of time between calls to clean.
|
|
#
|
|
|
|
CLEAN_INTERVAL = 5.0
|
|
|
|
#
|
|
# How long before a connection becomes "stale" and won't be reused
|
|
# again. The intention is that this time is less that the timeout
|
|
# period that AWS uses, so we'll never try to reuse a connection
|
|
# and find that AWS is timing it out.
|
|
#
|
|
# Experimentation in July 2011 shows that AWS starts timing things
|
|
# out after three minutes. The 60 seconds here is conservative so
|
|
# we should never hit that 3-minute timout.
|
|
#
|
|
|
|
STALE_DURATION = 60.0
|
|
|
|
def __init__(self):
|
|
# Mapping from (host,is_secure) to HostConnectionPool.
|
|
# If a pool becomes empty, it is removed.
|
|
self.host_to_pool = {}
|
|
# The last time the pool was cleaned.
|
|
self.last_clean_time = 0.0
|
|
self.mutex = threading.Lock()
|
|
ConnectionPool.STALE_DURATION = \
|
|
config.getfloat('Boto', 'connection_stale_duration',
|
|
ConnectionPool.STALE_DURATION)
|
|
|
|
def __getstate__(self):
|
|
pickled_dict = copy.copy(self.__dict__)
|
|
pickled_dict['host_to_pool'] = {}
|
|
del pickled_dict['mutex']
|
|
return pickled_dict
|
|
|
|
def __setstate__(self, dct):
|
|
self.__init__()
|
|
|
|
def size(self):
|
|
"""
|
|
Returns the number of connections in the pool.
|
|
"""
|
|
return sum(pool.size() for pool in self.host_to_pool.values())
|
|
|
|
def get_http_connection(self, host, is_secure):
|
|
"""
|
|
Gets a connection from the pool for the named host. Returns
|
|
None if there is no connection that can be reused. It's the caller's
|
|
responsibility to call close() on the connection when it's no longer
|
|
needed.
|
|
"""
|
|
self.clean()
|
|
with self.mutex:
|
|
key = (host, is_secure)
|
|
if key not in self.host_to_pool:
|
|
return None
|
|
return self.host_to_pool[key].get()
|
|
|
|
def put_http_connection(self, host, is_secure, conn):
|
|
"""
|
|
Adds a connection to the pool of connections that can be
|
|
reused for the named host.
|
|
"""
|
|
with self.mutex:
|
|
key = (host, is_secure)
|
|
if key not in self.host_to_pool:
|
|
self.host_to_pool[key] = HostConnectionPool()
|
|
self.host_to_pool[key].put(conn)
|
|
|
|
def clean(self):
|
|
"""
|
|
Clean up the stale connections in all of the pools, and then
|
|
get rid of empty pools. Pools clean themselves every time a
|
|
connection is fetched; this cleaning takes care of pools that
|
|
aren't being used any more, so nothing is being gotten from
|
|
them.
|
|
"""
|
|
with self.mutex:
|
|
now = time.time()
|
|
if self.last_clean_time + self.CLEAN_INTERVAL < now:
|
|
to_remove = []
|
|
for (host, pool) in self.host_to_pool.items():
|
|
pool.clean()
|
|
if pool.size() == 0:
|
|
to_remove.append(host)
|
|
for host in to_remove:
|
|
del self.host_to_pool[host]
|
|
self.last_clean_time = now
|
|
|
|
|
|
class HTTPRequest(object):
|
|
|
|
def __init__(self, method, protocol, host, port, path, auth_path,
|
|
params, headers, body):
|
|
"""Represents an HTTP request.
|
|
|
|
:type method: string
|
|
:param method: The HTTP method name, 'GET', 'POST', 'PUT' etc.
|
|
|
|
:type protocol: string
|
|
:param protocol: The http protocol used, 'http' or 'https'.
|
|
|
|
:type host: string
|
|
:param host: Host to which the request is addressed. eg. abc.com
|
|
|
|
:type port: int
|
|
:param port: port on which the request is being sent. Zero means unset,
|
|
in which case default port will be chosen.
|
|
|
|
:type path: string
|
|
:param path: URL path that is being accessed.
|
|
|
|
:type auth_path: string
|
|
:param path: The part of the URL path used when creating the
|
|
authentication string.
|
|
|
|
:type params: dict
|
|
:param params: HTTP url query parameters, with key as name of
|
|
the param, and value as value of param.
|
|
|
|
:type headers: dict
|
|
:param headers: HTTP headers, with key as name of the header and value
|
|
as value of header.
|
|
|
|
:type body: string
|
|
:param body: Body of the HTTP request. If not present, will be None or
|
|
empty string ('').
|
|
"""
|
|
self.method = method
|
|
self.protocol = protocol
|
|
self.host = host
|
|
self.port = port
|
|
self.path = path
|
|
if auth_path is None:
|
|
auth_path = path
|
|
self.auth_path = auth_path
|
|
self.params = params
|
|
# chunked Transfer-Encoding should act only on PUT request.
|
|
if headers and 'Transfer-Encoding' in headers and \
|
|
headers['Transfer-Encoding'] == 'chunked' and \
|
|
self.method != 'PUT':
|
|
self.headers = headers.copy()
|
|
del self.headers['Transfer-Encoding']
|
|
else:
|
|
self.headers = headers
|
|
self.body = body
|
|
|
|
def __str__(self):
|
|
return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) '
|
|
'params(%s) headers(%s) body(%s)') % (self.method,
|
|
self.protocol, self.host, self.port, self.path, self.params,
|
|
self.headers, self.body))
|
|
|
|
def authorize(self, connection, **kwargs):
|
|
for key in self.headers:
|
|
val = self.headers[key]
|
|
if isinstance(val, unicode):
|
|
self.headers[key] = urllib.quote_plus(val.encode('utf-8'))
|
|
|
|
connection._auth_handler.add_auth(self, **kwargs)
|
|
|
|
self.headers['User-Agent'] = UserAgent
|
|
# I'm not sure if this is still needed, now that add_auth is
|
|
# setting the content-length for POST requests.
|
|
if 'Content-Length' not in self.headers:
|
|
if 'Transfer-Encoding' not in self.headers or \
|
|
self.headers['Transfer-Encoding'] != 'chunked':
|
|
self.headers['Content-Length'] = str(len(self.body))
|
|
|
|
|
|
class HTTPResponse(httplib.HTTPResponse):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
httplib.HTTPResponse.__init__(self, *args, **kwargs)
|
|
self._cached_response = ''
|
|
|
|
def read(self, amt=None):
|
|
"""Read the response.
|
|
|
|
This method does not have the same behavior as
|
|
httplib.HTTPResponse.read. Instead, if this method is called with
|
|
no ``amt`` arg, then the response body will be cached. Subsequent
|
|
calls to ``read()`` with no args **will return the cached response**.
|
|
|
|
"""
|
|
if amt is None:
|
|
# The reason for doing this is that many places in boto call
|
|
# response.read() and except to get the response body that they
|
|
# can then process. To make sure this always works as they expect
|
|
# we're caching the response so that multiple calls to read()
|
|
# will return the full body. Note that this behavior only
|
|
# happens if the amt arg is not specified.
|
|
if not self._cached_response:
|
|
self._cached_response = httplib.HTTPResponse.read(self)
|
|
return self._cached_response
|
|
else:
|
|
return httplib.HTTPResponse.read(self, amt)
|
|
|
|
|
|
class AWSAuthConnection(object):
|
|
def __init__(self, host, aws_access_key_id=None,
|
|
aws_secret_access_key=None,
|
|
is_secure=True, port=None, proxy=None, proxy_port=None,
|
|
proxy_user=None, proxy_pass=None, debug=0,
|
|
https_connection_factory=None, path='/',
|
|
provider='aws', security_token=None,
|
|
suppress_consec_slashes=True,
|
|
validate_certs=True):
|
|
"""
|
|
:type host: str
|
|
:param host: The host to make the connection to
|
|
|
|
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
|
|
Amazon). If none is specified, the value in your
|
|
``AWS_ACCESS_KEY_ID`` environmental variable is used.
|
|
:keyword str aws_secret_access_key: Your AWS Secret Access Key
|
|
(provided by Amazon). If none is specified, the value in your
|
|
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
|
|
|
|
:type is_secure: boolean
|
|
:param is_secure: Whether the connection is over SSL
|
|
|
|
:type https_connection_factory: list or tuple
|
|
:param https_connection_factory: A pair of an HTTP connection
|
|
factory and the exceptions to catch. The factory should have
|
|
a similar interface to L{httplib.HTTPSConnection}.
|
|
|
|
:param str proxy: Address/hostname for a proxy server
|
|
|
|
:type proxy_port: int
|
|
:param proxy_port: The port to use when connecting over a proxy
|
|
|
|
:type proxy_user: str
|
|
:param proxy_user: The username to connect with on the proxy
|
|
|
|
:type proxy_pass: str
|
|
:param proxy_pass: The password to use when connection over a proxy.
|
|
|
|
:type port: int
|
|
:param port: The port to use to connect
|
|
|
|
:type suppress_consec_slashes: bool
|
|
:param suppress_consec_slashes: If provided, controls whether
|
|
consecutive slashes will be suppressed in key paths.
|
|
|
|
:type validate_certs: bool
|
|
:param validate_certs: Controls whether SSL certificates
|
|
will be validated or not. Defaults to True.
|
|
"""
|
|
self.suppress_consec_slashes = suppress_consec_slashes
|
|
self.num_retries = 6
|
|
# Override passed-in is_secure setting if value was defined in config.
|
|
if config.has_option('Boto', 'is_secure'):
|
|
is_secure = config.getboolean('Boto', 'is_secure')
|
|
self.is_secure = is_secure
|
|
# Whether or not to validate server certificates.
|
|
# The default is now to validate certificates. This can be
|
|
# overridden in the boto config file are by passing an
|
|
# explicit validate_certs parameter to the class constructor.
|
|
self.https_validate_certificates = config.getbool(
|
|
'Boto', 'https_validate_certificates',
|
|
validate_certs)
|
|
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
|
|
raise BotoClientError(
|
|
"SSL server certificate validation is enabled in boto "
|
|
"configuration, but Python dependencies required to "
|
|
"support this feature are not available. Certificate "
|
|
"validation is only supported when running under Python "
|
|
"2.6 or later.")
|
|
self.ca_certificates_file = config.get_value(
|
|
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
|
|
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
|
|
# define exceptions from httplib that we want to catch and retry
|
|
self.http_exceptions = (httplib.HTTPException, socket.error,
|
|
socket.gaierror, httplib.BadStatusLine)
|
|
# define subclasses of the above that are not retryable.
|
|
self.http_unretryable_exceptions = []
|
|
if HAVE_HTTPS_CONNECTION:
|
|
self.http_unretryable_exceptions.append(
|
|
https_connection.InvalidCertificateException)
|
|
|
|
# define values in socket exceptions we don't want to catch
|
|
self.socket_exception_values = (errno.EINTR,)
|
|
if https_connection_factory is not None:
|
|
self.https_connection_factory = https_connection_factory[0]
|
|
self.http_exceptions += https_connection_factory[1]
|
|
else:
|
|
self.https_connection_factory = None
|
|
if (is_secure):
|
|
self.protocol = 'https'
|
|
else:
|
|
self.protocol = 'http'
|
|
self.host = host
|
|
self.path = path
|
|
# if the value passed in for debug
|
|
if not isinstance(debug, (int, long)):
|
|
debug = 0
|
|
self.debug = config.getint('Boto', 'debug', debug)
|
|
if port:
|
|
self.port = port
|
|
else:
|
|
self.port = PORTS_BY_SECURITY[is_secure]
|
|
|
|
# Timeout used to tell httplib how long to wait for socket timeouts.
|
|
# Default is to leave timeout unchanged, which will in turn result in
|
|
# the socket's default global timeout being used. To specify a
|
|
# timeout, set http_socket_timeout in Boto config. Regardless,
|
|
# timeouts will only be applied if Python is 2.6 or greater.
|
|
self.http_connection_kwargs = {}
|
|
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
|
|
if config.has_option('Boto', 'http_socket_timeout'):
|
|
timeout = config.getint('Boto', 'http_socket_timeout')
|
|
self.http_connection_kwargs['timeout'] = timeout
|
|
|
|
if isinstance(provider, Provider):
|
|
# Allow overriding Provider
|
|
self.provider = provider
|
|
else:
|
|
self._provider_type = provider
|
|
self.provider = Provider(self._provider_type,
|
|
aws_access_key_id,
|
|
aws_secret_access_key,
|
|
security_token)
|
|
|
|
# allow config file to override default host
|
|
if self.provider.host:
|
|
self.host = self.provider.host
|
|
|
|
self._pool = ConnectionPool()
|
|
self._connection = (self.server_name(), self.is_secure)
|
|
self._last_rs = None
|
|
self._auth_handler = auth.get_auth_handler(
|
|
host, config, self.provider, self._required_auth_capability())
|
|
if getattr(self, 'AuthServiceName', None) is not None:
|
|
self.auth_service_name = self.AuthServiceName
|
|
|
|
def __repr__(self):
|
|
return '%s:%s' % (self.__class__.__name__, self.host)
|
|
|
|
def _required_auth_capability(self):
|
|
return []
|
|
|
|
def _get_auth_service_name(self):
|
|
return getattr(self._auth_handler, 'service_name')
|
|
|
|
# For Sigv4, the auth_service_name/auth_region_name properties allow
|
|
# the service_name/region_name to be explicitly set instead of being
|
|
# derived from the endpoint url.
|
|
def _set_auth_service_name(self, value):
|
|
self._auth_handler.service_name = value
|
|
auth_service_name = property(_get_auth_service_name, _set_auth_service_name)
|
|
|
|
def _get_auth_region_name(self):
|
|
return getattr(self._auth_handler, 'region_name')
|
|
|
|
def _set_auth_region_name(self, value):
|
|
self._auth_handler.region_name = value
|
|
auth_region_name = property(_get_auth_region_name, _set_auth_region_name)
|
|
|
|
def connection(self):
|
|
return self.get_http_connection(*self._connection)
|
|
connection = property(connection)
|
|
|
|
def aws_access_key_id(self):
|
|
return self.provider.access_key
|
|
aws_access_key_id = property(aws_access_key_id)
|
|
gs_access_key_id = aws_access_key_id
|
|
access_key = aws_access_key_id
|
|
|
|
def aws_secret_access_key(self):
|
|
return self.provider.secret_key
|
|
aws_secret_access_key = property(aws_secret_access_key)
|
|
gs_secret_access_key = aws_secret_access_key
|
|
secret_key = aws_secret_access_key
|
|
|
|
def get_path(self, path='/'):
|
|
# The default behavior is to suppress consecutive slashes for reasons
|
|
# discussed at
|
|
# https://groups.google.com/forum/#!topic/boto-dev/-ft0XPUy0y8
|
|
# You can override that behavior with the suppress_consec_slashes param.
|
|
if not self.suppress_consec_slashes:
|
|
return self.path + re.sub('^/*', "", path)
|
|
pos = path.find('?')
|
|
if pos >= 0:
|
|
params = path[pos:]
|
|
path = path[:pos]
|
|
else:
|
|
params = None
|
|
if path[-1] == '/':
|
|
need_trailing = True
|
|
else:
|
|
need_trailing = False
|
|
path_elements = self.path.split('/')
|
|
path_elements.extend(path.split('/'))
|
|
path_elements = [p for p in path_elements if p]
|
|
path = '/' + '/'.join(path_elements)
|
|
if path[-1] != '/' and need_trailing:
|
|
path += '/'
|
|
if params:
|
|
path = path + params
|
|
return path
|
|
|
|
def server_name(self, port=None):
|
|
if not port:
|
|
port = self.port
|
|
if port == 80:
|
|
signature_host = self.host
|
|
else:
|
|
# This unfortunate little hack can be attributed to
|
|
# a difference in the 2.6 version of httplib. In old
|
|
# versions, it would append ":443" to the hostname sent
|
|
# in the Host header and so we needed to make sure we
|
|
# did the same when calculating the V2 signature. In 2.6
|
|
# (and higher!)
|
|
# it no longer does that. Hence, this kludge.
|
|
if ((ON_APP_ENGINE and sys.version[:3] == '2.5') or
|
|
sys.version[:3] in ('2.6', '2.7')) and port == 443:
|
|
signature_host = self.host
|
|
else:
|
|
signature_host = '%s:%d' % (self.host, port)
|
|
return signature_host
|
|
|
|
def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
|
|
self.proxy = proxy
|
|
self.proxy_port = proxy_port
|
|
self.proxy_user = proxy_user
|
|
self.proxy_pass = proxy_pass
|
|
if 'http_proxy' in os.environ and not self.proxy:
|
|
pattern = re.compile(
|
|
'(?:http://)?' \
|
|
'(?:(?P<user>\w+):(?P<pass>.*)@)?' \
|
|
'(?P<host>[\w\-\.]+)' \
|
|
'(?::(?P<port>\d+))?'
|
|
)
|
|
match = pattern.match(os.environ['http_proxy'])
|
|
if match:
|
|
self.proxy = match.group('host')
|
|
self.proxy_port = match.group('port')
|
|
self.proxy_user = match.group('user')
|
|
self.proxy_pass = match.group('pass')
|
|
else:
|
|
if not self.proxy:
|
|
self.proxy = config.get_value('Boto', 'proxy', None)
|
|
if not self.proxy_port:
|
|
self.proxy_port = config.get_value('Boto', 'proxy_port', None)
|
|
if not self.proxy_user:
|
|
self.proxy_user = config.get_value('Boto', 'proxy_user', None)
|
|
if not self.proxy_pass:
|
|
self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)
|
|
|
|
if not self.proxy_port and self.proxy:
|
|
print "http_proxy environment variable does not specify " \
|
|
"a port, using default"
|
|
self.proxy_port = self.port
|
|
self.use_proxy = (self.proxy != None)
|
|
|
|
def get_http_connection(self, host, is_secure):
|
|
conn = self._pool.get_http_connection(host, is_secure)
|
|
if conn is not None:
|
|
return conn
|
|
else:
|
|
return self.new_http_connection(host, is_secure)
|
|
|
|
def new_http_connection(self, host, is_secure):
|
|
if self.use_proxy and not is_secure:
|
|
host = '%s:%d' % (self.proxy, int(self.proxy_port))
|
|
if host is None:
|
|
host = self.server_name()
|
|
if is_secure:
|
|
boto.log.debug(
|
|
'establishing HTTPS connection: host=%s, kwargs=%s',
|
|
host, self.http_connection_kwargs)
|
|
if self.use_proxy:
|
|
connection = self.proxy_ssl(host, is_secure and 443 or 80)
|
|
elif self.https_connection_factory:
|
|
connection = self.https_connection_factory(host)
|
|
elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
|
|
connection = https_connection.CertValidatingHTTPSConnection(
|
|
host, ca_certs=self.ca_certificates_file,
|
|
**self.http_connection_kwargs)
|
|
else:
|
|
connection = httplib.HTTPSConnection(host,
|
|
**self.http_connection_kwargs)
|
|
else:
|
|
boto.log.debug('establishing HTTP connection: kwargs=%s' %
|
|
self.http_connection_kwargs)
|
|
if self.https_connection_factory:
|
|
# even though the factory says https, this is too handy
|
|
# to not be able to allow overriding for http also.
|
|
connection = self.https_connection_factory(host,
|
|
**self.http_connection_kwargs)
|
|
else:
|
|
connection = httplib.HTTPConnection(host,
|
|
**self.http_connection_kwargs)
|
|
if self.debug > 1:
|
|
connection.set_debuglevel(self.debug)
|
|
# self.connection must be maintained for backwards-compatibility
|
|
# however, it must be dynamically pulled from the connection pool
|
|
# set a private variable which will enable that
|
|
if host.split(':')[0] == self.host and is_secure == self.is_secure:
|
|
self._connection = (host, is_secure)
|
|
# Set the response class of the http connection to use our custom
|
|
# class.
|
|
connection.response_class = HTTPResponse
|
|
return connection
|
|
|
|
def put_http_connection(self, host, is_secure, connection):
|
|
self._pool.put_http_connection(host, is_secure, connection)
|
|
|
|
def proxy_ssl(self, host=None, port=None):
|
|
if host and port:
|
|
host = '%s:%d' % (host, port)
|
|
else:
|
|
host = '%s:%d' % (self.host, self.port)
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
try:
|
|
sock.connect((self.proxy, int(self.proxy_port)))
|
|
except:
|
|
raise
|
|
boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host)
|
|
sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
|
|
sock.sendall("User-Agent: %s\r\n" % UserAgent)
|
|
if self.proxy_user and self.proxy_pass:
|
|
for k, v in self.get_proxy_auth_header().items():
|
|
sock.sendall("%s: %s\r\n" % (k, v))
|
|
# See discussion about this config option at
|
|
# https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc
|
|
if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False):
|
|
sock.sendall("\r\n")
|
|
else:
|
|
sock.sendall("\r\n")
|
|
resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug)
|
|
resp.begin()
|
|
|
|
if resp.status != 200:
|
|
# Fake a socket error, use a code that make it obvious it hasn't
|
|
# been generated by the socket library
|
|
raise socket.error(-71,
|
|
"Error talking to HTTP proxy %s:%s: %s (%s)" %
|
|
(self.proxy, self.proxy_port,
|
|
resp.status, resp.reason))
|
|
|
|
# We can safely close the response, it duped the original socket
|
|
resp.close()
|
|
|
|
h = httplib.HTTPConnection(host)
|
|
|
|
if self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
|
|
boto.log.debug("wrapping ssl socket for proxied connection; "
|
|
"CA certificate file=%s",
|
|
self.ca_certificates_file)
|
|
key_file = self.http_connection_kwargs.get('key_file', None)
|
|
cert_file = self.http_connection_kwargs.get('cert_file', None)
|
|
sslSock = ssl.wrap_socket(sock, keyfile=key_file,
|
|
certfile=cert_file,
|
|
cert_reqs=ssl.CERT_REQUIRED,
|
|
ca_certs=self.ca_certificates_file)
|
|
cert = sslSock.getpeercert()
|
|
hostname = self.host.split(':', 0)[0]
|
|
if not https_connection.ValidateCertificateHostname(cert, hostname):
|
|
raise https_connection.InvalidCertificateException(
|
|
hostname, cert, 'hostname mismatch')
|
|
else:
|
|
# Fallback for old Python without ssl.wrap_socket
|
|
if hasattr(httplib, 'ssl'):
|
|
sslSock = httplib.ssl.SSLSocket(sock)
|
|
else:
|
|
sslSock = socket.ssl(sock, None, None)
|
|
sslSock = httplib.FakeSocket(sock, sslSock)
|
|
|
|
# This is a bit unclean
|
|
h.sock = sslSock
|
|
return h
|
|
|
|
def prefix_proxy_to_path(self, path, host=None):
|
|
path = self.protocol + '://' + (host or self.server_name()) + path
|
|
return path
|
|
|
|
def get_proxy_auth_header(self):
|
|
auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass)
|
|
return {'Proxy-Authorization': 'Basic %s' % auth}
|
|
|
|
def _mexe(self, request, sender=None, override_num_retries=None,
|
|
retry_handler=None):
|
|
"""
|
|
mexe - Multi-execute inside a loop, retrying multiple times to handle
|
|
transient Internet errors by simply trying again.
|
|
Also handles redirects.
|
|
|
|
This code was inspired by the S3Utils classes posted to the boto-users
|
|
Google group by Larry Bates. Thanks!
|
|
|
|
"""
|
|
boto.log.debug('Method: %s' % request.method)
|
|
boto.log.debug('Path: %s' % request.path)
|
|
boto.log.debug('Data: %s' % request.body)
|
|
boto.log.debug('Headers: %s' % request.headers)
|
|
boto.log.debug('Host: %s' % request.host)
|
|
response = None
|
|
body = None
|
|
e = None
|
|
if override_num_retries is None:
|
|
num_retries = config.getint('Boto', 'num_retries', self.num_retries)
|
|
else:
|
|
num_retries = override_num_retries
|
|
i = 0
|
|
connection = self.get_http_connection(request.host, self.is_secure)
|
|
while i <= num_retries:
|
|
# Use binary exponential backoff to desynchronize client requests.
|
|
next_sleep = random.random() * (2 ** i)
|
|
try:
|
|
# we now re-sign each request before it is retried
|
|
boto.log.debug('Token: %s' % self.provider.security_token)
|
|
request.authorize(connection=self)
|
|
if callable(sender):
|
|
response = sender(connection, request.method, request.path,
|
|
request.body, request.headers)
|
|
else:
|
|
connection.request(request.method, request.path,
|
|
request.body, request.headers)
|
|
response = connection.getresponse()
|
|
location = response.getheader('location')
|
|
# -- gross hack --
|
|
# httplib gets confused with chunked responses to HEAD requests
|
|
# so I have to fake it out
|
|
if request.method == 'HEAD' and getattr(response,
|
|
'chunked', False):
|
|
response.chunked = 0
|
|
if callable(retry_handler):
|
|
status = retry_handler(response, i, next_sleep)
|
|
if status:
|
|
msg, i, next_sleep = status
|
|
if msg:
|
|
boto.log.debug(msg)
|
|
time.sleep(next_sleep)
|
|
continue
|
|
if response.status == 500 or response.status == 503:
|
|
msg = 'Received %d response. ' % response.status
|
|
msg += 'Retrying in %3.1f seconds' % next_sleep
|
|
boto.log.debug(msg)
|
|
body = response.read()
|
|
elif response.status < 300 or response.status >= 400 or \
|
|
not location:
|
|
self.put_http_connection(request.host, self.is_secure,
|
|
connection)
|
|
return response
|
|
else:
|
|
scheme, request.host, request.path, \
|
|
params, query, fragment = urlparse.urlparse(location)
|
|
if query:
|
|
request.path += '?' + query
|
|
msg = 'Redirecting: %s' % scheme + '://'
|
|
msg += request.host + request.path
|
|
boto.log.debug(msg)
|
|
connection = self.get_http_connection(request.host,
|
|
scheme == 'https')
|
|
response = None
|
|
continue
|
|
except self.http_exceptions, e:
|
|
for unretryable in self.http_unretryable_exceptions:
|
|
if isinstance(e, unretryable):
|
|
boto.log.debug(
|
|
'encountered unretryable %s exception, re-raising' %
|
|
e.__class__.__name__)
|
|
raise e
|
|
boto.log.debug('encountered %s exception, reconnecting' % \
|
|
e.__class__.__name__)
|
|
connection = self.new_http_connection(request.host,
|
|
self.is_secure)
|
|
time.sleep(next_sleep)
|
|
i += 1
|
|
# If we made it here, it's because we have exhausted our retries
|
|
# and stil haven't succeeded. So, if we have a response object,
|
|
# use it to raise an exception.
|
|
# Otherwise, raise the exception that must have already h#appened.
|
|
if response:
|
|
raise BotoServerError(response.status, response.reason, body)
|
|
elif e:
|
|
raise e
|
|
else:
|
|
msg = 'Please report this exception as a Boto Issue!'
|
|
raise BotoClientError(msg)
|
|
|
|
def build_base_http_request(self, method, path, auth_path,
|
|
params=None, headers=None, data='', host=None):
|
|
path = self.get_path(path)
|
|
if auth_path is not None:
|
|
auth_path = self.get_path(auth_path)
|
|
if params == None:
|
|
params = {}
|
|
else:
|
|
params = params.copy()
|
|
if headers == None:
|
|
headers = {}
|
|
else:
|
|
headers = headers.copy()
|
|
host = host or self.host
|
|
if self.use_proxy:
|
|
if not auth_path:
|
|
auth_path = path
|
|
path = self.prefix_proxy_to_path(path, host)
|
|
if self.proxy_user and self.proxy_pass and not self.is_secure:
|
|
# If is_secure, we don't have to set the proxy authentication
|
|
# header here, we did that in the CONNECT to the proxy.
|
|
headers.update(self.get_proxy_auth_header())
|
|
return HTTPRequest(method, self.protocol, host, self.port,
|
|
path, auth_path, params, headers, data)
|
|
|
|
def make_request(self, method, path, headers=None, data='', host=None,
|
|
auth_path=None, sender=None, override_num_retries=None,
|
|
params=None):
|
|
"""Makes a request to the server, with stock multiple-retry logic."""
|
|
if params is None:
|
|
params = {}
|
|
http_request = self.build_base_http_request(method, path, auth_path,
|
|
params, headers, data, host)
|
|
return self._mexe(http_request, sender, override_num_retries)
|
|
|
|
def close(self):
|
|
"""(Optional) Close any open HTTP connections. This is non-destructive,
|
|
and making a new request will open a connection again."""
|
|
|
|
boto.log.debug('closing all HTTP connections')
|
|
self._connection = None # compat field
|
|
|
|
|
|
class AWSQueryConnection(AWSAuthConnection):
|
|
|
|
APIVersion = ''
|
|
ResponseError = BotoServerError
|
|
|
|
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
|
|
is_secure=True, port=None, proxy=None, proxy_port=None,
|
|
proxy_user=None, proxy_pass=None, host=None, debug=0,
|
|
https_connection_factory=None, path='/', security_token=None,
|
|
validate_certs=True):
|
|
AWSAuthConnection.__init__(self, host, aws_access_key_id,
|
|
aws_secret_access_key,
|
|
is_secure, port, proxy,
|
|
proxy_port, proxy_user, proxy_pass,
|
|
debug, https_connection_factory, path,
|
|
security_token=security_token,
|
|
validate_certs=validate_certs)
|
|
|
|
def _required_auth_capability(self):
|
|
return []
|
|
|
|
def get_utf8_value(self, value):
|
|
return boto.utils.get_utf8_value(value)
|
|
|
|
def make_request(self, action, params=None, path='/', verb='GET'):
|
|
http_request = self.build_base_http_request(verb, path, None,
|
|
params, {}, '',
|
|
self.server_name())
|
|
if action:
|
|
http_request.params['Action'] = action
|
|
if self.APIVersion:
|
|
http_request.params['Version'] = self.APIVersion
|
|
return self._mexe(http_request)
|
|
|
|
def build_list_params(self, params, items, label):
|
|
if isinstance(items, basestring):
|
|
items = [items]
|
|
for i in range(1, len(items) + 1):
|
|
params['%s.%d' % (label, i)] = items[i - 1]
|
|
|
|
def build_complex_list_params(self, params, items, label, names):
|
|
"""Serialize a list of structures.
|
|
|
|
For example::
|
|
|
|
items = [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')]
|
|
label = 'ParamName.member'
|
|
names = ('One', 'Two', 'Three')
|
|
self.build_complex_list_params(params, items, label, names)
|
|
|
|
would result in the params dict being updated with these params::
|
|
|
|
ParamName.member.1.One = foo
|
|
ParamName.member.1.Two = bar
|
|
ParamName.member.1.Three = baz
|
|
|
|
ParamName.member.2.One = foo2
|
|
ParamName.member.2.Two = bar2
|
|
ParamName.member.2.Three = baz2
|
|
|
|
:type params: dict
|
|
:param params: The params dict. The complex list params
|
|
will be added to this dict.
|
|
|
|
:type items: list of tuples
|
|
:param items: The list to serialize.
|
|
|
|
:type label: string
|
|
:param label: The prefix to apply to the parameter.
|
|
|
|
:type names: tuple of strings
|
|
:param names: The names associated with each tuple element.
|
|
|
|
"""
|
|
for i, item in enumerate(items, 1):
|
|
current_prefix = '%s.%s' % (label, i)
|
|
for key, value in zip(names, item):
|
|
full_key = '%s.%s' % (current_prefix, key)
|
|
params[full_key] = value
|
|
|
|
# generics
|
|
|
|
def get_list(self, action, params, markers, path='/',
|
|
parent=None, verb='GET'):
|
|
if not parent:
|
|
parent = self
|
|
response = self.make_request(action, params, path, verb)
|
|
body = response.read()
|
|
boto.log.debug(body)
|
|
if not body:
|
|
boto.log.error('Null body %s' % body)
|
|
raise self.ResponseError(response.status, response.reason, body)
|
|
elif response.status == 200:
|
|
rs = ResultSet(markers)
|
|
h = boto.handler.XmlHandler(rs, parent)
|
|
xml.sax.parseString(body, h)
|
|
return rs
|
|
else:
|
|
boto.log.error('%s %s' % (response.status, response.reason))
|
|
boto.log.error('%s' % body)
|
|
raise self.ResponseError(response.status, response.reason, body)
|
|
|
|
def get_object(self, action, params, cls, path='/',
|
|
parent=None, verb='GET'):
|
|
if not parent:
|
|
parent = self
|
|
response = self.make_request(action, params, path, verb)
|
|
body = response.read()
|
|
boto.log.debug(body)
|
|
if not body:
|
|
boto.log.error('Null body %s' % body)
|
|
raise self.ResponseError(response.status, response.reason, body)
|
|
elif response.status == 200:
|
|
obj = cls(parent)
|
|
h = boto.handler.XmlHandler(obj, parent)
|
|
xml.sax.parseString(body, h)
|
|
return obj
|
|
else:
|
|
boto.log.error('%s %s' % (response.status, response.reason))
|
|
boto.log.error('%s' % body)
|
|
raise self.ResponseError(response.status, response.reason, body)
|
|
|
|
def get_status(self, action, params, path='/', parent=None, verb='GET'):
|
|
if not parent:
|
|
parent = self
|
|
response = self.make_request(action, params, path, verb)
|
|
body = response.read()
|
|
boto.log.debug(body)
|
|
if not body:
|
|
boto.log.error('Null body %s' % body)
|
|
raise self.ResponseError(response.status, response.reason, body)
|
|
elif response.status == 200:
|
|
rs = ResultSet()
|
|
h = boto.handler.XmlHandler(rs, parent)
|
|
xml.sax.parseString(body, h)
|
|
return rs.status
|
|
else:
|
|
boto.log.error('%s %s' % (response.status, response.reason))
|
|
boto.log.error('%s' % body)
|
|
raise self.ResponseError(response.status, response.reason, body)
|