diff --git a/third_party/fancy_urllib/README b/third_party/fancy_urllib/README index 91da20ec7d..4fdaf2a513 100644 --- a/third_party/fancy_urllib/README +++ b/third_party/fancy_urllib/README @@ -1,7 +1,5 @@ -The fancy_urllib library was obtained from -http://googleappengine.googlecode.com/svn/trunk/python/lib/fancy_urllib/fancy_urllib/__init__.py -under the following license (http://googleappengine.googlecode.com/svn/trunk/python/LICENSE): - +The fancy_urllib library was obtained from [1] under the following +license ([2]): GOOGLE APP ENGINE SDK ===================== @@ -19,3 +17,6 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +[1] https://github.com/GoogleCloudPlatform/appengine-python-vm-runtime/blob/master/python_vm_runtime/lib/fancy_urllib/fancy_urllib/__init__.py +[2] https://github.com/GoogleCloudPlatform/appengine-python-vm-runtime/blob/master/python_vm_runtime/LICENSE diff --git a/third_party/fancy_urllib/__init__.py b/third_party/fancy_urllib/__init__.py index d4da0dd192..945d83bc7d 100644 --- a/third_party/fancy_urllib/__init__.py +++ b/third_party/fancy_urllib/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software # Foundation; All Rights Reserved @@ -14,13 +12,12 @@ __author__ = "{frew,nick.johnson}@google.com (Fred Wulff and Nick Johnson)" import base64 import httplib import logging -import re import socket -import urllib2 - +from urllib import splitpasswd from urllib import splittype from urllib import splituser -from urllib import splitpasswd +import urllib2 + class InvalidCertificateException(httplib.HTTPException): """Raised when a certificate is provided with an invalid hostname.""" @@ -31,6 +28,7 @@ class InvalidCertificateException(httplib.HTTPException): Args: host: The hostname the connection was made to. cert: The SSL certificate (as a dictionary) the host returned. + reason: user readable error reason. """ httplib.HTTPException.__init__(self) self.host = host @@ -38,21 +36,35 @@ class InvalidCertificateException(httplib.HTTPException): self.reason = reason def __str__(self): - return ('Host %s returned an invalid certificate (%s): %s\n' - 'To learn more, see ' - 'http://code.google.com/appengine/kb/general.html#rpcssl' % + return ("Host %s returned an invalid certificate (%s): %s\n" + "To learn more, see " + "http://code.google.com/appengine/kb/general.html#rpcssl" % (self.host, self.reason, self.cert)) + +try: + import ssl + _CAN_VALIDATE_CERTS = True +except ImportError: + _CAN_VALIDATE_CERTS = False + + def can_validate_certs(): """Return True if we have the SSL package and can validate certificates.""" - try: - import ssl - return True - except ImportError: - return False - -def _create_fancy_connection(tunnel_host=None, key_file=None, - cert_file=None, ca_certs=None): + return _CAN_VALIDATE_CERTS + + +# Reexport SSLError so clients don't have to to do their own checking for ssl's +# existence. +if can_validate_certs(): + SSLError = ssl.SSLError +else: + SSLError = None + + +def create_fancy_connection(tunnel_host=None, key_file=None, + cert_file=None, ca_certs=None, + proxy_authorization=None): # This abomination brought to you by the fact that # the HTTPHandler creates the connection instance in the middle # of do_open so we need to add the tunnel host to the class. @@ -70,28 +82,64 @@ def _create_fancy_connection(tunnel_host=None, key_file=None, self.key_file = key_file self.cert_file = cert_file self.ca_certs = ca_certs - try: - import ssl + if can_validate_certs(): if self.ca_certs: self.cert_reqs = ssl.CERT_REQUIRED else: self.cert_reqs = ssl.CERT_NONE - except ImportError: - pass + + def _get_hostport(self, host, port): + # Python 2.7.7rc1 (hg r90728:568041fd8090), 3.4.1 and 3.5 rename + # _set_hostport to _get_hostport and changes it's functionality. The + # Python 2.7.7rc1 version of this method is included here for + # compatibility with earlier versions of Python. Without this, HTTPS over + # HTTP CONNECT proxies cannot be used. + + # This method may be removed if compatibility with Python <2.7.7rc1 is not + # required. + + # Python bug: http://bugs.python.org/issue7776 + if port is None: + i = host.rfind(":") + j = host.rfind("]") # ipv6 addresses have [...] + if i > j: + try: + port = int(host[i+1:]) + except ValueError: + if host[i+1:] == "": # http://foo.com:/ == http://foo.com/ + port = self.default_port + else: + raise httplib.InvalidURL("nonnumeric port: '%s'" % host[i+1:]) + host = host[:i] + else: + port = self.default_port + if host and host[0] == "[" and host[-1] == "]": + host = host[1:-1] + + return (host, port) def _tunnel(self): - self._set_hostport(self._tunnel_host, None) + self.host, self.port = self._get_hostport(self._tunnel_host, None) logging.info("Connecting through tunnel to: %s:%d", self.host, self.port) - self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port)) + + self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)) + + if proxy_authorization: + self.send("Proxy-Authorization: %s\r\n" % proxy_authorization) + + # blank line + self.send("\r\n") + response = self.response_class(self.sock, strict=self.strict, method=self._method) + # pylint: disable=protected-access (_, code, message) = response._read_status() if code != 200: self.close() - raise socket.error, "Tunnel connection failed: %d %s" % ( - code, message.strip()) + raise socket.error("Tunnel connection failed: %d %s" % + (code, message.strip())) while True: line = response.fp.readline() @@ -106,15 +154,15 @@ def _create_fancy_connection(tunnel_host=None, key_file=None, Returns: list: A list of valid host globs. """ - if 'subjectAltName' in cert: - return [x[1] for x in cert['subjectAltName'] if x[0].lower() == 'dns'] + if "subjectAltName" in cert: + return [x[1] for x in cert["subjectAltName"] if x[0].lower() == "dns"] else: # Return a list of commonName fields - return [x[0][1] for x in cert['subject'] - if x[0][0].lower() == 'commonname'] + return [x[0][1] for x in cert["subject"] + if x[0][0].lower() == "commonname"] def _validate_certificate_hostname(self, cert, hostname): - """Validates that a given hostname is valid for an SSL certificate. + """Perform RFC2818/6125 validation against a cert and hostname. Args: cert: A dictionary representing an SSL certificate. @@ -124,14 +172,19 @@ def _create_fancy_connection(tunnel_host=None, key_file=None, """ hosts = self._get_valid_hosts_for_cert(cert) for host in hosts: - # Convert the glob-style hostname expression (eg, '*.google.com') into a - # valid regular expression. - host_re = host.replace('.', '\.').replace('*', '[^.]*') - if re.search('^%s$' % (host_re,), hostname, re.I): + # Wildcards are only valid when the * exists at the end of the last + # (left-most) label, and there are at least 3 labels in the expression. + if ("*." in host and host.count("*") == 1 and + host.count(".") > 1 and "." in hostname): + left_expected, right_expected = host.split("*.") + left_hostname, right_hostname = hostname.split(".", 1) + if (left_hostname.startswith(left_expected) and + right_expected == right_hostname): + return True + elif host == hostname: return True return False - def connect(self): # TODO(frew): When we drop support for <2.6 (in the far distant future), # change this to socket.create_connection. @@ -141,9 +194,11 @@ def _create_fancy_connection(tunnel_host=None, key_file=None, self._tunnel() # ssl and FakeSocket got deprecated. Try for the new hotness of wrap_ssl, - # with fallback. - try: - import ssl + # with fallback. Note: Since can_validate_certs() just checks for the + # ssl module, it's equivalent to attempting to import ssl from + # the function, but doesn't require a dynamic import, which doesn't + # play nicely with dev_appserver. + if can_validate_certs(): self.sock = ssl.wrap_socket(self.sock, keyfile=self.key_file, certfile=self.cert_file, @@ -152,15 +207,15 @@ def _create_fancy_connection(tunnel_host=None, key_file=None, if self.cert_reqs & ssl.CERT_REQUIRED: cert = self.sock.getpeercert() - hostname = self.host.split(':', 0)[0] + hostname = self.host.split(":", 0)[0] if not self._validate_certificate_hostname(cert, hostname): raise InvalidCertificateException(hostname, cert, - 'hostname mismatch') - except ImportError: - ssl = socket.ssl(self.sock, - keyfile=self.key_file, - certfile=self.cert_file) - self.sock = httplib.FakeSocket(self.sock, ssl) + "hostname mismatch") + else: + ssl_socket = socket.ssl(self.sock, + keyfile=self.key_file, + certfile=self.cert_file) + self.sock = httplib.FakeSocket(self.sock, ssl_socket) return PresetProxyHTTPSConnection @@ -329,17 +384,24 @@ class FancyProxyHandler(urllib2.ProxyHandler): class FancyHTTPSHandler(urllib2.HTTPSHandler): """An HTTPSHandler that works with CONNECT-enabled proxies.""" - def do_open(self, http_class, req): + def do_open(self, http_class, req, *args, **kwargs): + proxy_authorization = None + for header in req.headers: + if header.lower() == "proxy-authorization": + proxy_authorization = req.headers[header] + break + # Intentionally very specific so as to opt for false negatives # rather than false positives. try: return urllib2.HTTPSHandler.do_open( self, - _create_fancy_connection(req._tunnel_host, - req._key_file, - req._cert_file, - req._ca_certs), - req) + create_fancy_connection(req._tunnel_host, + req._key_file, + req._cert_file, + req._ca_certs, + proxy_authorization), + req, *args, **kwargs) except urllib2.URLError, url_error: try: import ssl @@ -347,7 +409,7 @@ class FancyHTTPSHandler(urllib2.HTTPSHandler): url_error.reason.args[0] == 1): # Display the reason to the user. Need to use args for python2.5 # compat. - raise InvalidCertificateException(req.host, '', + raise InvalidCertificateException(req.host, "", url_error.reason.args[1]) except ImportError: pass