|
|
|
|
@ -11,29 +11,82 @@ from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import base64
|
|
|
|
|
import contextlib
|
|
|
|
|
import functools
|
|
|
|
|
import http.cookiejar
|
|
|
|
|
import json
|
|
|
|
|
import logging
|
|
|
|
|
import os
|
|
|
|
|
import random
|
|
|
|
|
import re
|
|
|
|
|
import shutil
|
|
|
|
|
import socket
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
import tempfile
|
|
|
|
|
import threading
|
|
|
|
|
import time
|
|
|
|
|
import urllib.parse
|
|
|
|
|
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from io import StringIO
|
|
|
|
|
from multiprocessing.pool import ThreadPool
|
|
|
|
|
from typing import Any, Container, Dict, List, Optional, Tuple, Type, TypedDict
|
|
|
|
|
from typing import Any, Container, Dict, List, Optional
|
|
|
|
|
from typing import Tuple, Type, TypedDict, cast
|
|
|
|
|
|
|
|
|
|
import httplib2
|
|
|
|
|
import httplib2.socks
|
|
|
|
|
|
|
|
|
|
import auth
|
|
|
|
|
import gclient_utils
|
|
|
|
|
import metrics
|
|
|
|
|
import metrics_utils
|
|
|
|
|
import scm
|
|
|
|
|
|
|
|
|
|
import subprocess2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# HACK: httplib2 has significant bugs with its proxy support in
|
|
|
|
|
# python3. All httplib2 code should be rewritten to just use python
|
|
|
|
|
# stdlib which does not have these bugs.
|
|
|
|
|
#
|
|
|
|
|
# Prior to that, however, we will directly patch the buggy
|
|
|
|
|
# implementation of httplib2.socks.socksocket.__rewriteproxy which does
|
|
|
|
|
# not properly expect bytes as its argument instead of str.
|
|
|
|
|
#
|
|
|
|
|
# Note that __rewriteproxy is inherently buggy, as it relies on the
|
|
|
|
|
# python stdlib client to send the entire request header in a single
|
|
|
|
|
# call to socket.sendall, which is not explicitly guaranteed.
|
|
|
|
|
#
|
|
|
|
|
# Changes:
|
|
|
|
|
# * all string literals changed to bytes literals.
|
|
|
|
|
# * all __symbols changed to _socksocket__symbols.
|
|
|
|
|
# * Type annotations added to function signature.
|
|
|
|
|
def __fixed_rewrite_proxy(self: httplib2.socks.socksocket, header: bytes):
|
|
|
|
|
""" rewrite HTTP request headers to support non-tunneling proxies
|
|
|
|
|
(i.e. those which do not support the CONNECT method).
|
|
|
|
|
This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
|
|
|
|
|
"""
|
|
|
|
|
host, endpt = None, None
|
|
|
|
|
hdrs = header.split(b"\r\n")
|
|
|
|
|
for hdr in hdrs:
|
|
|
|
|
if hdr.lower().startswith(b"host:"):
|
|
|
|
|
host = hdr
|
|
|
|
|
elif hdr.lower().startswith(b"get") or hdr.lower().startswith(b"post"):
|
|
|
|
|
endpt = hdr
|
|
|
|
|
if host and endpt:
|
|
|
|
|
hdrs.remove(host)
|
|
|
|
|
hdrs.remove(endpt)
|
|
|
|
|
host = host.split(b" ")[1]
|
|
|
|
|
endpt = endpt.split(b" ")
|
|
|
|
|
if self._socksocket__proxy[4] != None \
|
|
|
|
|
and self._socksocket__proxy[5] != None:
|
|
|
|
|
hdrs.insert(0, self._socksocket__getauthheader())
|
|
|
|
|
hdrs.insert(0, b"Host: %s" % host)
|
|
|
|
|
hdrs.insert(0,
|
|
|
|
|
b"%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
|
|
|
|
|
return b"\r\n".join(hdrs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
httplib2.socks.socksocket._socksocket__rewriteproxy = __fixed_rewrite_proxy
|
|
|
|
|
|
|
|
|
|
# TODO: Should fix these warnings.
|
|
|
|
|
# pylint: disable=line-too-long
|
|
|
|
|
@ -125,6 +178,8 @@ class Authenticator(object):
|
|
|
|
|
Authenticator instance to use.
|
|
|
|
|
"""
|
|
|
|
|
authenticators: List[Type[Authenticator]] = [
|
|
|
|
|
SSOAuthenticator,
|
|
|
|
|
|
|
|
|
|
# LUCI Context takes priority since it's normally present only on bots,
|
|
|
|
|
# which then must use it.
|
|
|
|
|
LuciContextAuthenticator,
|
|
|
|
|
@ -142,6 +197,182 @@ class Authenticator(object):
|
|
|
|
|
f"Could not find suitable authenticator, tried: {authenticators}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SSOAuthenticator(Authenticator):
|
|
|
|
|
"""SSOAuthenticator implements auth with an SSO helper.
|
|
|
|
|
|
|
|
|
|
This helper has the same protocol as `git-remote-persistent-https`.
|
|
|
|
|
|
|
|
|
|
TEMPORARY configuration for Googlers (one `url` block for each
|
|
|
|
|
Gerrit host):
|
|
|
|
|
|
|
|
|
|
[url "sso://chromium/"]
|
|
|
|
|
insteadOf = https://chromium.googlesource.com/
|
|
|
|
|
insteadOf = http://chromium.googlesource.com/
|
|
|
|
|
[depot-tools]
|
|
|
|
|
useNewAuthStack = 1
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# The running persistent sso proxy process.
|
|
|
|
|
#
|
|
|
|
|
# Lazily spawned by `authenticate`.
|
|
|
|
|
_sso_info_lock = threading.Lock()
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class SSOInfo:
|
|
|
|
|
proxy: httplib2.ProxyInfo
|
|
|
|
|
cookies: http.cookiejar.CookieJar
|
|
|
|
|
headers: Dict[str, str]
|
|
|
|
|
|
|
|
|
|
_sso_info: Optional[SSOInfo] = None
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
@functools.lru_cache
|
|
|
|
|
def _resolve_sso_binary_path():
|
|
|
|
|
return shutil.which('git-remote-sso') or ''
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def is_applicable(cls) -> bool:
|
|
|
|
|
"""If the git-remote-sso binary is in $PATH, we consider this
|
|
|
|
|
authenticator to be applicable."""
|
|
|
|
|
if scm.GIT.GetConfig(os.getcwd(), 'depot-tools.usenewauthstack') != '1':
|
|
|
|
|
LOGGER.debug('SSOAuthenticator: skipping due to missing opt-in.')
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
pth = cls._resolve_sso_binary_path()
|
|
|
|
|
if pth:
|
|
|
|
|
LOGGER.debug('SSOAuthenticator: enabled %r.', pth)
|
|
|
|
|
return bool(pth)
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def _parse_config(cls, config: str) -> SSOInfo:
|
|
|
|
|
parsed: Dict[str, str] = dict(line.strip().split('=', 1)
|
|
|
|
|
for line in config.splitlines())
|
|
|
|
|
|
|
|
|
|
fullAuthHeader = cast(
|
|
|
|
|
str,
|
|
|
|
|
scm.GIT.Capture([
|
|
|
|
|
'config',
|
|
|
|
|
'-f',
|
|
|
|
|
parsed['include.path'],
|
|
|
|
|
'http.extraHeader',
|
|
|
|
|
]))
|
|
|
|
|
headerKey, headerValue = fullAuthHeader.split(':', 1)
|
|
|
|
|
headers = {headerKey.strip(): headerValue.strip()}
|
|
|
|
|
|
|
|
|
|
proxy_host, proxy_port = parsed['http.proxy'].split(':', 1)
|
|
|
|
|
cj = http.cookiejar.MozillaCookieJar(parsed['http.cookiefile'])
|
|
|
|
|
cj.load()
|
|
|
|
|
|
|
|
|
|
return cls.SSOInfo(proxy=httplib2.ProxyInfo(
|
|
|
|
|
httplib2.socks.PROXY_TYPE_HTTP_NO_TUNNEL, proxy_host.encode(),
|
|
|
|
|
int(proxy_port)),
|
|
|
|
|
cookies=cj,
|
|
|
|
|
headers=headers)
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def _launch_sso_helper(cls) -> SSOInfo:
|
|
|
|
|
"""Launches the git-remote-sso process and extracts the parsed SSOInfo.
|
|
|
|
|
|
|
|
|
|
Raises an exception if something goes wrong.
|
|
|
|
|
"""
|
|
|
|
|
tdir = tempfile.mkdtemp(suffix='gerrit_util')
|
|
|
|
|
tf = os.path.join(tdir, 'git-remote-sso.stderr')
|
|
|
|
|
|
|
|
|
|
with tempdir() as tdir:
|
|
|
|
|
cmd = [
|
|
|
|
|
cls._resolve_sso_binary_path(),
|
|
|
|
|
'-print_config',
|
|
|
|
|
'sso://*.git.corp.google.com',
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
stderr_file = open(tf, mode='w')
|
|
|
|
|
|
|
|
|
|
# NOTE: The git-remote-sso process does the following in order:
|
|
|
|
|
#
|
|
|
|
|
# 1. writes file to disk
|
|
|
|
|
# 2. writes config to stdout, and closes stdout
|
|
|
|
|
# 3. waits for stdin to be closed
|
|
|
|
|
# 4. deletes file on disk before exiting
|
|
|
|
|
#
|
|
|
|
|
# Thus, we must fully parse stdout AND consume the
|
|
|
|
|
# cookiefile BEFORE stopping the process.
|
|
|
|
|
with subprocess2.Popen(cmd,
|
|
|
|
|
stdout=subprocess2.PIPE,
|
|
|
|
|
stderr=stderr_file,
|
|
|
|
|
stdin=subprocess2.PIPE,
|
|
|
|
|
encoding='utf-8') as proc:
|
|
|
|
|
timedout = False
|
|
|
|
|
|
|
|
|
|
def _fire_timeout():
|
|
|
|
|
nonlocal timedout
|
|
|
|
|
timedout = True
|
|
|
|
|
proc.kill()
|
|
|
|
|
|
|
|
|
|
timer = threading.Timer(5, _fire_timeout)
|
|
|
|
|
timer.start()
|
|
|
|
|
try:
|
|
|
|
|
ret = cls._parse_config(proc.stdout.read())
|
|
|
|
|
finally:
|
|
|
|
|
timer.cancel()
|
|
|
|
|
|
|
|
|
|
if timedout:
|
|
|
|
|
LOGGER.error(
|
|
|
|
|
'SSOAuthenticator: Timeout: %r: reading config.', cmd)
|
|
|
|
|
raise subprocess.TimeoutExpired(cmd=cmd, timeout=5)
|
|
|
|
|
|
|
|
|
|
proc.stdin.close()
|
|
|
|
|
if proc.wait(timeout=2) is None:
|
|
|
|
|
proc.kill()
|
|
|
|
|
LOGGER.warning(
|
|
|
|
|
'SSOAuthenticator: Helper did not exit properly after waiting; forcefully killed'
|
|
|
|
|
' (This is cleanup and likely not the root cause of any error)'
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def _get_sso_info(cls) -> SSOInfo:
|
|
|
|
|
with cls._sso_info_lock:
|
|
|
|
|
info = cls._sso_info
|
|
|
|
|
if not info:
|
|
|
|
|
info = cls._launch_sso_helper()
|
|
|
|
|
cls._sso_info = info
|
|
|
|
|
return info
|
|
|
|
|
|
|
|
|
|
def authenticate(self, conn: HttpConn):
|
|
|
|
|
sso_info = self._get_sso_info()
|
|
|
|
|
conn.proxy_info = sso_info.proxy
|
|
|
|
|
conn.req_headers.update(sso_info.headers)
|
|
|
|
|
|
|
|
|
|
# Now we must rewrite:
|
|
|
|
|
# https://xxx.googlesource.com ->
|
|
|
|
|
# http://xxx.git.corp.google.com
|
|
|
|
|
parsed = urllib.parse.urlparse(conn.req_uri)
|
|
|
|
|
parsed = parsed._replace(scheme='http')
|
|
|
|
|
if (hostname :=
|
|
|
|
|
parsed.hostname) and hostname.endswith('.googlesource.com'):
|
|
|
|
|
assert not parsed.port, "SSOAuthenticator: netloc: port not supported"
|
|
|
|
|
assert not parsed.username, "SSOAuthenticator: netloc: username not supported"
|
|
|
|
|
assert not parsed.password, "SSOAuthenticator: netloc: password not supported"
|
|
|
|
|
|
|
|
|
|
hostname_parts = hostname.rsplit('.', 2) # X, googlesource, com
|
|
|
|
|
conn.req_host = hostname_parts[0] + '.git.corp.google.com'
|
|
|
|
|
parsed = parsed._replace(netloc=conn.req_host)
|
|
|
|
|
conn.req_uri = parsed.geturl()
|
|
|
|
|
|
|
|
|
|
# Finally, add cookies
|
|
|
|
|
sso_info.cookies.add_cookie_header(conn)
|
|
|
|
|
if 'Cookie' not in conn.req_headers:
|
|
|
|
|
LOGGER.debug("SSOAuthenticator: request headers missing 'Cookie'")
|
|
|
|
|
LOGGER.debug("SSO cookies: %r", sso_info.cookies._cookies)
|
|
|
|
|
LOGGER.debug("Request URI: %r", conn.req_uri)
|
|
|
|
|
LOGGER.debug("Request host: %r", conn.req_host)
|
|
|
|
|
LOGGER.debug("Request headers: %r", conn.req_headers)
|
|
|
|
|
|
|
|
|
|
def debug_summary_state(self) -> str:
|
|
|
|
|
return ''
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class CookiesAuthenticator(Authenticator):
|
|
|
|
|
"""Authenticator implementation that uses ".gitcookies" for token.
|
|
|
|
|
|
|
|
|
|
@ -434,6 +665,43 @@ class HttpConn(httplib2.Http):
|
|
|
|
|
'body': self.req_body,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# NOTE: We want to use HttpConn with CookieJar.add_cookie_header, so have
|
|
|
|
|
# compatible interface for that here.
|
|
|
|
|
#
|
|
|
|
|
# NOTE: Someone should really normalize this 'HttpConn' and httplib2
|
|
|
|
|
# implementation to just be plain python3 stdlib instead. All of this was
|
|
|
|
|
# written during the bad old days of python2.6/2.7, pre-vpython.
|
|
|
|
|
def has_header(self, header: str) -> bool:
|
|
|
|
|
return header in self.req_headers
|
|
|
|
|
|
|
|
|
|
def get_full_url(self) -> str:
|
|
|
|
|
return self.req_uri
|
|
|
|
|
|
|
|
|
|
def get_header(self,
|
|
|
|
|
header: str,
|
|
|
|
|
default: Optional[str] = None) -> Optional[str]:
|
|
|
|
|
return self.req_headers.get(header, default)
|
|
|
|
|
|
|
|
|
|
def add_unredirected_header(self, header: str, value: str):
|
|
|
|
|
# NOTE: httplib2 does not support unredirected headers.
|
|
|
|
|
self.req_headers[header] = value
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def unverifiable(self) -> bool:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def origin_req_host(self) -> str:
|
|
|
|
|
return self.req_host
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def type(self) -> str:
|
|
|
|
|
return urllib.parse.urlparse(self.req_uri).scheme
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def host(self) -> str:
|
|
|
|
|
return self.req_host
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def CreateHttpConn(host,
|
|
|
|
|
path,
|
|
|
|
|
@ -477,8 +745,9 @@ def CreateHttpConn(host,
|
|
|
|
|
|
|
|
|
|
if LOGGER.isEnabledFor(logging.DEBUG):
|
|
|
|
|
LOGGER.debug('%s %s', conn.req_method, conn.req_uri)
|
|
|
|
|
LOGGER.debug('conn.proxy_info%s', conn.proxy_info)
|
|
|
|
|
for key, val in conn.req_headers.items():
|
|
|
|
|
if key == 'Authorization':
|
|
|
|
|
if key in ('Authorization', 'Cookie'):
|
|
|
|
|
val = 'HIDDEN'
|
|
|
|
|
LOGGER.debug('%s: %s', key, val)
|
|
|
|
|
if conn.req_body:
|
|
|
|
|
|
Robert Iannucci
9 months ago
committed by
LUCI CQ