From 3d429cf5131a10f0e6ced89e8809820398ece78b Mon Sep 17 00:00:00 2001 From: Vadim Shtayura Date: Thu, 16 Aug 2018 00:15:08 +0000 Subject: [PATCH] [cipd] Check CIPD client hash against pinned SHA256 during updates. Linux and OSX only for now. This also rolls CIPD client to a version that supports pinned hashes (v2.2.5). CIPD_CLIENT_VER and CIPD_CLIENT_SRV are no longer supported as env vars, since it makes no sense when pinning hashes of the binaries at specific version on the specific backend. Also somewhat cleanup 'cipd' script to use "${VAR}", stderr and colored output consistently. R=iannucci@chromium.org, nodir@chromium.org BUG=870166, 874586 Change-Id: Iac67fbb6b5d07dcd81d44536737b03b146f1ad14 Reviewed-on: https://chromium-review.googlesource.com/1176727 Reviewed-by: Nodir Turakulov Commit-Queue: Vadim Shtayura --- cipd | 186 ++++++++++++++++++++++++++--------- cipd_client_version | 2 +- cipd_client_version.digests | 21 ++++ tests/cipd_bootstrap_test.py | 39 +++++++- 4 files changed, 195 insertions(+), 53 deletions(-) create mode 100644 cipd_client_version.digests diff --git a/cipd b/cipd index f5ad3311a..df485c1bd 100755 --- a/cipd +++ b/cipd @@ -6,34 +6,31 @@ set -e -o pipefail -CYGWIN=false MYPATH=$(dirname "${BASH_SOURCE[0]}") - -: ${CIPD_CLIENT_VER:=`cat $MYPATH/cipd_client_version`} -: ${CIPD_CLIENT_SRV:='https://chrome-infra-packages.appspot.com'} +CYGWIN=false UNAME=`uname -s | tr '[:upper:]' '[:lower:]'` -case $UNAME in +case "${UNAME}" in linux) - PLAT=linux + OS=linux ;; cygwin*) - PLAT=windows + OS=windows CYGWIN=true ;; msys*|mingw*) - PLAT=windows + OS=windows ;; darwin) - PLAT=mac + OS=mac ;; *) - echo "cipd not supported on $UNAME" + >&2 echo "CIPD not supported on ${UNAME}" exit 1 esac UNAME=`uname -m | tr '[:upper:]' '[:lower:]'` -case $UNAME in +case "${UNAME}" in x86_64|amd64) ARCH=amd64 ;; @@ -53,89 +50,184 @@ case $UNAME in ARCH=armv6l ;; arm*) - ARCH=$UNAME + ARCH="${UNAME}" ;; *86) ARCH=386 ;; mips*) # detect mips64le vs mips64. - ARCH=$UNAME + ARCH="${UNAME}" if lscpu | grep -q "Little Endian"; then ARCH+=le fi ;; *) - echo "UNKNOWN Machine architecture: $UNAME" + >&2 echo "UNKNOWN Machine architecture: ${UNAME}" exit 1 esac -URL="$CIPD_CLIENT_SRV/client?platform=${PLAT}-${ARCH}&version=$CIPD_CLIENT_VER" -CLIENT="$MYPATH/.cipd_client" +# CIPD_BACKEND can be changed to ...-dev for manual testing. +CIPD_BACKEND="https://chrome-infra-packages.appspot.com" +VERSION_FILE="${MYPATH}/cipd_client_version" + +CLIENT="${MYPATH}/.cipd_client" +VERSION=`cat "${VERSION_FILE}"` +PLATFORM="${OS}-${ARCH}" + +URL="${CIPD_BACKEND}/client?platform=${PLATFORM}&version=${VERSION}" +USER_AGENT="depot_tools/$(git -C ${MYPATH} rev-parse HEAD 2>/dev/null || echo "???")" + + +# calc_sha256 is "portable" variant of sha256sum. It uses sha256sum when +# available (most Linuxes and cygwin) and 'shasum -a 256' otherwise (for OSX). +# +# Args: +# Path to a file. +# Stdout: +# Lowercase SHA256 hex digest of the file. +function calc_sha256() { + if hash sha256sum 2> /dev/null ; then + sha256sum "$1" | cut -d' ' -f1 + elif hash shasum 2> /dev/null ; then + shasum -a 256 "$1" | cut -d' ' -f1 + else + >&2 echo -n "" + >&2 echo -n "Don't know how to calculate SHA256 on your platform. " + >&2 echo -n "Please use your package manager to install one before continuing:" + >&2 echo + >&2 echo " sha256sum" + >&2 echo -n " shasum" + >&2 echo "" + return 1 + fi +} + + +# expected_sha256 reads the expected SHA256 hex digest from *.digests file. +# +# Args: +# Name of the platform to get client's digest for. +# Stdout: +# Lowercase SHA256 hex digest. +function expected_sha256() { + local line + while read -r line; do + if [[ "${line}" =~ ^([0-9a-z\-]+)[[:blank:]]+sha256[[:blank:]]+([0-9a-f]+)$ ]] ; then + local plat="${BASH_REMATCH[1]}" + local hash="${BASH_REMATCH[2]}" + if [ "${plat}" == "$1" ]; then + echo "${hash}" + return 0 + fi + fi + done < "${VERSION_FILE}.digests" + + >&2 echo -n "" + >&2 echo -n "Platform $1 is not supported by the CIPD client bootstrap: " + >&2 echo -n "there's no pinned SHA256 hash for it in the *.digests file." + >&2 echo "" + + return 1 +} -USER_AGENT="depot_tools/$(git -C $MYPATH rev-parse HEAD 2>/dev/null || echo "???")" # clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'. +# +# It checks that the SHA256 of the downloaded file is known. Exits the script +# if the client can't be downloaded or its hash doesn't match the expected one. function clean_bootstrap() { - echo "Bootstrapping cipd client for ${PLAT}-${ARCH} from ${URL}..." + local expected_hash=$(expected_sha256 "${PLATFORM}") + if [ -z "${expected_hash}" ] ; then + exit 1 + fi - # Download the client into a temporary file, then move it into the final - # location atomically. + # Download the client into a temporary file, check its hash, then move it into + # the final location. # # This wonky tempdir method works on Linux and Mac. local CIPD_CLIENT_TMP=$(\ - mktemp -p "$MYPATH" 2>/dev/null || \ - mktemp "$MYPATH/.cipd_client.XXXXXXX") + mktemp -p "${MYPATH}" 2>/dev/null || \ + mktemp "${MYPATH}/.cipd_client.XXXXXXX") if hash curl 2> /dev/null ; then - curl "$URL" -s --show-error -f -A "$USER_AGENT" -L -o "$CIPD_CLIENT_TMP" + curl "${URL}" -s --show-error -f -A "${USER_AGENT}" -L -o "${CIPD_CLIENT_TMP}" elif hash wget 2> /dev/null ; then - wget "$URL" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}" + wget "${URL}" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}" else - echo Your platform is missing a supported fetch command. Please use your package - echo manager to install one before continuing: - echo - echo curl - echo wget - echo - echo Alternately, manually download: - echo "$URL" - echo To $CLIENT, and then re-run this command. + >&2 echo -n "" + >&2 echo -n "Your platform is missing a supported fetch command. " + >&2 echo "Please use your package manager to install one before continuing:" + >&2 echo + >&2 echo " curl" + >&2 echo " wget" + >&2 echo + >&2 echo "Alternately, manually download:" + >&2 echo " ${URL}" + >&2 echo -n "To ${CLIENT}, and then re-run this command." + >&2 echo "" + rm "${CIPD_CLIENT_TMP}" + exit 1 + fi + + local actual_hash=$(calc_sha256 "${CIPD_CLIENT_TMP}") + if [ -z "${actual_hash}" ] ; then rm "${CIPD_CLIENT_TMP}" exit 1 fi - chmod +x "$CIPD_CLIENT_TMP" + if [ "${actual_hash}" != "${expected_hash}" ]; then + >&2 echo -n "" + >&2 echo "SHA256 digest of the downloaded CIPD client is incorrect:" + >&2 echo " Expecting ${expected_hash}" + >&2 echo " Got ${actual_hash}" + >&2 echo -n "Refusing to run it. Check that *.digests file is up-to-date." + >&2 echo "" + rm "${CIPD_CLIENT_TMP}" + exit 1 + fi set +e - mv "$CIPD_CLIENT_TMP" "$CLIENT" + chmod +x "${CIPD_CLIENT_TMP}" + mv "${CIPD_CLIENT_TMP}" "${CLIENT}" set -e } + +# self_update launches CIPD client's built-in selfupdate mechanism. +# +# It is more efficient that redownloading the binary all the time. function self_update() { - "$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" -service-url "$CIPD_CLIENT_SRV" + "${CLIENT}" selfupdate -version-file "${VERSION_FILE}" -service-url "${CIPD_BACKEND}" } -if [ ! -x "$CLIENT" ]; then + +if [ ! -x "${CLIENT}" ]; then clean_bootstrap fi -export CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT -if ! self_update ; then - echo -n "CIPD selfupdate failed. " 1>&2 - echo "Trying to bootstrap the CIPD client from scratch... " 1>&2 +export CIPD_HTTP_USER_AGENT_PREFIX="${USER_AGENT}" +if ! self_update 2> /dev/null ; then + >&2 echo -n "" + >&2 echo -n "CIPD selfupdate failed. " + >&2 echo -n "Trying to bootstrap the CIPD client from scratch..." + >&2 echo "" clean_bootstrap if ! self_update ; then # need to run it again to setup .cipd_version file - echo -n "Bootstrap from scratch failed, something is seriously broken " 1>&2 - echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose if this is repeating." 1>&2 - echo "" 1>&2 + >&2 echo -n "" + >&2 echo -n "Bootstrap from scratch failed, something is seriously broken. " + >&2 echo "Run the following commands to diagnose if this is repeating:" + >&2 echo " export CIPD_HTTP_USER_AGENT_PREFIX=${USER_AGENT}/manual" + >&2 echo -n " ${CLIENT} selfupdate -version-file ${VERSION_FILE}" + >&2 echo "" + exit 1 fi fi # CygWin requires changing absolute paths to Windows form. Relative paths # are typically okay as Windows generally accepts both forward and back # slashes. This could possibly be constrained to only /tmp/ and /cygdrive/. -if $CYGWIN; then +if ${CYGWIN}; then args=("$@") for i in `seq 2 $#`; do arg="${@:$i:1}" @@ -145,7 +237,7 @@ if $CYGWIN; then set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}" fi done - echo "$CLIENT" "${@}" + echo "${CLIENT}" "${@}" fi -exec "$CLIENT" "${@}" +exec "${CLIENT}" "${@}" diff --git a/cipd_client_version b/cipd_client_version index ab26c00d9..dc70306aa 100644 --- a/cipd_client_version +++ b/cipd_client_version @@ -1 +1 @@ -git_revision:9a931a5307c46b16b1c12e01e8239d4a73830b89 +git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 diff --git a/cipd_client_version.digests b/cipd_client_version.digests new file mode 100644 index 000000000..6624d47d6 --- /dev/null +++ b/cipd_client_version.digests @@ -0,0 +1,21 @@ +# This file was generated by +# +# cipd selfupdate-roll -version-file cipd_client_version \ +# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 +# +# Do not modify manually. All changes will be overwritten. +# Use 'cipd selfupdate-roll ...' to modify. + +linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971 +linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed +linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64 +linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a +linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a +linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d +linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810 +linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6 +linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec +linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e +mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66 +windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c +windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216 diff --git a/tests/cipd_bootstrap_test.py b/tests/cipd_bootstrap_test.py index 4683bce3e..9f7f25d35 100755 --- a/tests/cipd_bootstrap_test.py +++ b/tests/cipd_bootstrap_test.py @@ -15,8 +15,26 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # CIPD client version to use for self-update from an "old" checkout to the tip. # -# This version is from Jan 2018. -OLD_VERSION = 'git_revision:a1f61935faa60feb73e37556fdf791262c2dedce' +# This version is from Aug 2018. Digests were generated using: +# cipd selfupdate-roll -version-file tmp \ +# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 +# cat tmp.cat +OLD_VERSION = 'git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8' +OLD_DIGESTS = """ +linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971 +linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed +linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64 +linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a +linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a +linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d +linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810 +linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6 +linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec +linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e +mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66 +windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c +windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216 +""" class CipdBootstrapTest(unittest.TestCase): @@ -33,17 +51,28 @@ class CipdBootstrapTest(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def stage_files(self, cipd_version=None): + def stage_files(self, cipd_version=None, digests=None): """Copies files needed for cipd bootstrap into the temp dir. Args: cipd_version: if not None, a value to put into cipd_client_version file. """ - for f in ('cipd', 'cipd.bat', 'cipd.ps1', 'cipd_client_version'): + names = ( + 'cipd', + 'cipd.bat', + 'cipd.ps1', + 'cipd_client_version', + 'cipd_client_version.digests', + ) + for f in names: shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f)) if cipd_version is not None: with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f: f.write(cipd_version+'\n') + if digests is not None: + p = os.path.join(self.tempdir, 'cipd_client_version.digests') + with open(p, 'wt') as f: + f.write(digests+'\n') def call_cipd_help(self): """Calls 'cipd help' bootstrapping the client in tempdir. @@ -66,7 +95,7 @@ class CipdBootstrapTest(unittest.TestCase): def test_self_update(self): """Updating the existing client in-place.""" - self.stage_files(cipd_version=OLD_VERSION) + self.stage_files(cipd_version=OLD_VERSION, digests=OLD_DIGESTS) ret, out = self.call_cipd_help() if ret: self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out))