diff --git a/cipd b/cipd index f5ad3311a..df485c1bd 100755 --- a/cipd +++ b/cipd @@ -6,34 +6,31 @@ set -e -o pipefail -CYGWIN=false MYPATH=$(dirname "${BASH_SOURCE[0]}") - -: ${CIPD_CLIENT_VER:=`cat $MYPATH/cipd_client_version`} -: ${CIPD_CLIENT_SRV:='https://chrome-infra-packages.appspot.com'} +CYGWIN=false UNAME=`uname -s | tr '[:upper:]' '[:lower:]'` -case $UNAME in +case "${UNAME}" in linux) - PLAT=linux + OS=linux ;; cygwin*) - PLAT=windows + OS=windows CYGWIN=true ;; msys*|mingw*) - PLAT=windows + OS=windows ;; darwin) - PLAT=mac + OS=mac ;; *) - echo "cipd not supported on $UNAME" + >&2 echo "CIPD not supported on ${UNAME}" exit 1 esac UNAME=`uname -m | tr '[:upper:]' '[:lower:]'` -case $UNAME in +case "${UNAME}" in x86_64|amd64) ARCH=amd64 ;; @@ -53,89 +50,184 @@ case $UNAME in ARCH=armv6l ;; arm*) - ARCH=$UNAME + ARCH="${UNAME}" ;; *86) ARCH=386 ;; mips*) # detect mips64le vs mips64. - ARCH=$UNAME + ARCH="${UNAME}" if lscpu | grep -q "Little Endian"; then ARCH+=le fi ;; *) - echo "UNKNOWN Machine architecture: $UNAME" + >&2 echo "UNKNOWN Machine architecture: ${UNAME}" exit 1 esac -URL="$CIPD_CLIENT_SRV/client?platform=${PLAT}-${ARCH}&version=$CIPD_CLIENT_VER" -CLIENT="$MYPATH/.cipd_client" +# CIPD_BACKEND can be changed to ...-dev for manual testing. +CIPD_BACKEND="https://chrome-infra-packages.appspot.com" +VERSION_FILE="${MYPATH}/cipd_client_version" + +CLIENT="${MYPATH}/.cipd_client" +VERSION=`cat "${VERSION_FILE}"` +PLATFORM="${OS}-${ARCH}" + +URL="${CIPD_BACKEND}/client?platform=${PLATFORM}&version=${VERSION}" +USER_AGENT="depot_tools/$(git -C ${MYPATH} rev-parse HEAD 2>/dev/null || echo "???")" + + +# calc_sha256 is "portable" variant of sha256sum. It uses sha256sum when +# available (most Linuxes and cygwin) and 'shasum -a 256' otherwise (for OSX). +# +# Args: +# Path to a file. +# Stdout: +# Lowercase SHA256 hex digest of the file. +function calc_sha256() { + if hash sha256sum 2> /dev/null ; then + sha256sum "$1" | cut -d' ' -f1 + elif hash shasum 2> /dev/null ; then + shasum -a 256 "$1" | cut -d' ' -f1 + else + >&2 echo -n "" + >&2 echo -n "Don't know how to calculate SHA256 on your platform. " + >&2 echo -n "Please use your package manager to install one before continuing:" + >&2 echo + >&2 echo " sha256sum" + >&2 echo -n " shasum" + >&2 echo "" + return 1 + fi +} + + +# expected_sha256 reads the expected SHA256 hex digest from *.digests file. +# +# Args: +# Name of the platform to get client's digest for. +# Stdout: +# Lowercase SHA256 hex digest. +function expected_sha256() { + local line + while read -r line; do + if [[ "${line}" =~ ^([0-9a-z\-]+)[[:blank:]]+sha256[[:blank:]]+([0-9a-f]+)$ ]] ; then + local plat="${BASH_REMATCH[1]}" + local hash="${BASH_REMATCH[2]}" + if [ "${plat}" == "$1" ]; then + echo "${hash}" + return 0 + fi + fi + done < "${VERSION_FILE}.digests" + + >&2 echo -n "" + >&2 echo -n "Platform $1 is not supported by the CIPD client bootstrap: " + >&2 echo -n "there's no pinned SHA256 hash for it in the *.digests file." + >&2 echo "" + + return 1 +} -USER_AGENT="depot_tools/$(git -C $MYPATH rev-parse HEAD 2>/dev/null || echo "???")" # clean_bootstrap bootstraps the client from scratch using 'curl' or 'wget'. +# +# It checks that the SHA256 of the downloaded file is known. Exits the script +# if the client can't be downloaded or its hash doesn't match the expected one. function clean_bootstrap() { - echo "Bootstrapping cipd client for ${PLAT}-${ARCH} from ${URL}..." + local expected_hash=$(expected_sha256 "${PLATFORM}") + if [ -z "${expected_hash}" ] ; then + exit 1 + fi - # Download the client into a temporary file, then move it into the final - # location atomically. + # Download the client into a temporary file, check its hash, then move it into + # the final location. # # This wonky tempdir method works on Linux and Mac. local CIPD_CLIENT_TMP=$(\ - mktemp -p "$MYPATH" 2>/dev/null || \ - mktemp "$MYPATH/.cipd_client.XXXXXXX") + mktemp -p "${MYPATH}" 2>/dev/null || \ + mktemp "${MYPATH}/.cipd_client.XXXXXXX") if hash curl 2> /dev/null ; then - curl "$URL" -s --show-error -f -A "$USER_AGENT" -L -o "$CIPD_CLIENT_TMP" + curl "${URL}" -s --show-error -f -A "${USER_AGENT}" -L -o "${CIPD_CLIENT_TMP}" elif hash wget 2> /dev/null ; then - wget "$URL" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}" + wget "${URL}" -q -U "${USER_AGENT}" -O "${CIPD_CLIENT_TMP}" else - echo Your platform is missing a supported fetch command. Please use your package - echo manager to install one before continuing: - echo - echo curl - echo wget - echo - echo Alternately, manually download: - echo "$URL" - echo To $CLIENT, and then re-run this command. + >&2 echo -n "" + >&2 echo -n "Your platform is missing a supported fetch command. " + >&2 echo "Please use your package manager to install one before continuing:" + >&2 echo + >&2 echo " curl" + >&2 echo " wget" + >&2 echo + >&2 echo "Alternately, manually download:" + >&2 echo " ${URL}" + >&2 echo -n "To ${CLIENT}, and then re-run this command." + >&2 echo "" + rm "${CIPD_CLIENT_TMP}" + exit 1 + fi + + local actual_hash=$(calc_sha256 "${CIPD_CLIENT_TMP}") + if [ -z "${actual_hash}" ] ; then rm "${CIPD_CLIENT_TMP}" exit 1 fi - chmod +x "$CIPD_CLIENT_TMP" + if [ "${actual_hash}" != "${expected_hash}" ]; then + >&2 echo -n "" + >&2 echo "SHA256 digest of the downloaded CIPD client is incorrect:" + >&2 echo " Expecting ${expected_hash}" + >&2 echo " Got ${actual_hash}" + >&2 echo -n "Refusing to run it. Check that *.digests file is up-to-date." + >&2 echo "" + rm "${CIPD_CLIENT_TMP}" + exit 1 + fi set +e - mv "$CIPD_CLIENT_TMP" "$CLIENT" + chmod +x "${CIPD_CLIENT_TMP}" + mv "${CIPD_CLIENT_TMP}" "${CLIENT}" set -e } + +# self_update launches CIPD client's built-in selfupdate mechanism. +# +# It is more efficient that redownloading the binary all the time. function self_update() { - "$CLIENT" selfupdate -version "$CIPD_CLIENT_VER" -service-url "$CIPD_CLIENT_SRV" + "${CLIENT}" selfupdate -version-file "${VERSION_FILE}" -service-url "${CIPD_BACKEND}" } -if [ ! -x "$CLIENT" ]; then + +if [ ! -x "${CLIENT}" ]; then clean_bootstrap fi -export CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT -if ! self_update ; then - echo -n "CIPD selfupdate failed. " 1>&2 - echo "Trying to bootstrap the CIPD client from scratch... " 1>&2 +export CIPD_HTTP_USER_AGENT_PREFIX="${USER_AGENT}" +if ! self_update 2> /dev/null ; then + >&2 echo -n "" + >&2 echo -n "CIPD selfupdate failed. " + >&2 echo -n "Trying to bootstrap the CIPD client from scratch..." + >&2 echo "" clean_bootstrap if ! self_update ; then # need to run it again to setup .cipd_version file - echo -n "Bootstrap from scratch failed, something is seriously broken " 1>&2 - echo "run \`CIPD_HTTP_USER_AGENT_PREFIX=$USER_AGENT/manual $CLIENT selfupdate -version '$CIPD_CLIENT_VER'\` to diagnose if this is repeating." 1>&2 - echo "" 1>&2 + >&2 echo -n "" + >&2 echo -n "Bootstrap from scratch failed, something is seriously broken. " + >&2 echo "Run the following commands to diagnose if this is repeating:" + >&2 echo " export CIPD_HTTP_USER_AGENT_PREFIX=${USER_AGENT}/manual" + >&2 echo -n " ${CLIENT} selfupdate -version-file ${VERSION_FILE}" + >&2 echo "" + exit 1 fi fi # CygWin requires changing absolute paths to Windows form. Relative paths # are typically okay as Windows generally accepts both forward and back # slashes. This could possibly be constrained to only /tmp/ and /cygdrive/. -if $CYGWIN; then +if ${CYGWIN}; then args=("$@") for i in `seq 2 $#`; do arg="${@:$i:1}" @@ -145,7 +237,7 @@ if $CYGWIN; then set -- "${@:1:$last}" `cygpath -w "$arg"` "${@:$next}" fi done - echo "$CLIENT" "${@}" + echo "${CLIENT}" "${@}" fi -exec "$CLIENT" "${@}" +exec "${CLIENT}" "${@}" diff --git a/cipd_client_version b/cipd_client_version index ab26c00d9..dc70306aa 100644 --- a/cipd_client_version +++ b/cipd_client_version @@ -1 +1 @@ -git_revision:9a931a5307c46b16b1c12e01e8239d4a73830b89 +git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 diff --git a/cipd_client_version.digests b/cipd_client_version.digests new file mode 100644 index 000000000..6624d47d6 --- /dev/null +++ b/cipd_client_version.digests @@ -0,0 +1,21 @@ +# This file was generated by +# +# cipd selfupdate-roll -version-file cipd_client_version \ +# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 +# +# Do not modify manually. All changes will be overwritten. +# Use 'cipd selfupdate-roll ...' to modify. + +linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971 +linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed +linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64 +linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a +linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a +linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d +linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810 +linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6 +linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec +linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e +mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66 +windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c +windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216 diff --git a/tests/cipd_bootstrap_test.py b/tests/cipd_bootstrap_test.py index 4683bce3e..9f7f25d35 100755 --- a/tests/cipd_bootstrap_test.py +++ b/tests/cipd_bootstrap_test.py @@ -15,8 +15,26 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # CIPD client version to use for self-update from an "old" checkout to the tip. # -# This version is from Jan 2018. -OLD_VERSION = 'git_revision:a1f61935faa60feb73e37556fdf791262c2dedce' +# This version is from Aug 2018. Digests were generated using: +# cipd selfupdate-roll -version-file tmp \ +# -version git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8 +# cat tmp.cat +OLD_VERSION = 'git_revision:ea6c07cfcb596be6b63a1e6deb95bba79524b0c8' +OLD_DIGESTS = """ +linux-386 sha256 ee90bd655b90baf7586ab80c289c00233b96bfac3fa70e64cc5c48feb1998971 +linux-amd64 sha256 73bd62cb72cde6f12d9b42cda12941c53e1e21686f6f2b1cd98db5c6718b7bed +linux-arm64 sha256 1f2619f3e7f5f6876d0a446bacc6cc61eb32ca1464315d7230034a832500ed64 +linux-armv6l sha256 98c873097c460fe8f6b4311f6e00b4df41ca50e9bd2d26f06995913a9d647d3a +linux-mips64 sha256 05e37c85502eb2b72abd8a51ff13a4914c5e071e25326c9c8fc257290749138a +linux-mips64le sha256 5b3af8be6ea8a62662006f1a86fdc387dc765edace9f530acbeca77c0850a32d +linux-mipsle sha256 cfa6539af00db69b7da00d46316f1aaaa90b38a5e6b33ce4823be17533e71810 +linux-ppc64 sha256 faa49f2b59a25134e8a13b68f5addb00c434c7feeee03940413917eca1d333e6 +linux-ppc64le sha256 6fa51348e6039b864171426b02cfbfa1d533b9f86e3c72875e0ed116994a2fec +linux-s390x sha256 6cd4bfff7e2025f2d3da55013036e39eea4e8f631060a5e2b32b9975fab08b0e +mac-amd64 sha256 6427b87fdaa1615a229d45c2fab1ba7fdb748ce785f2c09cd6e10adc48c58a66 +windows-386 sha256 809c727a31e5f8c34656061b96839fbca63833140b90cab8e2491137d6e4fc4c +windows-amd64 sha256 3e21561b45acb2845c309a04cbedb2ce1e0567b7b24bf89857e7673607b09216 +""" class CipdBootstrapTest(unittest.TestCase): @@ -33,17 +51,28 @@ class CipdBootstrapTest(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def stage_files(self, cipd_version=None): + def stage_files(self, cipd_version=None, digests=None): """Copies files needed for cipd bootstrap into the temp dir. Args: cipd_version: if not None, a value to put into cipd_client_version file. """ - for f in ('cipd', 'cipd.bat', 'cipd.ps1', 'cipd_client_version'): + names = ( + 'cipd', + 'cipd.bat', + 'cipd.ps1', + 'cipd_client_version', + 'cipd_client_version.digests', + ) + for f in names: shutil.copy2(os.path.join(ROOT_DIR, f), os.path.join(self.tempdir, f)) if cipd_version is not None: with open(os.path.join(self.tempdir, 'cipd_client_version'), 'wt') as f: f.write(cipd_version+'\n') + if digests is not None: + p = os.path.join(self.tempdir, 'cipd_client_version.digests') + with open(p, 'wt') as f: + f.write(digests+'\n') def call_cipd_help(self): """Calls 'cipd help' bootstrapping the client in tempdir. @@ -66,7 +95,7 @@ class CipdBootstrapTest(unittest.TestCase): def test_self_update(self): """Updating the existing client in-place.""" - self.stage_files(cipd_version=OLD_VERSION) + self.stage_files(cipd_version=OLD_VERSION, digests=OLD_DIGESTS) ret, out = self.call_cipd_help() if ret: self.fail('Update to %s fails:\n%s' % (OLD_VERSION, out))