From 71756329cba7ca6c3a89b31f6281e4edcb0a6215 Mon Sep 17 00:00:00 2001 From: "scottmg@chromium.org" Date: Sun, 2 Mar 2014 22:59:57 +0000 Subject: [PATCH] Add toolchain simple mirror option used for bots I tried a git-ish tree/blob model, but way too slow for a ton of files. This maps easily on to the current way the tree is mapped, saves temp space on bots (because the full isos aren't downloaded), and is a lot faster too. It means we can pull old revisions too now. TBR=iannucci@chromium.org BUG=348350 Review URL: https://codereview.chromium.org/185423004 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@254421 0039d316-1c4b-4281-b951-d872f2087c98 --- win_toolchain/get_toolchain_if_necessary.py | 10 +++--- win_toolchain/toolchain2013.py | 35 ++++++++++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py index 266806b06..27a4034a5 100755 --- a/win_toolchain/get_toolchain_if_necessary.py +++ b/win_toolchain/get_toolchain_if_necessary.py @@ -152,7 +152,8 @@ def main(): help='write information about toolchain to FILE') options, args = parser.parse_args() - desired_hashes = set(args) + # We assume that the Pro hash is the first one. + desired_hashes = args # Move to depot_tools\win_toolchain where we'll store our files, and where # the downloader script is. @@ -171,7 +172,7 @@ def main(): print('Windows toolchain out of date or doesn\'t exist, updating (%s)...' % ('Pro' if should_get_pro else 'Express')) print(' current_hash: %s' % current_hash) - print(' desired_hashes: %s' % desired_hashes) + print(' desired_hashes: %s' % ', '.join(desired_hashes)) DelayBeforeRemoving(target_dir) # This stays resident and will make the rmdir below fail. with open(os.devnull, 'wb') as nul: @@ -181,7 +182,8 @@ def main(): subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True) args = [sys.executable, 'toolchain2013.py', - '--targetdir', target_dir] + '--targetdir', target_dir, + '--sha1', desired_hashes[0]] if not should_get_pro: args.append('--express') subprocess.check_call(args) @@ -190,7 +192,7 @@ def main(): print >> sys.stderr, ( 'Got wrong hash after pulling a new toolchain. ' 'Wanted one of \'%s\', got \'%s\'.' % ( - desired_hashes, current_hash)) + ', '.join(desired_hashes), current_hash)) return 1 SaveTimestampsAndHash(target_dir, current_hash) diff --git a/win_toolchain/toolchain2013.py b/win_toolchain/toolchain2013.py index cfeeef3ad..0cda59039 100755 --- a/win_toolchain/toolchain2013.py +++ b/win_toolchain/toolchain2013.py @@ -248,7 +248,7 @@ class SourceImages(object): self.wdk_path = wdk_path -def GetSourceImages(local_dir, pro, bot_mode): +def GetSourceImages(local_dir, pro): """Downloads the various sources that we need. Of note: Because Express does not include ATL, there's an additional download @@ -256,9 +256,7 @@ def GetSourceImages(local_dir, pro, bot_mode): |pro| this is not necessary (and CHROME_HEADLESS always implies Pro). """ url = GetMainIsoUrl(pro) - if bot_mode: - return SourceImages(GetVSInternal(), GetSDKInternal(), wdk_path=None) - elif local_dir: + if local_dir: wdk_path = (os.path.join(local_dir, os.path.basename(WDK_ISO_URL)) if not pro else None) return SourceImages(os.path.join(local_dir, os.path.basename(url)), @@ -425,6 +423,16 @@ def GenerateSetEnvCmd(target_dir, pro): '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n') +def DoTreeMirror(target_dir, tree_sha1): + """In order to save temporary space on bots that do not have enough space to + download ISOs, unpack them, and copy to the target location, the whole tree + is uploaded as a zip to internal storage, and then mirrored here.""" + local_zip = DownloadUsingGsutil(tree_sha1 + '.zip') + sys.stdout.write('Extracting %s...\n' % local_zip) + sys.stdout.flush() + RunOrDie('7z x "%s" -y "-o%s" >nul' % (local_zip, target_dir)) + + def main(): parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) parser.add_option('--targetdir', metavar='DIR', @@ -437,6 +445,9 @@ def main(): help='use downloaded files from DIR') parser.add_option('--express', help='use VS Express instead of Pro', action='store_true') + parser.add_option('--sha1', + help='tree sha1 that can be used to mirror an internal ' + 'copy (used if --bot-mode)') parser.add_option('--bot-mode', help='Use internal servers to pull isos', default=bool(int(os.environ.get('CHROME_HEADLESS', 0))), @@ -446,18 +457,20 @@ def main(): target_dir = os.path.abspath(options.targetdir) if os.path.exists(target_dir): parser.error('%s already exists. Please [re]move it or use ' - '--targetdir to select a different target.\n' % - target_dir) + '--targetdir to select a different target.\n' % + target_dir) # Set the working directory to 7z subdirectory. 7-zip doesn't find its # codec dll very well, so this is the simplest way to make sure it runs # correctly, as we don't otherwise care about working directory. os.chdir(os.path.join(BASEDIR, '7z')) - images = GetSourceImages( - options.local, not options.express, options.bot_mode) - extracted = ExtractComponents(images) - CopyToFinalLocation(extracted, target_dir) + if options.bot_mode and options.sha1: + DoTreeMirror(target_dir, options.sha1) + else: + images = GetSourceImages(options.local, not options.express) + extracted = ExtractComponents(images) + CopyToFinalLocation(extracted, target_dir) + GenerateSetEnvCmd(target_dir, not options.express) - GenerateSetEnvCmd(target_dir, not options.express) data = { 'path': target_dir, 'version': '2013e' if options.express else '2013',