From 26cb836b5bbc68a4ebceed17b0858a5c44c84aaf Mon Sep 17 00:00:00 2001 From: "scottmg@chromium.org" Date: Tue, 4 Feb 2014 00:07:02 +0000 Subject: [PATCH] Support VS2013 Express for automatic toolchain too And for now, defaults to Express. Main changes: - Valid hash becomes a set of hashes one for Pro, one for Express - Include WDK to get an old copy of ATL as that doesn't come with Express BUG=323300 R=maruel@chromium.org Review URL: https://codereview.chromium.org/148453008 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@248622 0039d316-1c4b-4281-b951-d872f2087c98 --- .gitignore | 1 + win_toolchain/get_toolchain_if_necessary.py | 30 +++++---- win_toolchain/toolchain2013.py | 74 +++++++++++++++++---- win_toolchain/toolchain_vs2013.hash | 3 +- 4 files changed, 81 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 4f6bf5e842..0a94d2d952 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ /svn_bin /win_toolchain/vs2013_files /win_toolchain/.timestamps +/win_toolchain/.vspro # Ignore ctags/cscope index files /TAGS diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py index fe1ec85023..654d5659c9 100755 --- a/win_toolchain/get_toolchain_if_necessary.py +++ b/win_toolchain/get_toolchain_if_necessary.py @@ -103,7 +103,7 @@ def CalculateHash(root): def SaveTimestampsAndHash(root, sha1): - """Save timestamps and the final hash to be able to early-out more quickly + """Saves timestamps and the final hash to be able to early-out more quickly next time.""" file_list = GetFileList(root) timestamps_data = { @@ -125,36 +125,42 @@ def main(): # Move to depot_tools\win_toolchain where we'll store our files, and where # the downloader script is. os.chdir(os.path.normpath(os.path.join(BASEDIR))) + # TODO(scottmg): http://crbug.com/323300 Attempt to locate a src-internal + # pull and use that as a signal to install Pro also. + should_get_pro = os.path.isfile(os.path.join(BASEDIR, '.vspro')) toolchain_dir = '.' target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files')) sha1path = os.path.join(toolchain_dir, 'toolchain_vs2013.hash') - desired_hash = '' + desired_hashes = set() if os.path.isfile(sha1path): with open(sha1path, 'rb') as f: - desired_hash = f.read().strip() + desired_hashes = set(f.read().strip().splitlines()) # If the current hash doesn't match what we want in the file, nuke and pave. # Typically this script is only run when the .sha1 one file is updated, but # directly calling "gclient runhooks" will also run it, so we cache # based on timestamps to make that case fast. current_hash = CalculateHash(target_dir) - if current_hash != desired_hash: - print 'Windows toolchain out of date or doesn\'t exist, updating...' + if current_hash not in desired_hashes: + print('Windows toolchain out of date or doesn\'t exist, updating (%s)...' % + ('Pro' if should_get_pro else 'Express')) # This stays resident and will make the rmdir below fail. subprocess.call(['taskkill', '/f', '/im', 'mspdbsrv.exe']) if os.path.isdir(target_dir): subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True) - subprocess.check_call([ - sys.executable, - 'toolchain2013.py', - '--targetdir', target_dir]) + args = [sys.executable, + 'toolchain2013.py', + '--targetdir', target_dir] + if not should_get_pro: + args.append('--express') + subprocess.check_call(args) current_hash = CalculateHash(target_dir) - if current_hash != desired_hash: + if current_hash not in desired_hashes: print >> sys.stderr, ( 'Got wrong hash after pulling a new toolchain. ' - 'Wanted \'%s\', got \'%s\'.' % ( - desired_hash, current_hash)) + 'Wanted one of \'%s\', got \'%s\'.' % ( + desired_hashes, current_hash)) return 1 SaveTimestampsAndHash(target_dir, current_hash) diff --git a/win_toolchain/toolchain2013.py b/win_toolchain/toolchain2013.py index 47963f094a..cd91175086 100755 --- a/win_toolchain/toolchain2013.py +++ b/win_toolchain/toolchain2013.py @@ -17,6 +17,9 @@ import urllib2 BASEDIR = os.path.dirname(os.path.abspath(__file__)) +WDK_ISO_URL = ( + 'http://download.microsoft.com/download/' + '4/A/2/4A25C7D5-EFBE-4182-B6A9-AE6850409A78/GRMWDK_EN_7600_1.ISO') g_temp_dirs = [] @@ -57,8 +60,8 @@ def DeleteAllTempDirs(): g_temp_dirs = [] -def GetIsoUrl(pro): - """Gets the .iso URL. +def GetMainIsoUrl(pro): + """Gets the main .iso URL. If |pro| is False, downloads the Express edition. If |CHROME_HEADLESS| is set in the environment, then we assume we're on an internal bot, and download @@ -151,6 +154,7 @@ def DownloadSDK8(): while count < 5: rc = os.system(target_path + ' /quiet ' '/features OptionId.WindowsDesktopDebuggers ' + 'OptionId.WindowsDesktopSoftwareDevelopmentKit ' '/layout ' + standalone_path) if rc == 0: return standalone_path @@ -159,6 +163,13 @@ def DownloadSDK8(): sys.exit('After multiple retries, couldn\'t download Win8 SDK') +def DownloadWDKIso(): + wdk_temp_dir = TempDir() + target_path = os.path.join(wdk_temp_dir, 'GRMWDK_EN_7600_1.ISO') + Download(WDK_ISO_URL, target_path) + return target_path + + def DownloadUsingGsutil(filename): """Downloads the given file from Google Storage chrome-wintoolchain bucket.""" temp_dir = TempDir() @@ -185,23 +196,36 @@ def GetSDKInternal(): class SourceImages(object): - def __init__(self, vs_path, sdk8_path): + """Local paths for components. |wdk_path| may be None if it's unnecessary for + the given configuration.""" + def __init__(self, vs_path, sdk8_path, wdk_path): self.vs_path = vs_path self.sdk8_path = sdk8_path + self.wdk_path = wdk_path -def GetSourceImages(local_dir, pro): - url = GetIsoUrl(pro) - if os.environ.get('CHROME_HEADLESS'): - return SourceImages(GetVSInternal(), GetSDKInternal()) +def GetSourceImages(local_dir, pro, bot_mode): + """Downloads the various sources that we need. + + Of note: Because Express does not include ATL, there's an additional download + of the 7.1 WDK which is the latest publically accessible source for ATL. When + |pro| this is not necessary (and CHROME_HEADLESS always implies Pro). + """ + url = GetMainIsoUrl(pro) + if bot_mode: + return SourceImages(GetVSInternal(), GetSDKInternal(), wdk_path=None) elif local_dir: + wdk_path = (os.path.join(local_dir, os.path.basename(WDK_ISO_URL)) + if not pro else None) return SourceImages(os.path.join(local_dir, os.path.basename(url)), - os.path.join(local_dir, 'Standalone')) + os.path.join(local_dir, 'Standalone'), + wdk_path=wdk_path) else: # Note that we do the SDK first, as it might cause an elevation prompt. sdk8_path = DownloadSDK8() vs_path = DownloadMainIso(url) - return SourceImages(vs_path, sdk8_path) + wdk_path = DownloadWDKIso() if not pro else None + return SourceImages(vs_path, sdk8_path, wdk_path=wdk_path) def ExtractMsiList(root_dir, packages): @@ -238,10 +262,6 @@ def ExtractComponents(image): (r'vc_libraryDesktop\x64\vc_LibraryDesktopX64.msi', True), (r'vc_libraryDesktop\x86\vc_LibraryDesktopX86.msi', True), (r'vc_libraryextended\vc_libraryextended.msi', False), - (r'Windows_SDK\Windows Software Development Kit-x86_en-us.msi', True), - ('Windows_SDK\\' - r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi', - True), ] extracted_iso = ExtractIso(image.vs_path) result = ExtractMsiList(os.path.join(extracted_iso, 'packages'), vs_packages) @@ -250,10 +270,25 @@ def ExtractComponents(image): (r'X86 Debuggers And Tools-x86_en-us.msi', True), (r'X64 Debuggers And Tools-x64_en-us.msi', True), (r'SDK Debuggers-x86_en-us.msi', True), + (r'Windows Software Development Kit-x86_en-us.msi', True), + (r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi', + True), ] result.extend(ExtractMsiList(os.path.join(image.sdk8_path, 'Installers'), sdk_packages)) + if image.wdk_path: + # This image will only be set when using Express, when we need the WDK + # headers and libs to supplement Express with ATL. + wdk_packages = [ + (r'headers.msi', True), + (r'libs_x86fre.msi', True), + (r'libs_x64fre.msi', True), + ] + extracted_iso = ExtractIso(image.wdk_path) + result.extend(ExtractMsiList(os.path.join(extracted_iso, 'WDK'), + wdk_packages)) + return result @@ -264,6 +299,7 @@ def CopyToFinalLocation(extracted_dirs, target_dir): 'System64\\': 'sys64\\', 'System\\': 'sys32\\', 'Windows Kits\\8.0\\': 'win8sdk\\', + 'WinDDK\\7600.16385.win7_wdk.100208-1538\\': 'wdk\\', } matches = [] for extracted_dir in extracted_dirs: @@ -327,6 +363,9 @@ def GenerateSetEnvCmd(target_dir, pro): f.write(':x64\n' 'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;' '%~dp0..\\..\\VC\\bin\\x86_amd64;' + # Needed for mspdb120.dll. Must be after above though, so + # that cl.exe is the x86_amd64 one. + '%~dp0..\\..\\VC\\bin;' '%PATH%\n') else: # x64 native. @@ -351,6 +390,10 @@ def main(): help='use downloaded files from DIR') parser.add_option('--express', help='use VS Express instead of Pro', action='store_true') + parser.add_option('--bot-mode', + help='Use internal servers to pull isos', + default=bool(int(os.environ.get('CHROME_HEADLESS', 0))), + action='store_true') options, _ = parser.parse_args() try: target_dir = os.path.abspath(options.targetdir) @@ -362,11 +405,14 @@ def main(): # codec dll very well, so this is the simplest way to make sure it runs # correctly, as we don't otherwise care about working directory. os.chdir(os.path.join(BASEDIR, '7z')) - images = GetSourceImages(options.local, not options.express) + images = GetSourceImages( + options.local, not options.express, options.bot_mode) extracted = ExtractComponents(images) CopyToFinalLocation(extracted, target_dir) GenerateSetEnvCmd(target_dir, not options.express) + with open(os.path.join(target_dir, '.version'), 'w') as f: + f.write('express' if options.express else 'pro') finally: if options.clean: DeleteAllTempDirs() diff --git a/win_toolchain/toolchain_vs2013.hash b/win_toolchain/toolchain_vs2013.hash index 68d0f24c1e..d6b0bee36b 100644 --- a/win_toolchain/toolchain_vs2013.hash +++ b/win_toolchain/toolchain_vs2013.hash @@ -1 +1,2 @@ -f8cf785f85a00d98b52b3614b52cae3f544b1143 +627d3dc8e237717ceed024554fede5f41522f5e6 +1dd80dff1c6b2fc79b586c7d3df303ad68426f58