diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py index 7bb3eba5a..cb4b5da2c 100755 --- a/win_toolchain/get_toolchain_if_necessary.py +++ b/win_toolchain/get_toolchain_if_necessary.py @@ -33,13 +33,20 @@ import os import shutil import subprocess import sys +import tempfile import time +import zipfile BASEDIR = os.path.dirname(os.path.abspath(__file__)) DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..') sys.path.append(DEPOT_TOOLS_PATH) -import download_from_google_storage +try: + import download_from_google_storage +except ImportError: + # Allow use of utility functions in this script from package_from_installed + # on bare VM that doesn't have a full depot_tools. + pass if sys.platform != 'cygwin': import ctypes.wintypes @@ -186,6 +193,32 @@ def DelayBeforeRemoving(target_dir): print +def DownloadUsingGsutil(filename): + """Downloads the given file from Google Storage chrome-wintoolchain bucket.""" + temp_dir = tempfile.mkdtemp() + assert os.path.basename(filename) == filename + target_path = os.path.join(temp_dir, filename) + gsutil = download_from_google_storage.Gsutil( + download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None) + code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path) + if code != 0: + sys.exit('gsutil failed') + return temp_dir, target_path + + +def DoTreeMirror(target_dir, tree_sha1): + """In order to save temporary space on bots that do not have enough space to + download ISOs, unpack them, and copy to the target location, the whole tree + is uploaded as a zip to internal storage, and then mirrored here.""" + temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip') + sys.stdout.write('Extracting %s...\n' % local_zip) + sys.stdout.flush() + with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf: + zf.extractall(target_dir) + if temp_dir: + subprocess.check_call('rmdir /s/q "%s"' % temp_dir, shell=True) + + def main(): if not sys.platform.startswith(('cygwin', 'win32')): return 0 @@ -215,7 +248,13 @@ def main(): # the downloader script is. os.chdir(os.path.normpath(os.path.join(BASEDIR))) toolchain_dir = '.' - target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files')) + if os.environ.get('GYP_MSVS_VERSION') == '2015': + target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files')) + else: + target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files')) + abs_target_dir = os.path.abspath(target_dir) + + got_new_toolchain = False # If the current hash doesn't match what we want in the file, nuke and pave. # Typically this script is only run when the .sha1 one file is updated, but @@ -224,8 +263,8 @@ def main(): current_hash = CalculateHash(target_dir) if current_hash not in desired_hashes: should_use_gs = False - if (HaveSrcInternalAccess() or - LooksLikeGoogler() or + if (HaveSrcInternalAccess() or + LooksLikeGoogler() or CanAccessToolchainBucket()): should_use_gs = True if not CanAccessToolchainBucket(): @@ -246,12 +285,35 @@ def main(): stdin=nul, stdout=nul, stderr=nul) if os.path.isdir(target_dir): subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True) - args = [sys.executable, - 'toolchain2013.py', - '--targetdir', target_dir, - '--sha1', desired_hashes[0], - '--use-gs'] - subprocess.check_call(args) + + DoTreeMirror(target_dir, desired_hashes[0]) + + got_new_toolchain = True + + win_sdk = os.path.join(abs_target_dir, 'win_sdk') + try: + with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f: + vs_version = f.read().strip() + except IOError: + # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk' + # instead of just 'win_sdk'. + vs_version = '2013' + win_sdk = os.path.join(abs_target_dir, 'win8sdk') + + data = { + 'path': abs_target_dir, + 'version': vs_version, + 'win_sdk': win_sdk, + 'wdk': os.path.join(abs_target_dir, 'wdk'), + 'runtime_dirs': [ + os.path.join(abs_target_dir, 'sys64'), + os.path.join(abs_target_dir, 'sys32'), + ], + } + with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f: + json.dump(data, f) + + if got_new_toolchain: current_hash = CalculateHash(target_dir) if current_hash not in desired_hashes: print >> sys.stderr, ( diff --git a/win_toolchain/package_from_installed.py b/win_toolchain/package_from_installed.py index 22f52acbd..6fe606f18 100644 --- a/win_toolchain/package_from_installed.py +++ b/win_toolchain/package_from_installed.py @@ -28,15 +28,16 @@ import tempfile import zipfile import get_toolchain_if_necessary -import toolchain2013 # pylint: disable=F0401 + + +VS_VERSION = None def BuildFileList(): result = [] # Subset of VS corresponding roughly to VC. - vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0' - for path in [ + paths = [ 'DIA SDK/bin', 'DIA SDK/idl', 'DIA SDK/include', @@ -47,15 +48,39 @@ def BuildFileList(): 'VC/include', 'VC/lib', 'VC/redist', - ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'), - ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'), - ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'), - ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'), - ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'), - ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'), - ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'), - ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'), - ]: + ] + + if VS_VERSION == '2013': + paths += [ + ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'), + ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'), + ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'), + ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'), + ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'), + ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'), + ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'), + ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'), + ] + elif VS_VERSION == '2015': + paths += [ + ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'), + ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'), + ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'), + ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'), + ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'), + ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'), + ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'), + ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'), + ] + else: + raise ValueError('VS_VERSION %s' % VS_VERSION) + + if VS_VERSION == '2013': + vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0' + else: + vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 14.0' + + for path in paths: src = path[0] if isinstance(path, tuple) else path combined = os.path.join(vs_path, src) assert os.path.exists(combined) and os.path.isdir(combined) @@ -68,8 +93,8 @@ def BuildFileList(): else: assert final_from.startswith(vs_path) dest = final_from[len(vs_path) + 1:] - if dest.lower().endswith('\\xtree'): - # Patch for C4702 in xtree. http://crbug.com/346399. + if VS_VERSION == '2013' and dest.lower().endswith('\\xtree'): + # Patch for C4702 in xtree on VS2013. http://crbug.com/346399. (handle, patched) = tempfile.mkstemp() with open(final_from, 'rb') as unpatched_f: unpatched_contents = unpatched_f.read() @@ -85,21 +110,80 @@ def BuildFileList(): for root, _, files in os.walk(sdk_path): for f in files: combined = os.path.normpath(os.path.join(root, f)) - to = os.path.join('win8sdk', combined[len(sdk_path) + 1:]) + to = os.path.join('win_sdk', combined[len(sdk_path) + 1:]) result.append((combined, to)) + if VS_VERSION == '2015': + for ucrt_path in ( + (r'C:\Program Files (x86)\Windows Kits\10\Include', 'Include'), + (r'C:\Program Files (x86)\Windows Kits\10\Lib', 'Lib'), + (r'C:\Program Files (x86)\Windows Kits\10\Source', 'Source')): + src, target = ucrt_path + for root, _, files in os.walk(src): + for f in files: + combined = os.path.normpath(os.path.join(root, f)) + to = os.path.join('ucrt', target, combined[len(src) + 1:]) + result.append((combined, to)) + # Generically drop all arm stuff that we don't need. - return [(f, t) for f, t in result if 'arm\\' not in f.lower()] + return [(f, t) for f, t in result if 'arm\\' not in f.lower() and + 'arm64\\' not in f.lower()] + + +def GenerateSetEnvCmd(target_dir): + """Generate a batch file that gyp expects to exist to set up the compiler + environment. + + This is normally generated by a full install of the SDK, but we + do it here manually since we do not do a full install.""" + with open(os.path.join( + target_dir, r'win_sdk\bin\SetEnv.cmd'), 'w') as f: + f.write('@echo off\n' + ':: Generated by win_toolchain\\package_from_installed.py.\n' + # Common to x86 and x64 + 'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n' + 'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\um;' + '%~dp0..\\..\\win_sdk\\Include\\shared;' + '%~dp0..\\..\\win_sdk\\Include\\winrt;' + '%~dp0..\\..\\ucrt\\Include\\10.0.10056.0\\ucrt;' + '%~dp0..\\..\\VC\\include;' + '%~dp0..\\..\\VC\\atlmfc\\include\n' + 'if "%1"=="/x64" goto x64\n') + + # x86. Always use amd64_x86 cross, not x86 on x86. + f.write('set PATH=%~dp0..\\..\\win_sdk\\bin\\x86;' + '%~dp0..\\..\\VC\\bin\\amd64_x86;' + '%~dp0..\\..\\VC\\bin\\amd64;' # Needed for mspdb1x0.dll. + '%PATH%\n') + f.write('set LIB=%~dp0..\\..\\VC\\lib;' + '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x86;' + '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x86;' + '%~dp0..\\..\\VC\\atlmfc\\lib\n' + 'goto :EOF\n') + + # x64. + f.write(':x64\n' + 'set PATH=%~dp0..\\..\\win_sdk\\bin\\x64;' + '%~dp0..\\..\\VC\\bin\\amd64;' + '%PATH%\n') + f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;' + '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x64;' + '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x64;' + '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n') def AddEnvSetup(files): """We need to generate this file in the same way that the "from pieces" script does, so pull that in here.""" tempdir = tempfile.mkdtemp() - os.makedirs(os.path.join(tempdir, 'win8sdk', 'bin')) - toolchain2013.GenerateSetEnvCmd(tempdir, True) - files.append((os.path.join(tempdir, 'win8sdk', 'bin', 'SetEnv.cmd'), - 'win8sdk\\bin\\SetEnv.cmd')) + os.makedirs(os.path.join(tempdir, 'win_sdk', 'bin')) + GenerateSetEnvCmd(tempdir) + files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.cmd'), + 'win_sdk\\bin\\SetEnv.cmd')) + vs_version_file = os.path.join(tempdir, 'VS_VERSION') + with open(vs_version_file, 'wb') as version: + print >>version, VS_VERSION + files.append((vs_version_file, 'VS_VERSION')) def RenameToSha1(output): @@ -109,7 +193,7 @@ def RenameToSha1(output): tempdir = tempfile.mkdtemp() old_dir = os.getcwd() os.chdir(tempdir) - rel_dir = 'vs2013_files' + rel_dir = 'vs_files' with zipfile.ZipFile( os.path.join(old_dir, output), 'r', zipfile.ZIP_DEFLATED, True) as zf: zf.extractall(rel_dir) @@ -123,6 +207,13 @@ def RenameToSha1(output): def main(): + if len(sys.argv) != 2 or sys.argv[1] not in ('2013', '2015'): + print 'Usage: package_from_installed.py 2013|2015' + return 1 + + global VS_VERSION + VS_VERSION = sys.argv[1] + print 'Building file list...' files = BuildFileList()