diff --git a/.gitignore b/.gitignore index 4144e3695..4f6bf5e84 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ /git_bin /git-1.8.0_bin /svn_bin +/win_toolchain/vs2013_files +/win_toolchain/.timestamps # Ignore ctags/cscope index files /TAGS diff --git a/update_depot_tools.bat b/update_depot_tools.bat index 58d3d8497..d461aab64 100644 --- a/update_depot_tools.bat +++ b/update_depot_tools.bat @@ -30,6 +30,13 @@ if errorlevel 1 goto :EOF :: Now clear errorlevel so it can be set by other programs later. set errorlevel= +:: Opt-in for now. +IF "%DEPOT_TOOLS_WIN_TOOLCHAIN%" == "" GOTO :NOTOOLCHAIN +call python %~dp0win_toolchain\get_toolchain_if_necessary.py +if errorlevel 1 goto :EOF +set errorlevel= +:NOTOOLCHAIN + :: Shall skip automatic update? IF "%DEPOT_TOOLS_UPDATE%" == "0" GOTO :EOF diff --git a/win_toolchain/7z/7z.dll b/win_toolchain/7z/7z.dll new file mode 100644 index 000000000..cea996e4f Binary files /dev/null and b/win_toolchain/7z/7z.dll differ diff --git a/win_toolchain/7z/7z.exe b/win_toolchain/7z/7z.exe new file mode 100644 index 000000000..b55fefe6d Binary files /dev/null and b/win_toolchain/7z/7z.exe differ diff --git a/win_toolchain/7z/LICENSE b/win_toolchain/7z/LICENSE new file mode 100644 index 000000000..21bb84e78 --- /dev/null +++ b/win_toolchain/7z/LICENSE @@ -0,0 +1,57 @@ + 7-Zip + ~~~~~ + License for use and distribution + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + 7-Zip Copyright (C) 1999-2012 Igor Pavlov. + + Licenses for files are: + + 1) 7z.dll: GNU LGPL + unRAR restriction + 2) All other files: GNU LGPL + + The GNU LGPL + unRAR restriction means that you must follow both + GNU LGPL rules and unRAR restriction rules. + + + Note: + You can use 7-Zip on any computer, including a computer in a commercial + organization. You don't need to register or pay for 7-Zip. + + + GNU LGPL information + -------------------- + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You can receive a copy of the GNU Lesser General Public License from + http://www.gnu.org/ + + + unRAR restriction + ----------------- + + The decompression engine for RAR archives was developed using source + code of unRAR program. + All copyrights to original unRAR code are owned by Alexander Roshal. + + The license for original unRAR code has the following restriction: + + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form + or as a part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + + -- + Igor Pavlov + diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py new file mode 100755 index 000000000..ecced5fe1 --- /dev/null +++ b/win_toolchain/get_toolchain_if_necessary.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Downloads and unpacks a toolchain for building on Windows. The contents are +matched by sha1 which will be updated when the toolchain is updated. + +Having a toolchain script in depot_tools means that it's not versioned +directly with the source code. That is, if the toolchain is upgraded, but +you're trying to build an historical version of Chromium from before the +toolchain upgrade, this will cause you to build with a newer toolchain than +was available when that code was committed. This is done for a two main +reasons: 1) it would likely be annoying to have the up-to-date toolchain +removed and replaced by one without a service pack applied); 2) it would +require maintaining scripts that can build older not-up-to-date revisions of +the toolchain. This is likely to be a poorly tested code path that probably +won't be properly maintained. See http://crbug.com/323300. + +This does not extend to major versions of the toolchain however, on the +assumption that there are more likely to be source incompatibilities between +major revisions. This script calls a subscript (currently, toolchain2013.py) +to do the main work. It is expected that toolchain2013.py will always be able +to acquire/build the most current revision of a VS2013-based toolchain. In the +future when a hypothetical VS2015 is released, the 2013 script will be +maintained, and a new 2015 script would be added. +""" + +import ctypes.wintypes +import hashlib +import json +import os +import subprocess +import sys + + +BASEDIR = os.path.dirname(os.path.abspath(__file__)) + + +GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW +GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,) +GetFileAttributes.restype = ctypes.wintypes.DWORD +FILE_ATTRIBUTE_HIDDEN = 0x2 +FILE_ATTRIBUTE_SYSTEM = 0x4 + + +def IsHidden(file_path): + """Returns whether the given |file_path| has the 'system' or 'hidden' + attribute set.""" + p = GetFileAttributes(file_path) + assert p != 0xffffffff + return bool(p & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) + + +def GetFileList(root): + """Gets a normalized list of files under |root|.""" + assert not os.path.isabs(root) + assert os.path.normpath(root) == root + file_list = [] + for base, _, files in os.walk(root): + paths = [os.path.join(base, f) for f in files] + file_list.extend(x.lower() for x in paths if not IsHidden(x)) + return sorted(file_list) + + +def MakeTimestampsFileName(root): + return os.path.join(root, '..', '.timestamps') + + +def CalculateHash(root): + """Calculates the sha1 of the paths to all files in the given |root| and the + contents of those files, and returns as a hex string.""" + file_list = GetFileList(root) + + # Check whether we previously saved timestamps in $root/../.timestamps. If + # we didn't, or they don't match, then do the full calculation, otherwise + # return the saved value. + timestamps_file = MakeTimestampsFileName(root) + timestamps_data = {'files': [], 'sha1': ''} + if os.path.exists(timestamps_file): + with open(timestamps_file, 'rb') as f: + try: + timestamps_data = json.load(f) + except ValueError: + # json couldn't be loaded, empty data will force a re-hash. + pass + + matches = len(file_list) == len(timestamps_data['files']) + if matches: + for disk, cached in zip(file_list, timestamps_data['files']): + if disk != cached[0] or os.stat(disk).st_mtime != cached[1]: + matches = False + break + if matches: + return timestamps_data['sha1'] + + digest = hashlib.sha1() + for path in file_list: + digest.update(path) + with open(path, 'rb') as f: + digest.update(f.read()) + return digest.hexdigest() + + +def SaveTimestampsAndHash(root, sha1): + """Save timestamps and the final hash to be able to early-out more quickly + next time.""" + file_list = GetFileList(root) + timestamps_data = { + 'files': [[f, os.stat(f).st_mtime] for f in file_list], + 'sha1': sha1, + } + with open(MakeTimestampsFileName(root), 'wb') as f: + json.dump(timestamps_data, f) + + +def main(): + if not sys.platform.startswith(('cygwin', 'win32')): + return 0 + + if len(sys.argv) != 1: + print >> sys.stderr, 'Unexpected arguments.' + return 1 + + # Move to depot_tools\win_toolchain where we'll store our files, and where + # the downloader script is. + os.chdir(os.path.normpath(os.path.join(BASEDIR))) + toolchain_dir = '.' + target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files')) + + sha1path = os.path.join(toolchain_dir, 'toolchain_vs2013.hash') + desired_hash = '' + if os.path.isfile(sha1path): + with open(sha1path, 'rb') as f: + desired_hash = f.read().strip() + + # If the current hash doesn't match what we want in the file, nuke and pave. + # Typically this script is only run when the .sha1 one file is updated, but + # directly calling "gclient runhooks" will also run it, so we cache + # based on timestamps to make that case fast. + current_hash = CalculateHash(target_dir) + if current_hash != desired_hash: + print 'Windows toolchain out of date or doesn\'t exist, updating...' + if os.path.isdir(target_dir): + subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True) + subprocess.check_call([ + sys.executable, + 'toolchain2013.py', + '--targetdir', target_dir]) + current_hash = CalculateHash(target_dir) + if current_hash != desired_hash: + print >> sys.stderr, ( + 'Got wrong hash after pulling a new toolchain. ' + 'Wanted \'%s\', got \'%s\'.' % ( + desired_hash, current_hash)) + return 1 + SaveTimestampsAndHash(target_dir, current_hash) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/win_toolchain/toolchain2013.py b/win_toolchain/toolchain2013.py new file mode 100755 index 000000000..2bce633a2 --- /dev/null +++ b/win_toolchain/toolchain2013.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Extracts a Windows VS2013 toolchain from various downloadable pieces.""" + + +import ctypes +import optparse +import os +import shutil +import subprocess +import sys +import tempfile +import urllib2 + + +BASEDIR = os.path.dirname(os.path.abspath(__file__)) +g_temp_dirs = [] + + +def GetLongPathName(path): + """Converts any 8dot3 names in the path to the full name.""" + buf = ctypes.create_unicode_buffer(260) + size = ctypes.windll.kernel32.GetLongPathNameW(unicode(path), buf, 260) + if (size > 260): + sys.exit('Long form of path longer than 260 chars: %s' % path) + return buf.value + + +def RunOrDie(command): + subprocess.check_call(command, shell=True) + + +def TempDir(): + """Generates a temporary directory (for downloading or extracting to) and keep + track of the directory that's created for cleaning up later. + """ + global g_temp_dirs + temp = tempfile.mkdtemp() + g_temp_dirs.append(temp) + return temp + + +def DeleteAllTempDirs(): + """Removes all temporary directories created by |TempDir()|.""" + global g_temp_dirs + if g_temp_dirs: + sys.stdout.write('Cleaning up temporaries...\n') + for temp in g_temp_dirs: + # shutil.rmtree errors out on read only attributes. + RunOrDie('rmdir /s/q "%s"' % temp) + g_temp_dirs = [] + + +def GetIsoUrl(pro): + """Gets the .iso URL. + + If |pro| is False, downloads the Express edition. + """ + prefix = 'http://download.microsoft.com/download/' + if pro: + return (prefix + + 'A/F/1/AF128362-A6A8-4DB3-A39A-C348086472CC/VS2013_RTM_PRO_ENU.iso') + else: + return (prefix + + '7/2/E/72E0F986-D247-4289-B9DC-C4FB07374894/VS2013_RTM_DskExp_ENU.iso') + + +def Download(url, local_path): + """Downloads a large-ish binary file and print some status information while + doing so. + """ + sys.stdout.write('Downloading %s...\n' % url) + req = urllib2.urlopen(url) + content_length = int(req.headers.get('Content-Length', 0)) + bytes_read = 0L + terminator = '\r' if sys.stdout.isatty() else '\n' + with open(local_path, 'wb') as file: + while True: + chunk = req.read(1024 * 1024) + if not chunk: + break + bytes_read += len(chunk) + file.write(chunk) + sys.stdout.write('... %d/%d%s' % (bytes_read, content_length, terminator)) + sys.stdout.flush() + sys.stdout.write('\n') + if content_length and content_length != bytes_read: + raise SystemExit('Got incorrect number of bytes downloading %s' % url) + + +def ExtractIso(iso_path): + """Uses 7zip to extract the contents of the given .iso (or self-extracting + .exe). + """ + target_path = TempDir() + sys.stdout.write('Extracting %s...\n' % iso_path) + sys.stdout.flush() + # TODO(scottmg): Do this (and exe) manually with python code. + # Note that at the beginning of main() we set the working directory to 7z's + # location so that 7z can find its codec dll. + RunOrDie('7z x "%s" -y "-o%s" >nul' % (iso_path, target_path)) + return target_path + + +def ExtractMsi(msi_path): + """Uses msiexec to extract the contents of the given .msi file.""" + sys.stdout.write('Extracting %s...\n' % msi_path) + target_path = TempDir() + RunOrDie('msiexec /a "%s" /qn TARGETDIR="%s"' % (msi_path, target_path)) + return target_path + + +def DownloadMainIso(url): + temp_dir = TempDir() + target_path = os.path.join(temp_dir, os.path.basename(url)) + Download(url, target_path) + return target_path + + +def GetSourceImage(local_dir, pro): + url = GetIsoUrl(pro) + if local_dir: + return os.path.join(local_dir, os.path.basename(url)) + else: + return DownloadMainIso(url) + + +def ExtractMsiList(iso_dir, packages): + """Extracts the contents of a list of .msi files from an already extracted + .iso file. + + |packages| is a list of pairs (msi, required). If required is not True, the + msi is optional (this is set for packages that are in Pro but not Express). + """ + results = [] + for (package, required) in packages: + path_to_package = os.path.join(iso_dir, 'packages', package) + if not os.path.exists(path_to_package) and not required: + continue + results.append(ExtractMsi(path_to_package)) + return results + + +def ExtractComponents(image): + packages = [ + (r'vcRuntimeAdditional_amd64\vc_runtimeAdditional_x64.msi', True), + (r'vcRuntimeAdditional_x86\vc_runtimeAdditional_x86.msi', True), + (r'vcRuntimeDebug_amd64\vc_runtimeDebug_x64.msi', True), + (r'vcRuntimeDebug_x86\vc_runtimeDebug_x86.msi', True), + (r'vcRuntimeMinimum_amd64\vc_runtimeMinimum_x64.msi', True), + (r'vcRuntimeMinimum_x86\vc_runtimeMinimum_x86.msi', True), + (r'vc_compilerCore86\vc_compilerCore86.msi', True), + (r'vc_compilerCore86res\vc_compilerCore86res.msi', True), + (r'vc_compilerx64nat\vc_compilerx64nat.msi', False), + (r'vc_compilerx64natres\vc_compilerx64natres.msi', False), + (r'vc_compilerx64x86\vc_compilerx64x86.msi', False), + (r'vc_compilerx64x86res\vc_compilerx64x86res.msi', False), + (r'vc_librarycore86\vc_librarycore86.msi', True), + (r'vc_libraryDesktop\x64\vc_LibraryDesktopX64.msi', True), + (r'vc_libraryDesktop\x86\vc_LibraryDesktopX86.msi', True), + (r'vc_libraryextended\vc_libraryextended.msi', False), + (r'Windows_SDK\Windows Software Development Kit-x86_en-us.msi', True), + ('Windows_SDK\\' + r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi', + True), + ] + extracted_iso = ExtractIso(image) + return ExtractMsiList(extracted_iso, packages) + + +def CopyToFinalLocation(extracted_dirs, target_dir): + sys.stdout.write('Copying to final location...\n') + mappings = { + 'Program Files\\Microsoft Visual Studio 12.0\\': '.\\', + 'System64\\': 'sys64\\', + 'System\\': 'sys32\\', + 'Windows Kits\\8.0\\': 'win8sdk\\', + } + matches = [] + for extracted_dir in extracted_dirs: + for root, dirnames, filenames in os.walk(extracted_dir): + for filename in filenames: + matches.append((extracted_dir, os.path.join(root, filename))) + + copies = [] + for prefix, full_path in matches: + # +1 for trailing \. + partial_path = full_path[len(prefix) + 1:] + for map_from, map_to in mappings.iteritems(): + if partial_path.startswith(map_from): + target_path = os.path.join(map_to, partial_path[len(map_from):]) + copies.append((full_path, os.path.join(target_dir, target_path))) + + for full_source, full_target in copies: + target_dir = os.path.dirname(full_target) + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + shutil.copy2(full_source, full_target) + + +def GenerateSetEnvCmd(target_dir, pro): + """Generate a batch file that gyp expects to exist to set up the compiler + environment. + + This is normally generated by a full install of the SDK, but we + do it here manually since we do not do a full install.""" + with open(os.path.join( + target_dir, r'win8sdk\bin\SetEnv.cmd'), 'w') as f: + f.write('@echo off\n' + ':: Generated by win_toolchain\\toolchain2013.py.\n' + # Common to x86 and x64 + 'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n' + 'set INCLUDE=%~dp0..\\..\\win8sdk\\Include\\um;' + '%~dp0..\\..\\win8sdk\\Include\\shared;' + '%~dp0..\\..\\VC\\include;' + '%~dp0..\\..\\VC\\atlmfc\\include\n' + 'if "%1"=="/x64" goto x64\n') + + # x86. If we're Pro, then use the amd64_x86 cross (we don't support x86 + # host at all). + if pro: + f.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x86;' + '%~dp0..\\..\\VC\\bin\\amd64_x86;' + '%~dp0..\\..\\VC\\bin\\amd64;' # Needed for mspdb120.dll. + '%PATH%\n') + else: + f.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x86;' + '%~dp0..\\..\\VC\\bin;%PATH%\n') + f.write('set LIB=%~dp0..\\..\\VC\\lib;' + '%~dp0..\\..\\win8sdk\\Lib\\win8\\um\\x86;' + '%~dp0..\\..\\VC\\atlmfc\\lib\n' + 'goto :EOF\n') + + # Express does not include a native 64 bit compiler, so we have to use + # the x86->x64 cross. + if not pro: + # x86->x64 cross. + f.write(':x64\n' + 'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;' + '%~dp0..\\..\\VC\\bin\\x86_amd64;' + '%PATH%\n') + else: + # x64 native. + f.write(':x64\n' + 'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;' + '%~dp0..\\..\\VC\\bin\\amd64;' + '%PATH%\n') + f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;' + '%~dp0..\\..\\win8sdk\\Lib\\win8\\um\\x64;' + '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n') + + +def main(): + parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) + parser.add_option('--targetdir', metavar='DIR', + help='put toolchain into DIR', + default=os.path.join(BASEDIR, 'win_toolchain_2013')) + parser.add_option('--noclean', action='store_false', dest='clean', + help='do not remove temp files', + default=True) + parser.add_option('--local', metavar='DIR', + help='use downloaded files from DIR') + parser.add_option('--express', + help='use VS Express instead of Pro', action='store_true') + options, args = parser.parse_args() + try: + target_dir = os.path.abspath(options.targetdir) + if os.path.exists(target_dir): + parser.error('%s already exists. Please [re]move it or use ' + '--targetdir to select a different target.\n' % + target_dir) + # Set the working directory to 7z subdirectory. 7-zip doesn't find its + # codec dll very well, so this is the simplest way to make sure it runs + # correctly, as we don't otherwise care about working directory. + os.chdir(os.path.join(BASEDIR, '7z')) + image = GetSourceImage(options.local, not options.express) + extracted = ExtractComponents(image) + CopyToFinalLocation(extracted, target_dir) + + GenerateSetEnvCmd(target_dir, not options.express) + finally: + if options.clean: + DeleteAllTempDirs() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/win_toolchain/toolchain_vs2013.hash b/win_toolchain/toolchain_vs2013.hash new file mode 100644 index 000000000..98dde202f --- /dev/null +++ b/win_toolchain/toolchain_vs2013.hash @@ -0,0 +1 @@ +f31aca625a2ebb7f05439641e1c2ce5e23ed1bf9