Rework/update toolchain script for vs2015

This is the other side of https://codereview.chromium.org/1163723003/

The changes here are to remove the use of 'vs2013_files' and 'win8sdk'
(as those will be different numbers soon enough) but still maintain
behaviour for the old "style" while in transition.

Secondarily, to remove the dependence of these two scripts on
'toolchain2013.py' as most of the script is now unused.

R=dpranke@chromium.org
BUG=440500, 492774

Review URL: https://codereview.chromium.org/1165563003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295485 0039d316-1c4b-4281-b951-d872f2087c98

win_toolchain/get_toolchain_if_necessary.py

@@ -33,13 +33,20 @@ import os
 import shutil
 import subprocess
 import sys
+import tempfile
 import time
+import zipfile
 
 
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
 DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..')
 sys.path.append(DEPOT_TOOLS_PATH)
-import download_from_google_storage
+try:
+  import download_from_google_storage
+except ImportError:
+  # Allow use of utility functions in this script from package_from_installed
+  # on bare VM that doesn't have a full depot_tools.
+  pass
 
 if sys.platform != 'cygwin':
   import ctypes.wintypes
@@ -186,6 +193,32 @@ def DelayBeforeRemoving(target_dir):
     print
 
 
+def DownloadUsingGsutil(filename):
+  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+  temp_dir = tempfile.mkdtemp()
+  assert os.path.basename(filename) == filename
+  target_path = os.path.join(temp_dir, filename)
+  gsutil = download_from_google_storage.Gsutil(
+      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
+  code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
+  if code != 0:
+    sys.exit('gsutil failed')
+  return temp_dir, target_path
+
+
+def DoTreeMirror(target_dir, tree_sha1):
+  """In order to save temporary space on bots that do not have enough space to
+  download ISOs, unpack them, and copy to the target location, the whole tree
+  is uploaded as a zip to internal storage, and then mirrored here."""
+  temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
+  sys.stdout.write('Extracting %s...\n' % local_zip)
+  sys.stdout.flush()
+  with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf:
+    zf.extractall(target_dir)
+  if temp_dir:
+    subprocess.check_call('rmdir /s/q "%s"' % temp_dir, shell=True)
+
+
 def main():
   if not sys.platform.startswith(('cygwin', 'win32')):
     return 0
@@ -215,7 +248,13 @@ def main():
   # the downloader script is.
   os.chdir(os.path.normpath(os.path.join(BASEDIR)))
   toolchain_dir = '.'
-  target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
+  if os.environ.get('GYP_MSVS_VERSION') == '2015':
+    target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files'))
+  else:
+    target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
+  abs_target_dir = os.path.abspath(target_dir)
+
+  got_new_toolchain = False
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
@@ -224,8 +263,8 @@ def main():
   current_hash = CalculateHash(target_dir)
   if current_hash not in desired_hashes:
     should_use_gs = False
-    if (HaveSrcInternalAccess() or 
-        LooksLikeGoogler() or 
+    if (HaveSrcInternalAccess() or
+        LooksLikeGoogler() or
         CanAccessToolchainBucket()):
       should_use_gs = True
       if not CanAccessToolchainBucket():
@@ -246,12 +285,35 @@ def main():
                       stdin=nul, stdout=nul, stderr=nul)
     if os.path.isdir(target_dir):
       subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True)
-    args = [sys.executable,
-            'toolchain2013.py',
-            '--targetdir', target_dir,
-            '--sha1', desired_hashes[0],
-            '--use-gs']
-    subprocess.check_call(args)
+
+    DoTreeMirror(target_dir, desired_hashes[0])
+
+    got_new_toolchain = True
+
+  win_sdk = os.path.join(abs_target_dir, 'win_sdk')
+  try:
+    with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f:
+      vs_version = f.read().strip()
+  except IOError:
+    # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk'
+    # instead of just 'win_sdk'.
+    vs_version = '2013'
+    win_sdk = os.path.join(abs_target_dir, 'win8sdk')
+
+  data = {
+      'path': abs_target_dir,
+      'version': vs_version,
+      'win_sdk': win_sdk,
+      'wdk': os.path.join(abs_target_dir, 'wdk'),
+      'runtime_dirs': [
+        os.path.join(abs_target_dir, 'sys64'),
+        os.path.join(abs_target_dir, 'sys32'),
+      ],
+  }
+  with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
+    json.dump(data, f)
+
+  if got_new_toolchain:
     current_hash = CalculateHash(target_dir)
     if current_hash not in desired_hashes:
       print >> sys.stderr, (

win_toolchain/package_from_installed.py

@@ -28,15 +28,16 @@ import tempfile
 import zipfile
 
 import get_toolchain_if_necessary
-import toolchain2013  # pylint: disable=F0401
+
+
+VS_VERSION = None
 
 
 def BuildFileList():
   result = []
 
   # Subset of VS corresponding roughly to VC.
-  vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0'
-  for path in [
+  paths = [
       'DIA SDK/bin',
       'DIA SDK/idl',
       'DIA SDK/include',
@@ -47,15 +48,39 @@ def BuildFileList():
       'VC/include',
       'VC/lib',
       'VC/redist',
-      ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'),
-      ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'),
-      ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'),
-      ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'),
-      ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'),
-      ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'),
-      ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'),
-      ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'),
-      ]:
+  ]
+
+  if VS_VERSION == '2013':
+    paths += [
+        ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'),
+        ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'),
+        ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'),
+        ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'),
+        ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'),
+        ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'),
+        ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'),
+        ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'),
+    ]
+  elif VS_VERSION == '2015':
+    paths += [
+        ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'),
+        ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'),
+        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'),
+        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'),
+        ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'),
+        ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'),
+        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'),
+        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'),
+    ]
+  else:
+    raise ValueError('VS_VERSION %s' % VS_VERSION)
+
+  if VS_VERSION == '2013':
+    vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0'
+  else:
+    vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 14.0'
+
+  for path in paths:
     src = path[0] if isinstance(path, tuple) else path
     combined = os.path.join(vs_path, src)
     assert os.path.exists(combined) and os.path.isdir(combined)
@@ -68,8 +93,8 @@ def BuildFileList():
         else:
           assert final_from.startswith(vs_path)
           dest = final_from[len(vs_path) + 1:]
-          if dest.lower().endswith('\\xtree'):
-            # Patch for C4702 in xtree. http://crbug.com/346399.
+          if VS_VERSION == '2013' and dest.lower().endswith('\\xtree'):
+            # Patch for C4702 in xtree on VS2013. http://crbug.com/346399.
             (handle, patched) = tempfile.mkstemp()
             with open(final_from, 'rb') as unpatched_f:
               unpatched_contents = unpatched_f.read()
@@ -85,21 +110,80 @@ def BuildFileList():
   for root, _, files in os.walk(sdk_path):
     for f in files:
       combined = os.path.normpath(os.path.join(root, f))
-      to = os.path.join('win8sdk', combined[len(sdk_path) + 1:])
+      to = os.path.join('win_sdk', combined[len(sdk_path) + 1:])
       result.append((combined, to))
 
+  if VS_VERSION == '2015':
+    for ucrt_path in (
+        (r'C:\Program Files (x86)\Windows Kits\10\Include', 'Include'),
+        (r'C:\Program Files (x86)\Windows Kits\10\Lib', 'Lib'),
+        (r'C:\Program Files (x86)\Windows Kits\10\Source', 'Source')):
+      src, target = ucrt_path
+      for root, _, files in os.walk(src):
+        for f in files:
+          combined = os.path.normpath(os.path.join(root, f))
+          to = os.path.join('ucrt', target, combined[len(src) + 1:])
+          result.append((combined, to))
+
   # Generically drop all arm stuff that we don't need.
-  return [(f, t) for f, t in result if 'arm\\' not in f.lower()]
+  return [(f, t) for f, t in result if 'arm\\' not in f.lower() and
+                                       'arm64\\' not in f.lower()]
+
+
+def GenerateSetEnvCmd(target_dir):
+  """Generate a batch file that gyp expects to exist to set up the compiler
+  environment.
+
+  This is normally generated by a full install of the SDK, but we
+  do it here manually since we do not do a full install."""
+  with open(os.path.join(
+        target_dir, r'win_sdk\bin\SetEnv.cmd'), 'w') as f:
+    f.write('@echo off\n'
+            ':: Generated by win_toolchain\\package_from_installed.py.\n'
+            # Common to x86 and x64
+            'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n'
+            'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\um;'
+               '%~dp0..\\..\\win_sdk\\Include\\shared;'
+               '%~dp0..\\..\\win_sdk\\Include\\winrt;'
+               '%~dp0..\\..\\ucrt\\Include\\10.0.10056.0\\ucrt;'
+               '%~dp0..\\..\\VC\\include;'
+               '%~dp0..\\..\\VC\\atlmfc\\include\n'
+            'if "%1"=="/x64" goto x64\n')
+
+    # x86. Always use amd64_x86 cross, not x86 on x86.
+    f.write('set PATH=%~dp0..\\..\\win_sdk\\bin\\x86;'
+              '%~dp0..\\..\\VC\\bin\\amd64_x86;'
+              '%~dp0..\\..\\VC\\bin\\amd64;'  # Needed for mspdb1x0.dll.
+              '%PATH%\n')
+    f.write('set LIB=%~dp0..\\..\\VC\\lib;'
+               '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x86;'
+               '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x86;'
+               '%~dp0..\\..\\VC\\atlmfc\\lib\n'
+            'goto :EOF\n')
+
+    # x64.
+    f.write(':x64\n'
+            'set PATH=%~dp0..\\..\\win_sdk\\bin\\x64;'
+                '%~dp0..\\..\\VC\\bin\\amd64;'
+                '%PATH%\n')
+    f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
+               '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x64;'
+               '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x64;'
+               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
 
 
 def AddEnvSetup(files):
   """We need to generate this file in the same way that the "from pieces"
   script does, so pull that in here."""
   tempdir = tempfile.mkdtemp()
-  os.makedirs(os.path.join(tempdir, 'win8sdk', 'bin'))
-  toolchain2013.GenerateSetEnvCmd(tempdir, True)
-  files.append((os.path.join(tempdir, 'win8sdk', 'bin', 'SetEnv.cmd'),
-                'win8sdk\\bin\\SetEnv.cmd'))
+  os.makedirs(os.path.join(tempdir, 'win_sdk', 'bin'))
+  GenerateSetEnvCmd(tempdir)
+  files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.cmd'),
+                'win_sdk\\bin\\SetEnv.cmd'))
+  vs_version_file = os.path.join(tempdir, 'VS_VERSION')
+  with open(vs_version_file, 'wb') as version:
+    print >>version, VS_VERSION
+  files.append((vs_version_file, 'VS_VERSION'))
 
 
 def RenameToSha1(output):
@@ -109,7 +193,7 @@ def RenameToSha1(output):
   tempdir = tempfile.mkdtemp()
   old_dir = os.getcwd()
   os.chdir(tempdir)
-  rel_dir = 'vs2013_files'
+  rel_dir = 'vs_files'
   with zipfile.ZipFile(
       os.path.join(old_dir, output), 'r', zipfile.ZIP_DEFLATED, True) as zf:
     zf.extractall(rel_dir)
@@ -123,6 +207,13 @@ def RenameToSha1(output):
 
 
 def main():
+  if len(sys.argv) != 2 or sys.argv[1] not in ('2013', '2015'):
+    print 'Usage: package_from_installed.py 2013|2015'
+    return 1
+
+  global VS_VERSION
+  VS_VERSION = sys.argv[1]
+
   print 'Building file list...'
   files = BuildFileList()