Revert "[win_tools] Use bundled Python CIPD packages."
This reverts commit bf1446791e.
Reason for revert: broke win bots http://o/410053
Original change's description:
> [win_tools] Use bundled Python CIPD packages.
>
> Enable bundled Python CIPD packages in bleeding-edge mode. This
> replaces the ZIP unpacking approach used before, and introduces
> validation and management through the CIPD tool. The bleeding edge
> version will only install if a sentinel file is present in the
> "depot_tools" root; otherwise, default behavior will continue.
>
> This method adds a upgrade and downgrade path to/from ZIP and
> CIPD installations. This is done by rewriting the "win_tools.bat"
> process:
>
> 1) Ensure that a bootstrap Python is present.
> 2) Use it to run "win_tools.py", which has the functionality of
> "git_bootstrap.py" plus Python installation.
> 3) Run "win_tools.py" with appropriate flags.
>
> Some tricks were employed to handle cases where there is an
> already-running Python instance that uses the current Python
> installation and executable. This happens on bots because the
> system uses the same "depot_tools" checkout at multiple launch
> layers. To this end, we use the "python.bat" as the "current Python"
> authority and refrain from cleaning up old Python directories if their
> "python.exe" binaries are currently in use.
>
> We change the Git bleeding edge file to share the same
> sentinel file as Python, ".bleeding_edge".
>
> The new Python should have the same facilities as the original Python
> bundle.
>
> BUG=chromium:740171
> TEST=local
>
> Change-Id: I1b3b7d31d47d1a37a9dba9114d31681bec558736
> Reviewed-on: https://chromium-review.googlesource.com/563036
> Commit-Queue: Daniel Jacques <dnj@chromium.org>
> Reviewed-by: Robbie Iannucci <iannucci@chromium.org>
TBR=iannucci@chromium.org,dnj@chromium.org
Change-Id: I84574a01bbad6596912e4aaa34f019d24720b638
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: chromium:740171
Reviewed-on: https://chromium-review.googlesource.com/567009
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Commit-Queue: Andrii Shyshkalov <tandrii@chromium.org>
changes/09/567009/2
Andrii Shyshkalov
8 years ago
committed by
Commit Bot
parent
7dadf05dad
commit
af5c20f5a4
@ -1,5 +1,5 @@
|
|||||||
@echo off
|
@echo off
|
||||||
setlocal
|
setlocal
|
||||||
if not defined EDITOR set EDITOR=notepad
|
if not defined EDITOR set EDITOR=notepad
|
||||||
set PATH=%~dp0${GIT_BIN_RELDIR}\cmd;%~dp0;%PATH%
|
set PATH=%~dp0GIT_BIN_DIR\cmd;%~dp0;%PATH%
|
||||||
"%~dp0${GIT_BIN_RELDIR}\${GIT_PROGRAM}" %*
|
"%~dp0GIT_BIN_DIR\GIT_PROGRAM" %*
|
||||||
|
|||||||
@ -0,0 +1,274 @@
|
|||||||
|
# Copyright 2016 The Chromium Authors. All rights reserved.
|
||||||
|
# Use of this source code is governed by a BSD-style license that can be
|
||||||
|
# found in the LICENSE file.
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import contextlib
|
||||||
|
import fnmatch
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
|
||||||
|
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
|
||||||
|
|
||||||
|
DEVNULL = open(os.devnull, 'w')
|
||||||
|
|
||||||
|
BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
|
||||||
|
|
||||||
|
|
||||||
|
# Top-level stubs to generate that fall through to executables within the Git
|
||||||
|
# directory.
|
||||||
|
STUBS = {
|
||||||
|
'git.bat': 'cmd\\git.exe',
|
||||||
|
'gitk.bat': 'cmd\\gitk.exe',
|
||||||
|
'ssh.bat': 'usr\\bin\\ssh.exe',
|
||||||
|
'ssh-keygen.bat': 'usr\\bin\\ssh-keygen.exe',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _check_call(argv, input=None, **kwargs):
|
||||||
|
"""Wrapper for subprocess.check_call that adds logging."""
|
||||||
|
logging.info('running %r', argv)
|
||||||
|
if input is not None:
|
||||||
|
kwargs['stdin'] = subprocess.PIPE
|
||||||
|
proc = subprocess.Popen(argv, **kwargs)
|
||||||
|
proc.communicate(input=input)
|
||||||
|
if proc.returncode:
|
||||||
|
raise subprocess.CalledProcessError(proc.returncode, argv, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_rmtree(path):
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return
|
||||||
|
|
||||||
|
def _make_writable_and_remove(path):
|
||||||
|
st = os.stat(path)
|
||||||
|
new_mode = st.st_mode | 0200
|
||||||
|
if st.st_mode == new_mode:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
os.chmod(path, new_mode)
|
||||||
|
os.remove(path)
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _on_error(function, path, excinfo):
|
||||||
|
if not _make_writable_and_remove(path):
|
||||||
|
logging.warning('Failed to %s: %s (%s)', function, path, excinfo)
|
||||||
|
|
||||||
|
shutil.rmtree(path, onerror=_on_error)
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def _tempdir():
|
||||||
|
tdir = None
|
||||||
|
try:
|
||||||
|
tdir = tempfile.mkdtemp()
|
||||||
|
yield tdir
|
||||||
|
finally:
|
||||||
|
_safe_rmtree(tdir)
|
||||||
|
|
||||||
|
|
||||||
|
def get_os_bitness():
|
||||||
|
"""Returns bitness of operating system as int."""
|
||||||
|
return 64 if platform.machine().endswith('64') else 32
|
||||||
|
|
||||||
|
|
||||||
|
def get_target_git_version(args):
|
||||||
|
"""Returns git version that should be installed."""
|
||||||
|
if (args.bleeding_edge or
|
||||||
|
os.path.exists(os.path.join(ROOT_DIR, '.git_bleeding_edge'))):
|
||||||
|
git_version_file = 'git_version_bleeding_edge.txt'
|
||||||
|
else:
|
||||||
|
git_version_file = 'git_version.txt'
|
||||||
|
with open(os.path.join(THIS_DIR, git_version_file)) as f:
|
||||||
|
return f.read().strip()
|
||||||
|
|
||||||
|
|
||||||
|
def clean_up_old_git_installations(git_directory, force):
|
||||||
|
"""Removes git installations other than |git_directory|."""
|
||||||
|
for entry in fnmatch.filter(os.listdir(ROOT_DIR), 'git-*_bin'):
|
||||||
|
full_entry = os.path.join(ROOT_DIR, entry)
|
||||||
|
if force or full_entry != git_directory:
|
||||||
|
logging.info('Cleaning up old git installation %r', entry)
|
||||||
|
_safe_rmtree(full_entry)
|
||||||
|
|
||||||
|
|
||||||
|
def cipd_install(args, dest_directory, package, version):
|
||||||
|
"""Installs CIPD |package| at |version| into |dest_directory|."""
|
||||||
|
logging.info('Installing CIPD package %r @ %r', package, version)
|
||||||
|
manifest = '%s %s\n' % (package, version)
|
||||||
|
cipd_args = [
|
||||||
|
args.cipd_client,
|
||||||
|
'ensure',
|
||||||
|
'-ensure-file', '-',
|
||||||
|
'-root', dest_directory,
|
||||||
|
]
|
||||||
|
if args.cipd_cache_directory:
|
||||||
|
cipd_args.extend(['-cache-dir', args.cipd_cache_directory])
|
||||||
|
if args.verbose:
|
||||||
|
cipd_args.append('-verbose')
|
||||||
|
_check_call(cipd_args, input=manifest)
|
||||||
|
|
||||||
|
|
||||||
|
def need_to_install_git(args, git_directory, legacy):
|
||||||
|
"""Returns True if git needs to be installed."""
|
||||||
|
if args.force:
|
||||||
|
return True
|
||||||
|
|
||||||
|
is_cipd_managed = os.path.exists(os.path.join(git_directory, '.cipd'))
|
||||||
|
if legacy:
|
||||||
|
if is_cipd_managed:
|
||||||
|
# Converting from non-legacy to legacy, need reinstall.
|
||||||
|
return True
|
||||||
|
if not os.path.exists(os.path.join(
|
||||||
|
git_directory, 'etc', 'profile.d', 'python.sh')):
|
||||||
|
return True
|
||||||
|
elif not is_cipd_managed:
|
||||||
|
# Converting from legacy to CIPD, need reinstall.
|
||||||
|
return True
|
||||||
|
|
||||||
|
git_exe_path = os.path.join(git_directory, 'bin', 'git.exe')
|
||||||
|
if not os.path.exists(git_exe_path):
|
||||||
|
return True
|
||||||
|
if subprocess.call(
|
||||||
|
[git_exe_path, '--version'],
|
||||||
|
stdout=DEVNULL, stderr=DEVNULL) != 0:
|
||||||
|
return True
|
||||||
|
|
||||||
|
gen_stubs = STUBS.keys()
|
||||||
|
gen_stubs.append('git-bash')
|
||||||
|
for stub in gen_stubs:
|
||||||
|
full_path = os.path.join(ROOT_DIR, stub)
|
||||||
|
if not os.path.exists(full_path):
|
||||||
|
return True
|
||||||
|
with open(full_path) as f:
|
||||||
|
if os.path.relpath(git_directory, ROOT_DIR) not in f.read():
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def install_git_legacy(args, git_version, git_directory, cipd_platform):
|
||||||
|
_safe_rmtree(git_directory)
|
||||||
|
with _tempdir() as temp_dir:
|
||||||
|
cipd_install(args,
|
||||||
|
temp_dir,
|
||||||
|
'infra/depot_tools/git_installer/%s' % cipd_platform,
|
||||||
|
'v' + git_version.replace('.', '_'))
|
||||||
|
|
||||||
|
# 7-zip has weird expectations for command-line syntax. Pass it as a string
|
||||||
|
# to avoid subprocess module quoting breaking it. Also double-escape
|
||||||
|
# backslashes in paths.
|
||||||
|
_check_call(' '.join([
|
||||||
|
os.path.join(temp_dir, 'git-installer.exe'),
|
||||||
|
'-y',
|
||||||
|
'-InstallPath="%s"' % git_directory.replace('\\', '\\\\'),
|
||||||
|
'-Directory="%s"' % git_directory.replace('\\', '\\\\'),
|
||||||
|
]))
|
||||||
|
|
||||||
|
|
||||||
|
def install_git(args, git_version, git_directory, legacy):
|
||||||
|
"""Installs |git_version| into |git_directory|."""
|
||||||
|
# TODO: Remove legacy version once everyone is on bundled Git.
|
||||||
|
cipd_platform = 'windows-%s' % ('amd64' if args.bits == 64 else '386')
|
||||||
|
if legacy:
|
||||||
|
install_git_legacy(args, git_version, git_directory, cipd_platform)
|
||||||
|
else:
|
||||||
|
# When migrating from legacy, we want to nuke this directory. In other
|
||||||
|
# cases, CIPD will handle the cleanup.
|
||||||
|
if not os.path.isdir(os.path.join(git_directory, '.cipd')):
|
||||||
|
logging.info('Deleting legacy Git directory: %s', git_directory)
|
||||||
|
_safe_rmtree(git_directory)
|
||||||
|
|
||||||
|
cipd_install(
|
||||||
|
args,
|
||||||
|
git_directory,
|
||||||
|
'infra/git/%s' % (cipd_platform,),
|
||||||
|
git_version)
|
||||||
|
|
||||||
|
# Create Git templates and configure its base layout.
|
||||||
|
with open(os.path.join(THIS_DIR, 'git.template.bat')) as f:
|
||||||
|
git_template = f.read()
|
||||||
|
git_template = git_template.replace(
|
||||||
|
'GIT_BIN_DIR', os.path.relpath(git_directory, ROOT_DIR))
|
||||||
|
|
||||||
|
for stub_name, relpath in STUBS.iteritems():
|
||||||
|
with open(os.path.join(ROOT_DIR, stub_name), 'w') as f:
|
||||||
|
f.write(git_template.replace('GIT_PROGRAM', relpath))
|
||||||
|
with open(os.path.join(THIS_DIR, 'git-bash.template.sh')) as f:
|
||||||
|
git_bash_template = f.read()
|
||||||
|
with open(os.path.join(ROOT_DIR, 'git-bash'), 'w') as f:
|
||||||
|
f.write(git_bash_template.replace(
|
||||||
|
'GIT_BIN_DIR', os.path.relpath(git_directory, ROOT_DIR)))
|
||||||
|
|
||||||
|
if legacy:
|
||||||
|
# The non-legacy Git bundle includes "python.sh".
|
||||||
|
#
|
||||||
|
# TODO: Delete "profile.d.python.sh" after legacy mode is removed.
|
||||||
|
shutil.copyfile(
|
||||||
|
os.path.join(THIS_DIR, 'profile.d.python.sh'),
|
||||||
|
os.path.join(git_directory, 'etc', 'profile.d', 'python.sh'))
|
||||||
|
|
||||||
|
git_bat_path = os.path.join(ROOT_DIR, 'git.bat')
|
||||||
|
_check_call([git_bat_path, 'config', '--system', 'core.autocrlf', 'false'])
|
||||||
|
_check_call([git_bat_path, 'config', '--system', 'core.filemode', 'false'])
|
||||||
|
_check_call([git_bat_path, 'config', '--system', 'core.preloadindex', 'true'])
|
||||||
|
_check_call([git_bat_path, 'config', '--system', 'core.fscache', 'true'])
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv):
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--bits', type=int, choices=(32,64),
|
||||||
|
help='Bitness of the client to install. Default on this'
|
||||||
|
' system: %(default)s', default=get_os_bitness())
|
||||||
|
parser.add_argument('--cipd-client',
|
||||||
|
help='Path to CIPD client binary. default: %(default)s',
|
||||||
|
default=os.path.join(ROOT_DIR, 'cipd'+BAT_EXT))
|
||||||
|
parser.add_argument('--cipd-cache-directory',
|
||||||
|
help='Path to CIPD cache directory.')
|
||||||
|
parser.add_argument('--bleeding-edge', action='store_true',
|
||||||
|
help='Force bleeding edge Git.')
|
||||||
|
parser.add_argument('--force', action='store_true',
|
||||||
|
help='Always re-install git.')
|
||||||
|
parser.add_argument('--verbose', action='store_true')
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
if os.environ.get('WIN_TOOLS_FORCE') == '1':
|
||||||
|
args.force = True
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO if args.verbose else logging.WARN)
|
||||||
|
|
||||||
|
git_version = get_target_git_version(args)
|
||||||
|
|
||||||
|
git_directory_tag = git_version.split(':')
|
||||||
|
git_directory = os.path.join(
|
||||||
|
ROOT_DIR, 'git-%s-%s_bin' % (git_directory_tag[-1], args.bits))
|
||||||
|
git_docs_dir = os.path.join(
|
||||||
|
git_directory, 'mingw%s' % args.bits, 'share', 'doc', 'git-doc')
|
||||||
|
|
||||||
|
clean_up_old_git_installations(git_directory, args.force)
|
||||||
|
|
||||||
|
# Modern Git versions use CIPD tags beginning with "version:". If the tag
|
||||||
|
# does not begin with that, use the legacy installer.
|
||||||
|
legacy = not git_version.startswith('version:')
|
||||||
|
if need_to_install_git(args, git_directory, legacy):
|
||||||
|
install_git(args, git_version, git_directory, legacy)
|
||||||
|
|
||||||
|
# Update depot_tools files for "git help <command>"
|
||||||
|
docsrc = os.path.join(ROOT_DIR, 'man', 'html')
|
||||||
|
for name in os.listdir(docsrc):
|
||||||
|
shutil.copy2(os.path.join(docsrc, name), os.path.join(git_docs_dir, name))
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main(sys.argv[1:]))
|
||||||
@ -1,18 +0,0 @@
|
|||||||
# CIPD manifest for Windows tools.
|
|
||||||
#
|
|
||||||
# We must install anything that we want included on PATH to a different
|
|
||||||
# subdirectory than Git, as Git's msys bash strips its root directory
|
|
||||||
# from PATH, hence the subdirs.
|
|
||||||
#
|
|
||||||
# If any paths or package layouts change, updates will be required in
|
|
||||||
# "win_tools.bat" and "win_tools.py" templates.
|
|
||||||
#
|
|
||||||
# "win_tools.bat" has a hard requirement that the Python package contains the
|
|
||||||
# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this
|
|
||||||
# to extract VERSION.
|
|
||||||
|
|
||||||
@Subdir python
|
|
||||||
infra/python/cpython/windows-386 version:2.7.6
|
|
||||||
|
|
||||||
@Subdir git
|
|
||||||
infra/git/${platform} version:2.10.0
|
|
||||||
@ -1,49 +0,0 @@
|
|||||||
@echo off
|
|
||||||
:: Copyright 2017 The Chromium Authors. All rights reserved.
|
|
||||||
:: Use of this source code is governed by a BSD-style license that can be
|
|
||||||
:: found in the LICENSE file.
|
|
||||||
|
|
||||||
setlocal
|
|
||||||
set PYTHON_BAT_RUNNER=1
|
|
||||||
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
||||||
|
|
||||||
:: This file is automatically generated by "bootstrap\win\win_tools.py", and
|
|
||||||
:: should not be modified.
|
|
||||||
::
|
|
||||||
:: The previous "::" block acts as a nop-sled. Each time a batch file executes
|
|
||||||
:: a command, it reloads itself and seeks to its previous execution offset to
|
|
||||||
:: begin execution. Updating this batch file is, therefore, risky, since any
|
|
||||||
:: running Python instance that is using the old batch file will reload the new
|
|
||||||
:: batch file once the Python command terminates and resume at some unknown
|
|
||||||
:: offset.
|
|
||||||
::
|
|
||||||
:: With the sled in place, a previous instance will resume mid-label. We are
|
|
||||||
:: assuming that the offset of the Python invocation is greater than the
|
|
||||||
:: PYTHON_BAT_RUNNER set command, which is the case since the old instance has
|
|
||||||
:: a large PATH set block before the Python execution. Old instances will hit
|
|
||||||
:: the next block of code without PYTHON_BAT_RUNNER set. New instances will have
|
|
||||||
:: it set.
|
|
||||||
::
|
|
||||||
:: We remedy this in the future by having the batch file load its core paths
|
|
||||||
:: from an external file via "set /p", removing the need to modify "python.bat"
|
|
||||||
:: during upgrade.
|
|
||||||
::
|
|
||||||
:: After all of the old batch files are believed to be replaced, we can remove
|
|
||||||
:: the PYTHON_BAT_RUNNER block and the sled. For this update, old instances
|
|
||||||
:: will resume past the end of the file and terminate.
|
|
||||||
|
|
||||||
if not "%PYTHON_BAT_RUNNER%" == "1" goto :END
|
|
||||||
|
|
||||||
set /p PYTHON_BIN_RELDIR=<%~dp0python_bin_reldir.txt
|
|
||||||
set PATH=%~dp0%PYTHON_BIN_RELDIR%;%~dp0%PYTHON_BIN_RELDIR%\Scripts;%PATH%
|
|
||||||
"%~dp0%PYTHON_BIN_RELDIR%\python.exe" %*
|
|
||||||
|
|
||||||
:END
|
|
||||||
@ -1,510 +0,0 @@
|
|||||||
# Copyright 2016 The Chromium Authors. All rights reserved.
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import collections
|
|
||||||
import contextlib
|
|
||||||
import fnmatch
|
|
||||||
import hashlib
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import platform
|
|
||||||
import posixpath
|
|
||||||
import shutil
|
|
||||||
import string
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
|
|
||||||
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
|
|
||||||
ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
|
|
||||||
|
|
||||||
DEVNULL = open(os.devnull, 'w')
|
|
||||||
|
|
||||||
BAT_EXT = '.bat' if sys.platform.startswith('win') else ''
|
|
||||||
|
|
||||||
# Top-level stubs to generate that fall through to executables within the Git
|
|
||||||
# directory.
|
|
||||||
STUBS = {
|
|
||||||
'git.bat': 'cmd\\git.exe',
|
|
||||||
'gitk.bat': 'cmd\\gitk.exe',
|
|
||||||
'ssh.bat': 'usr\\bin\\ssh.exe',
|
|
||||||
'ssh-keygen.bat': 'usr\\bin\\ssh-keygen.exe',
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Accumulated template parameters for generated stubs.
|
|
||||||
class Template(collections.namedtuple('Template', (
|
|
||||||
'PYTHON_RELDIR', 'PYTHON_BIN_RELDIR', 'PYTHON_BIN_RELDIR_UNIX',
|
|
||||||
'GIT_BIN_RELDIR', 'GIT_BIN_RELDIR_UNIX', 'GIT_PROGRAM',
|
|
||||||
))):
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def empty(cls):
|
|
||||||
return cls(**{k: None for k in cls._fields})
|
|
||||||
|
|
||||||
def maybe_install(self, name, dst_path):
|
|
||||||
"""Installs template |name| to |dst_path| if it has changed.
|
|
||||||
|
|
||||||
This loads the template |name| from THIS_DIR, resolves template parameters,
|
|
||||||
and installs it to |dst_path|. See `maybe_update` for more information.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
name (str): The name of the template to install.
|
|
||||||
dst_path (str): The destination filesystem path.
|
|
||||||
|
|
||||||
Returns (bool): True if |dst_path| was updated, False otherwise.
|
|
||||||
"""
|
|
||||||
template_path = os.path.join(THIS_DIR, name)
|
|
||||||
with open(template_path, 'r') as fd:
|
|
||||||
t = string.Template(fd.read())
|
|
||||||
return maybe_update(t.safe_substitute(self._asdict()), dst_path)
|
|
||||||
|
|
||||||
|
|
||||||
def maybe_update(content, dst_path):
|
|
||||||
"""Writes |content| to |dst_path| if |dst_path| does not already match.
|
|
||||||
|
|
||||||
This function will ensure that there is a file at |dst_path| containing
|
|
||||||
|content|. If |dst_path| already exists and contains |content|, no operation
|
|
||||||
will be performed, preserving filesystem modification times and avoiding
|
|
||||||
potential write contention.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
content (str): The file content.
|
|
||||||
dst_path (str): The destination filesystem path.
|
|
||||||
|
|
||||||
Returns (bool): True if |dst_path| was updated, False otherwise.
|
|
||||||
"""
|
|
||||||
# If the path already exists and matches the new content, refrain from writing
|
|
||||||
# a new one.
|
|
||||||
if os.path.exists(dst_path):
|
|
||||||
with open(dst_path, 'r') as fd:
|
|
||||||
if fd.read() == content:
|
|
||||||
return False
|
|
||||||
|
|
||||||
logging.debug('Updating %r', dst_path)
|
|
||||||
with open(dst_path, 'w') as fd:
|
|
||||||
fd.write(content)
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def maybe_copy(src_path, dst_path):
|
|
||||||
"""Writes the content of |src_path| to |dst_path| if needed.
|
|
||||||
|
|
||||||
See `maybe_update` for more information.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
src_path (str): The content source filesystem path.
|
|
||||||
dst_path (str): The destination filesystem path.
|
|
||||||
|
|
||||||
Returns (bool): True if |dst_path| was updated, False otherwise.
|
|
||||||
"""
|
|
||||||
with open(src_path, 'r') as fd:
|
|
||||||
content = fd.read()
|
|
||||||
return maybe_update(content, dst_path)
|
|
||||||
|
|
||||||
|
|
||||||
def call_if_outdated(stamp_path, stamp_version, fn):
|
|
||||||
"""Invokes |fn| if the stamp at |stamp_path| doesn't match |stamp_version|.
|
|
||||||
|
|
||||||
This can be used to keep a filesystem record of whether an operation has been
|
|
||||||
performed. The record is stored at |stamp_path|. To invalidate a record,
|
|
||||||
change the value of |stamp_version|.
|
|
||||||
|
|
||||||
After |fn| completes successfully, |stamp_path| will be updated to match
|
|
||||||
|stamp_version|, preventing the same update from happening in the future.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
stamp_path (str): The filesystem path of the stamp file.
|
|
||||||
stamp_version (str): The desired stamp version.
|
|
||||||
fn (callable): A callable to invoke if the current stamp version doesn't
|
|
||||||
match |stamp_version|.
|
|
||||||
|
|
||||||
Returns (bool): True if an update occurred.
|
|
||||||
"""
|
|
||||||
|
|
||||||
stamp_version = stamp_version.strip()
|
|
||||||
if os.path.isfile(stamp_path):
|
|
||||||
with open(stamp_path, 'r') as fd:
|
|
||||||
current_version = fd.read().strip()
|
|
||||||
if current_version == stamp_version:
|
|
||||||
return False
|
|
||||||
|
|
||||||
fn()
|
|
||||||
|
|
||||||
with open(stamp_path, 'w') as fd:
|
|
||||||
fd.write(stamp_version)
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _in_use(path):
|
|
||||||
"""Checks if a Windows file is in use.
|
|
||||||
|
|
||||||
When Windows is using an executable, it prevents other writers from
|
|
||||||
modifying or deleting that executable. We can safely test for an in-use
|
|
||||||
file by opening it in write mode and checking whether or not there was
|
|
||||||
an error.
|
|
||||||
|
|
||||||
Returns (bool): True if the file was in use, False if not.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
with open(path, 'r+'):
|
|
||||||
return False
|
|
||||||
except IOError:
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _check_call(argv, stdin_input=None, **kwargs):
|
|
||||||
"""Wrapper for subprocess.check_call that adds logging."""
|
|
||||||
logging.info('running %r', argv)
|
|
||||||
if stdin_input is not None:
|
|
||||||
kwargs['stdin'] = subprocess.PIPE
|
|
||||||
proc = subprocess.Popen(argv, **kwargs)
|
|
||||||
proc.communicate(input=stdin_input)
|
|
||||||
if proc.returncode:
|
|
||||||
raise subprocess.CalledProcessError(proc.returncode, argv, None)
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_rmtree(path):
|
|
||||||
if not os.path.exists(path):
|
|
||||||
return
|
|
||||||
|
|
||||||
def _make_writable_and_remove(path):
|
|
||||||
st = os.stat(path)
|
|
||||||
new_mode = st.st_mode | 0200
|
|
||||||
if st.st_mode == new_mode:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
os.chmod(path, new_mode)
|
|
||||||
os.remove(path)
|
|
||||||
return True
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _on_error(function, path, excinfo):
|
|
||||||
if not _make_writable_and_remove(path):
|
|
||||||
logging.warning('Failed to %s: %s (%s)', function, path, excinfo)
|
|
||||||
|
|
||||||
shutil.rmtree(path, onerror=_on_error)
|
|
||||||
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
|
||||||
def _tempdir():
|
|
||||||
tdir = None
|
|
||||||
try:
|
|
||||||
tdir = tempfile.mkdtemp()
|
|
||||||
yield tdir
|
|
||||||
finally:
|
|
||||||
_safe_rmtree(tdir)
|
|
||||||
|
|
||||||
|
|
||||||
def get_os_bitness():
|
|
||||||
"""Returns bitness of operating system as int."""
|
|
||||||
return 64 if platform.machine().endswith('64') else 32
|
|
||||||
|
|
||||||
|
|
||||||
def get_target_git_version(args):
|
|
||||||
"""Returns git version that should be installed."""
|
|
||||||
if args.bleeding_edge:
|
|
||||||
git_version_file = 'git_version_bleeding_edge.txt'
|
|
||||||
else:
|
|
||||||
git_version_file = 'git_version.txt'
|
|
||||||
with open(os.path.join(THIS_DIR, git_version_file)) as f:
|
|
||||||
return f.read().strip()
|
|
||||||
|
|
||||||
|
|
||||||
def clean_up_old_git_installations(git_directory, force):
|
|
||||||
"""Removes git installations other than |git_directory|."""
|
|
||||||
for entry in fnmatch.filter(os.listdir(ROOT_DIR), 'git-*_bin'):
|
|
||||||
full_entry = os.path.join(ROOT_DIR, entry)
|
|
||||||
if force or full_entry != git_directory:
|
|
||||||
logging.info('Cleaning up old git installation %r', entry)
|
|
||||||
_safe_rmtree(full_entry)
|
|
||||||
|
|
||||||
|
|
||||||
def clean_up_old_installations(skip_dir):
|
|
||||||
"""Removes Python installations other than |skip_dir|.
|
|
||||||
|
|
||||||
This includes an "in-use" check against the "python.exe" in a given directory
|
|
||||||
to avoid removing Python executables that are currently ruinning. We need
|
|
||||||
this because our Python bootstrap may be run after (and by) other software
|
|
||||||
that is using the bootstrapped Python!
|
|
||||||
"""
|
|
||||||
root_contents = os.listdir(ROOT_DIR)
|
|
||||||
for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin'):
|
|
||||||
for entry in fnmatch.filter(root_contents, f):
|
|
||||||
full_entry = os.path.join(ROOT_DIR, entry)
|
|
||||||
if full_entry == skip_dir or not os.path.isdir(full_entry):
|
|
||||||
continue
|
|
||||||
|
|
||||||
logging.info('Cleaning up old installation %r', entry)
|
|
||||||
for python_exe in (
|
|
||||||
os.path.join(full_entry, 'python', 'bin', 'python.exe'), # CIPD
|
|
||||||
os.path.join(full_entry, 'python.exe'), # Legacy ZIP distributions.
|
|
||||||
):
|
|
||||||
if os.path.isfile(python_exe) and _in_use(python_exe):
|
|
||||||
logging.info('Python executable %r is in-use; skipping', python_exe)
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
_safe_rmtree(full_entry)
|
|
||||||
|
|
||||||
|
|
||||||
def cipd_ensure(args, dest_directory, package, version):
|
|
||||||
"""Installs a CIPD package using "ensure"."""
|
|
||||||
logging.info('Installing CIPD package %r @ %r', package, version)
|
|
||||||
manifest_text = '%s %s\n' % (package, version)
|
|
||||||
|
|
||||||
cipd_args = [
|
|
||||||
args.cipd_client,
|
|
||||||
'ensure',
|
|
||||||
'-ensure-file', '-',
|
|
||||||
'-root', dest_directory,
|
|
||||||
]
|
|
||||||
if args.cipd_cache_directory:
|
|
||||||
cipd_args.extend(['-cache-dir', args.cipd_cache_directory])
|
|
||||||
if args.verbose:
|
|
||||||
cipd_args.append('-verbose')
|
|
||||||
_check_call(cipd_args, stdin_input=manifest_text)
|
|
||||||
|
|
||||||
|
|
||||||
def need_to_install_git(args, git_directory, legacy):
|
|
||||||
"""Returns True if git needs to be installed."""
|
|
||||||
if args.force:
|
|
||||||
return True
|
|
||||||
|
|
||||||
is_cipd_managed = os.path.exists(os.path.join(git_directory, '.cipd'))
|
|
||||||
if legacy:
|
|
||||||
if is_cipd_managed:
|
|
||||||
# Converting from non-legacy to legacy, need reinstall.
|
|
||||||
return True
|
|
||||||
if not os.path.exists(os.path.join(
|
|
||||||
git_directory, 'etc', 'profile.d', 'python.sh')):
|
|
||||||
return True
|
|
||||||
elif not is_cipd_managed:
|
|
||||||
# Converting from legacy to CIPD, need reinstall.
|
|
||||||
return True
|
|
||||||
|
|
||||||
git_exe_path = os.path.join(git_directory, 'bin', 'git.exe')
|
|
||||||
if not os.path.exists(git_exe_path):
|
|
||||||
return True
|
|
||||||
if subprocess.call(
|
|
||||||
[git_exe_path, '--version'],
|
|
||||||
stdout=DEVNULL, stderr=DEVNULL) != 0:
|
|
||||||
return True
|
|
||||||
|
|
||||||
gen_stubs = STUBS.keys()
|
|
||||||
gen_stubs.append('git-bash')
|
|
||||||
for stub in gen_stubs:
|
|
||||||
full_path = os.path.join(ROOT_DIR, stub)
|
|
||||||
if not os.path.exists(full_path):
|
|
||||||
return True
|
|
||||||
with open(full_path) as f:
|
|
||||||
if os.path.relpath(git_directory, ROOT_DIR) not in f.read():
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def install_git_legacy(args, git_version, git_directory, cipd_platform):
|
|
||||||
_safe_rmtree(git_directory)
|
|
||||||
with _tempdir() as temp_dir:
|
|
||||||
cipd_ensure(args, temp_dir,
|
|
||||||
package='infra/depot_tools/git_installer/%s' % cipd_platform,
|
|
||||||
version='v' + git_version.replace('.', '_'))
|
|
||||||
|
|
||||||
# 7-zip has weird expectations for command-line syntax. Pass it as a string
|
|
||||||
# to avoid subprocess module quoting breaking it. Also double-escape
|
|
||||||
# backslashes in paths.
|
|
||||||
_check_call(' '.join([
|
|
||||||
os.path.join(temp_dir, 'git-installer.exe'),
|
|
||||||
'-y',
|
|
||||||
'-InstallPath="%s"' % git_directory.replace('\\', '\\\\'),
|
|
||||||
'-Directory="%s"' % git_directory.replace('\\', '\\\\'),
|
|
||||||
]))
|
|
||||||
|
|
||||||
|
|
||||||
def install_git(args, git_version, git_directory, legacy):
|
|
||||||
"""Installs |git_version| into |git_directory|."""
|
|
||||||
# TODO: Remove legacy version once everyone is on bundled Git.
|
|
||||||
cipd_platform = 'windows-%s' % ('amd64' if args.bits == 64 else '386')
|
|
||||||
if legacy:
|
|
||||||
install_git_legacy(args, git_version, git_directory, cipd_platform)
|
|
||||||
else:
|
|
||||||
# When migrating from legacy, we want to nuke this directory. In other
|
|
||||||
# cases, CIPD will handle the cleanup.
|
|
||||||
if not os.path.isdir(os.path.join(git_directory, '.cipd')):
|
|
||||||
logging.info('Deleting legacy Git directory: %s', git_directory)
|
|
||||||
_safe_rmtree(git_directory)
|
|
||||||
|
|
||||||
cipd_ensure(args, git_directory,
|
|
||||||
package='infra/git/%s' % (cipd_platform,),
|
|
||||||
version=git_version)
|
|
||||||
|
|
||||||
if legacy:
|
|
||||||
# The non-legacy Git bundle includes "python.sh".
|
|
||||||
#
|
|
||||||
# TODO: Delete "profile.d.python.sh" after legacy mode is removed.
|
|
||||||
shutil.copyfile(
|
|
||||||
os.path.join(THIS_DIR, 'profile.d.python.sh'),
|
|
||||||
os.path.join(git_directory, 'etc', 'profile.d', 'python.sh'))
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_git(args, template):
|
|
||||||
git_version = get_target_git_version(args)
|
|
||||||
|
|
||||||
git_directory_tag = git_version.split(':')
|
|
||||||
git_directory = os.path.join(
|
|
||||||
ROOT_DIR, 'git-%s-%s_bin' % (git_directory_tag[-1], args.bits))
|
|
||||||
|
|
||||||
clean_up_old_git_installations(git_directory, args.force)
|
|
||||||
|
|
||||||
git_bin_dir = os.path.relpath(git_directory, ROOT_DIR)
|
|
||||||
template = template._replace(
|
|
||||||
GIT_BIN_RELDIR=git_bin_dir,
|
|
||||||
GIT_BIN_RELDIR_UNIX=git_bin_dir)
|
|
||||||
|
|
||||||
# Modern Git versions use CIPD tags beginning with "version:". If the tag
|
|
||||||
# does not begin with that, use the legacy installer.
|
|
||||||
legacy = not git_version.startswith('version:')
|
|
||||||
if need_to_install_git(args, git_directory, legacy):
|
|
||||||
install_git(args, git_version, git_directory, legacy)
|
|
||||||
|
|
||||||
git_postprocess(template, git_directory)
|
|
||||||
|
|
||||||
return template
|
|
||||||
|
|
||||||
|
|
||||||
# Version of "git_postprocess" system configuration (see |git_postprocess|).
|
|
||||||
GIT_POSTPROCESS_VERSION = '1'
|
|
||||||
|
|
||||||
|
|
||||||
def git_get_mingw_dir(git_directory):
|
|
||||||
"""Returns (str) The "mingw" directory in a Git installation, or None."""
|
|
||||||
for candidate in ('mingw64', 'mingw32'):
|
|
||||||
mingw_dir = os.path.join(git_directory, candidate)
|
|
||||||
if os.path.isdir(mingw_dir):
|
|
||||||
return mingw_dir
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def git_postprocess(template, git_directory):
|
|
||||||
# Update depot_tools files for "git help <command>"
|
|
||||||
mingw_dir = git_get_mingw_dir(git_directory)
|
|
||||||
if mingw_dir:
|
|
||||||
docsrc = os.path.join(ROOT_DIR, 'man', 'html')
|
|
||||||
git_docs_dir = os.path.join(mingw_dir, 'share', 'doc', 'git-doc')
|
|
||||||
for name in os.listdir(docsrc):
|
|
||||||
maybe_copy(
|
|
||||||
os.path.join(docsrc, name),
|
|
||||||
os.path.join(git_docs_dir, name))
|
|
||||||
else:
|
|
||||||
logging.info('Could not find mingw directory for %r.', git_directory)
|
|
||||||
|
|
||||||
# Create Git templates and configure its base layout.
|
|
||||||
for stub_name, relpath in STUBS.iteritems():
|
|
||||||
stub_template = template._replace(GIT_PROGRAM=relpath)
|
|
||||||
stub_template.maybe_install(
|
|
||||||
'git.template.bat',
|
|
||||||
os.path.join(ROOT_DIR, stub_name))
|
|
||||||
|
|
||||||
# Set-up our system configuration environment. The following set of
|
|
||||||
# parameters is versioned by "GIT_POSTPROCESS_VERSION". If they change,
|
|
||||||
# update "GIT_POSTPROCESS_VERSION" accordingly.
|
|
||||||
def configure_git_system():
|
|
||||||
git_bat_path = os.path.join(ROOT_DIR, 'git.bat')
|
|
||||||
_check_call([git_bat_path, 'config', '--system', 'core.autocrlf', 'false'])
|
|
||||||
_check_call([git_bat_path, 'config', '--system', 'core.filemode', 'false'])
|
|
||||||
_check_call([git_bat_path, 'config', '--system', 'core.preloadindex',
|
|
||||||
'true'])
|
|
||||||
_check_call([git_bat_path, 'config', '--system', 'core.fscache', 'true'])
|
|
||||||
|
|
||||||
call_if_outdated(
|
|
||||||
os.path.join(git_directory, '.git_postprocess'),
|
|
||||||
GIT_POSTPROCESS_VERSION,
|
|
||||||
configure_git_system)
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--verbose', action='store_true')
|
|
||||||
parser.add_argument('--win-tools-name',
|
|
||||||
help='The directory of the Python installation. '
|
|
||||||
'(legacy) If missing, use legacy Windows tools '
|
|
||||||
'processing')
|
|
||||||
parser.add_argument('--bleeding-edge', action='store_true',
|
|
||||||
help='Force bleeding edge Git.')
|
|
||||||
|
|
||||||
group = parser.add_argument_group('legacy flags')
|
|
||||||
group.add_argument('--force', action='store_true',
|
|
||||||
help='Always re-install everything.')
|
|
||||||
group.add_argument('--bits', type=int, choices=(32,64),
|
|
||||||
help='Bitness of the client to install. Default on this'
|
|
||||||
' system: %(default)s', default=get_os_bitness())
|
|
||||||
group.add_argument('--cipd-client',
|
|
||||||
help='Path to CIPD client binary. default: %(default)s',
|
|
||||||
default=os.path.join(ROOT_DIR, 'cipd'+BAT_EXT))
|
|
||||||
group.add_argument('--cipd-cache-directory',
|
|
||||||
help='Path to CIPD cache directory.')
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
|
|
||||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
|
|
||||||
|
|
||||||
template = Template.empty()
|
|
||||||
if not args.win_tools_name:
|
|
||||||
# Legacy support.
|
|
||||||
template = template._replace(
|
|
||||||
PYTHON_RELDIR='python276_bin',
|
|
||||||
PYTHON_BIN_RELDIR='python276_bin',
|
|
||||||
PYTHON_BIN_RELDIR_UNIX='python276_bin')
|
|
||||||
template = ensure_git(args, template)
|
|
||||||
else:
|
|
||||||
template = template._replace(
|
|
||||||
PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'),
|
|
||||||
PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'),
|
|
||||||
PYTHON_BIN_RELDIR_UNIX=posixpath.join(
|
|
||||||
args.win_tools_name, 'python', 'bin'),
|
|
||||||
GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'),
|
|
||||||
GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git'))
|
|
||||||
|
|
||||||
win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name)
|
|
||||||
git_postprocess(template, os.path.join(win_tools_dir, 'git'))
|
|
||||||
|
|
||||||
# Clean up any old Python installations.
|
|
||||||
clean_up_old_installations(win_tools_dir)
|
|
||||||
|
|
||||||
# Emit our Python bin depot-tools-relative directory. This is ready by
|
|
||||||
# "python.bat" to identify the path of the current Python installation.
|
|
||||||
#
|
|
||||||
# We use this indirection so that upgrades can change this pointer to
|
|
||||||
# redirect "python.bat" to a new Python installation. We can't just update
|
|
||||||
# "python.bat" because batch file executions reload the batch file and seek
|
|
||||||
# to the previous cursor in between every command, so changing the batch
|
|
||||||
# file contents could invalidate any existing executions.
|
|
||||||
#
|
|
||||||
# The intention is that the batch file itself never needs to change when
|
|
||||||
# switching Python versions.
|
|
||||||
maybe_update(
|
|
||||||
template.PYTHON_BIN_RELDIR,
|
|
||||||
os.path.join(ROOT_DIR, 'python_bin_reldir.txt'))
|
|
||||||
|
|
||||||
# Install our "python.bat" shim.
|
|
||||||
# TODO: Move this to generic shim installation once legacy support is
|
|
||||||
# removed and this code path is the only one.
|
|
||||||
template.maybe_install(
|
|
||||||
'python27.new.bat',
|
|
||||||
os.path.join(ROOT_DIR, 'python.bat'))
|
|
||||||
|
|
||||||
# Re-evaluate and regenerate our root templated files.
|
|
||||||
for src_name, dst_name in (
|
|
||||||
('git-bash.template.sh', 'git-bash'),
|
|
||||||
('pylint.new.bat', 'pylint.bat'),
|
|
||||||
):
|
|
||||||
template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main(sys.argv[1:]))
|
|
||||||
Loading…
Reference in New Issue