diff --git a/.gitignore b/.gitignore index da461b10f..503104720 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ /.subversion # Ignore locations where third-party tools are placed during bootstrapping. +/bootstrap*_bin /python*_bin /python_bin_reldir.txt /python3_bin_reldir.txt diff --git a/bootstrap/win/README.md b/bootstrap/README.md similarity index 100% rename from bootstrap/win/README.md rename to bootstrap/README.md diff --git a/bootstrap/win/win_tools.py b/bootstrap/bootstrap.py similarity index 80% rename from bootstrap/win/win_tools.py rename to bootstrap/bootstrap.py index 82f1ffc25..533bc8f1b 100644 --- a/bootstrap/win/win_tools.py +++ b/bootstrap/bootstrap.py @@ -19,15 +19,16 @@ import tempfile THIS_DIR = os.path.abspath(os.path.dirname(__file__)) -ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..')) +ROOT_DIR = os.path.abspath(os.path.join(THIS_DIR, '..')) DEVNULL = open(os.devnull, 'w') -BAT_EXT = '.bat' if sys.platform.startswith('win') else '' +IS_WIN = sys.platform.startswith('win') +BAT_EXT = '.bat' if IS_WIN else '' # Top-level stubs to generate that fall through to executables within the Git # directory. -STUBS = { +WIN_GIT_STUBS = { 'git.bat': 'cmd\\git.exe', 'gitk.bat': 'cmd\\gitk.exe', 'ssh.bat': 'usr\\bin\\ssh.exe', @@ -88,6 +89,7 @@ def maybe_update(content, dst_path): logging.debug('Updating %r', dst_path) with open(dst_path, 'w') as fd: fd.write(content) + os.chmod(dst_path, 0o755) return True @@ -200,7 +202,7 @@ def _safe_rmtree(path): def _make_writable_and_remove(path): st = os.stat(path) - new_mode = st.st_mode | 0200 + new_mode = st.st_mode | 0o200 if st.st_mode == new_mode: return False try: @@ -226,7 +228,7 @@ def clean_up_old_installations(skip_dir): that is using the bootstrapped Python! """ root_contents = os.listdir(ROOT_DIR) - for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin'): + for f in ('win_tools-*_bin', 'python27*_bin', 'git-*_bin', 'bootstrap-*_bin'): for entry in fnmatch.filter(root_contents, f): full_entry = os.path.join(ROOT_DIR, entry) if full_entry == skip_dir or not os.path.isdir(full_entry): @@ -266,7 +268,7 @@ def git_postprocess(template, git_directory): logging.info('Could not find mingw directory for %r.', git_directory) # Create Git templates and configure its base layout. - for stub_name, relpath in STUBS.iteritems(): + for stub_name, relpath in WIN_GIT_STUBS.items(): stub_template = template._replace(GIT_PROGRAM=relpath) stub_template.maybe_install( 'git.template.bat', @@ -293,7 +295,7 @@ def git_postprocess(template, git_directory): def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--verbose', action='store_true') - parser.add_argument('--win-tools-name', required=True, + parser.add_argument('--bootstrap-name', required=True, help='The directory of the Python installation.') parser.add_argument('--bleeding-edge', action='store_true', help='Force bleeding edge Git.') @@ -302,52 +304,59 @@ def main(argv): logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) template = Template.empty()._replace( - PYTHON_RELDIR=os.path.join(args.win_tools_name, 'python'), - PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'), + PYTHON_RELDIR=os.path.join(args.bootstrap_name, 'python'), + PYTHON_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python', 'bin'), PYTHON_BIN_RELDIR_UNIX=posixpath.join( - args.win_tools_name, 'python', 'bin'), - PYTHON3_BIN_RELDIR=os.path.join(args.win_tools_name, 'python3', 'bin'), + args.bootstrap_name, 'python', 'bin'), + PYTHON3_BIN_RELDIR=os.path.join(args.bootstrap_name, 'python3', 'bin'), PYTHON3_BIN_RELDIR_UNIX=posixpath.join( - args.win_tools_name, 'python3', 'bin'), - GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'), - GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git')) + args.bootstrap_name, 'python3', 'bin'), + GIT_BIN_RELDIR=os.path.join(args.bootstrap_name, 'git'), + GIT_BIN_RELDIR_UNIX=posixpath.join(args.bootstrap_name, 'git')) - win_tools_dir = os.path.join(ROOT_DIR, args.win_tools_name) - git_postprocess(template, os.path.join(win_tools_dir, 'git')) + bootstrap_dir = os.path.join(ROOT_DIR, args.bootstrap_name) # Clean up any old Python and Git installations. - clean_up_old_installations(win_tools_dir) - - # Emit our Python bin depot-tools-relative directory. This is ready by - # "python.bat" to identify the path of the current Python installation. - # - # We use this indirection so that upgrades can change this pointer to - # redirect "python.bat" to a new Python installation. We can't just update - # "python.bat" because batch file executions reload the batch file and seek - # to the previous cursor in between every command, so changing the batch - # file contents could invalidate any existing executions. - # - # The intention is that the batch file itself never needs to change when - # switching Python versions. - maybe_update( - template.PYTHON_BIN_RELDIR, - os.path.join(ROOT_DIR, 'python_bin_reldir.txt')) + clean_up_old_installations(bootstrap_dir) + + # Only bootstrap git and Python 2 on Windows. + if IS_WIN: + git_postprocess(template, os.path.join(bootstrap_dir, 'git')) + + # Emit our Python bin depot-tools-relative directory. This is ready by + # "python.bat" to identify the path of the current Python installation. + # + # We use this indirection so that upgrades can change this pointer to + # redirect "python.bat" to a new Python installation. We can't just update + # "python.bat" because batch file executions reload the batch file and seek + # to the previous cursor in between every command, so changing the batch + # file contents could invalidate any existing executions. + # + # The intention is that the batch file itself never needs to change when + # switching Python versions. + maybe_update( + template.PYTHON_BIN_RELDIR, + os.path.join(ROOT_DIR, 'python_bin_reldir.txt')) + + python_template = 'python27.%s.bat' % ( + 'bleeding_edge' if args.bleeding_edge else 'new') + for src_name, dst_name in ( + ('git-bash.template.sh', 'git-bash'), + (python_template, 'python' + BAT_EXT), + ): + # Re-evaluate and regenerate our root templated files. + template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name)) + maybe_update( template.PYTHON3_BIN_RELDIR, os.path.join(ROOT_DIR, 'python3_bin_reldir.txt')) - python_bat_template = ('python27.new.bat' if not args.bleeding_edge - else 'python27.bleeding_edge.bat') - python3_bat_template = ('python3.new.bat' if not args.bleeding_edge - else 'python3.bleeding_edge.bat') + python3_template = 'python3.' + python3_template += 'bleeding_edge' if args.bleeding_edge else 'new' + python3_template += BAT_EXT - # Re-evaluate and regenerate our root templated files. - for src_name, dst_name in ( - ('git-bash.template.sh', 'git-bash'), - (python_bat_template, 'python.bat'), - (python3_bat_template, 'python3.bat'), - ): - template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name)) + template.maybe_install( + python3_template, os.path.join(ROOT_DIR, 'python3' + BAT_EXT)) return 0 diff --git a/bootstrap/win/git-bash.template.sh b/bootstrap/git-bash.template.sh similarity index 100% rename from bootstrap/win/git-bash.template.sh rename to bootstrap/git-bash.template.sh diff --git a/bootstrap/win/git.template.bat b/bootstrap/git.template.bat similarity index 100% rename from bootstrap/win/git.template.bat rename to bootstrap/git.template.bat diff --git a/bootstrap/manifest.txt b/bootstrap/manifest.txt new file mode 100644 index 000000000..cd3cb8957 --- /dev/null +++ b/bootstrap/manifest.txt @@ -0,0 +1,27 @@ +# CIPD manifest for bootstrapping tools. +# +# We must install anything that we want included on PATH to a different +# subdirectory than Git, as Git's msys bash strips its root directory +# from PATH, hence the subdirs. +# +# If any paths or package layouts change, updates will be required in +# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates. +# +# "win_tools.bat" has a hard requirement that the Python packages contain the +# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with +# the CIPD tag "version:VERSION". It uses this to extract VERSION. +# +# "bootstrap_python3" has a hard requirement that the Python package contains +# the string "cpython3/" and ends with the CIPD tag +# "version:VERSION". It uses this to extract VERSION. + +$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64 + +@Subdir python +infra/python/cpython/${platform} version:2.7.15.chromium14 + +@Subdir python3 +infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1 + +@Subdir git +infra/git/${os}-${arch} version:2.21.0.chromium16 diff --git a/bootstrap/manifest_bleeding_edge.txt b/bootstrap/manifest_bleeding_edge.txt new file mode 100644 index 000000000..cd3cb8957 --- /dev/null +++ b/bootstrap/manifest_bleeding_edge.txt @@ -0,0 +1,27 @@ +# CIPD manifest for bootstrapping tools. +# +# We must install anything that we want included on PATH to a different +# subdirectory than Git, as Git's msys bash strips its root directory +# from PATH, hence the subdirs. +# +# If any paths or package layouts change, updates will be required in +# "win_tools.bat", "bootstrap.py" and "../bootstrap_python3" templates. +# +# "win_tools.bat" has a hard requirement that the Python packages contain the +# strings "cpython/" for Python 2 and "cpython3/" for Python 3, and ends with +# the CIPD tag "version:VERSION". It uses this to extract VERSION. +# +# "bootstrap_python3" has a hard requirement that the Python package contains +# the string "cpython3/" and ends with the CIPD tag +# "version:VERSION". It uses this to extract VERSION. + +$VerifiedPlatform windows-386 windows-amd64 linux-amd64 mac-amd64 + +@Subdir python +infra/python/cpython/${platform} version:2.7.15.chromium14 + +@Subdir python3 +infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1 + +@Subdir git +infra/git/${os}-${arch} version:2.21.0.chromium16 diff --git a/bootstrap/win/profile.d.python.sh b/bootstrap/profile.d.python.sh similarity index 100% rename from bootstrap/win/profile.d.python.sh rename to bootstrap/profile.d.python.sh diff --git a/bootstrap/win/python27.bleeding_edge.bat b/bootstrap/python27.bleeding_edge.bat similarity index 100% rename from bootstrap/win/python27.bleeding_edge.bat rename to bootstrap/python27.bleeding_edge.bat diff --git a/bootstrap/win/python27.new.bat b/bootstrap/python27.new.bat similarity index 100% rename from bootstrap/win/python27.new.bat rename to bootstrap/python27.new.bat diff --git a/bootstrap/python3.bleeding_edge b/bootstrap/python3.bleeding_edge new file mode 100644 index 000000000..3d8f4c693 --- /dev/null +++ b/bootstrap/python3.bleeding_edge @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +base_dir=$(dirname "$0") + +PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)" +PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH" +"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@" diff --git a/bootstrap/win/python3.bleeding_edge.bat b/bootstrap/python3.bleeding_edge.bat similarity index 100% rename from bootstrap/win/python3.bleeding_edge.bat rename to bootstrap/python3.bleeding_edge.bat diff --git a/bootstrap/python3.new b/bootstrap/python3.new new file mode 100644 index 000000000..3d8f4c693 --- /dev/null +++ b/bootstrap/python3.new @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +base_dir=$(dirname "$0") + +PYTHON3_BIN_RELDIR="$(cat $base_dir/python3_bin_reldir.txt | xargs echo)" +PATH="${PYTHON3_BIN_RELDIR}":"${PYTHON3_BIN_RELDIR}/Scripts":"$PATH" +"$base_dir/${PYTHON3_BIN_RELDIR}/python3" "$@" diff --git a/bootstrap/win/python3.new.bat b/bootstrap/python3.new.bat similarity index 100% rename from bootstrap/win/python3.new.bat rename to bootstrap/python3.new.bat diff --git a/bootstrap/win/manifest.txt b/bootstrap/win/manifest.txt deleted file mode 100644 index 170feb7eb..000000000 --- a/bootstrap/win/manifest.txt +++ /dev/null @@ -1,23 +0,0 @@ -# CIPD manifest for Windows tools. -# -# We must install anything that we want included on PATH to a different -# subdirectory than Git, as Git's msys bash strips its root directory -# from PATH, hence the subdirs. -# -# If any paths or package layouts change, updates will be required in -# "win_tools.bat" and "win_tools.py" templates. -# -# "win_tools.bat" has a hard requirement that the Python package contains the -# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this -# to extract VERSION. - -$VerifiedPlatform windows-386 windows-amd64 - -@Subdir python -infra/python/cpython/${platform} version:2.7.15.chromium14 - -@Subdir python3 -infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1 - -@Subdir git -infra/git/windows-${arch} version:2.21.0.chromium16 diff --git a/bootstrap/win/manifest_bleeding_edge.txt b/bootstrap/win/manifest_bleeding_edge.txt deleted file mode 100644 index 170feb7eb..000000000 --- a/bootstrap/win/manifest_bleeding_edge.txt +++ /dev/null @@ -1,23 +0,0 @@ -# CIPD manifest for Windows tools. -# -# We must install anything that we want included on PATH to a different -# subdirectory than Git, as Git's msys bash strips its root directory -# from PATH, hence the subdirs. -# -# If any paths or package layouts change, updates will be required in -# "win_tools.bat" and "win_tools.py" templates. -# -# "win_tools.bat" has a hard requirement that the Python package contains the -# string "cpython" and ends with the CIPD tag "version:VERSION". It uses this -# to extract VERSION. - -$VerifiedPlatform windows-386 windows-amd64 - -@Subdir python -infra/python/cpython/${platform} version:2.7.15.chromium14 - -@Subdir python3 -infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1 - -@Subdir git -infra/git/windows-${arch} version:2.21.0.chromium16 diff --git a/bootstrap/win/win_tools.bat b/bootstrap/win_tools.bat similarity index 73% rename from bootstrap/win/win_tools.bat rename to bootstrap/win_tools.bat index 3f3174a19..2b53afec0 100644 --- a/bootstrap/win/win_tools.bat +++ b/bootstrap/win_tools.bat @@ -11,19 +11,19 @@ setlocal EnableDelayedExpansion :: Get absolute root directory (.js scripts don't handle relative paths well). -pushd %~dp0..\.. -set WIN_TOOLS_ROOT_DIR=%CD% +pushd %~dp0.. +set BOOTSTRAP_ROOT_DIR=%CD% popd :: Extra arguments to pass to our "win_tools.py" script. -set WIN_TOOLS_EXTRA_ARGS= +set BOOTSTRAP_EXTRA_ARGS= :: Determine if we're running a bleeding-edge installation. -if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" ( +if not exist "%BOOTSTRAP_ROOT_DIR%\.bleeding_edge" ( set CIPD_MANIFEST=manifest.txt ) else ( set CIPD_MANIFEST=manifest_bleeding_edge.txt - set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --bleeding-edge + set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bleeding-edge ) :: Parse our CIPD manifest and identify the "cpython" version. We do this by @@ -51,25 +51,25 @@ if "%PYTHON3_VERSION%" == "" ( ) :: We will take the version string, replace "." with "_", and surround it with -:: "win-tools-_bin" so that it matches "win_tools.py"'s cleanup +:: "bootstrap-_bin" so that it matches "win_tools.py"'s cleanup :: expression and ".gitignore". :: -:: We incorporate PYTHON_VERSION into the "win_tools" directory name so that +:: We incorporate PYTHON3_VERSION into the "win_tools" directory name so that :: new installations don't interfere with long-running Python processes if :: Python is upgraded. -set WIN_TOOLS_NAME=win_tools-%PYTHON_VERSION:.=_%_bin -set WIN_TOOLS_PATH=%WIN_TOOLS_ROOT_DIR%\%WIN_TOOLS_NAME% -set WIN_TOOLS_EXTRA_ARGS=%WIN_TOOLS_EXTRA_ARGS% --win-tools-name "%WIN_TOOLS_NAME%" +set BOOTSTRAP_NAME=bootstrap-%PYTHON3_VERSION:.=_%_bin +set BOOTSTRAP_PATH=%BOOTSTRAP_ROOT_DIR%\%BOOTSTRAP_NAME% +set BOOTSTRAP_EXTRA_ARGS=%BOOTSTRAP_EXTRA_ARGS% --bootstrap-name "%BOOTSTRAP_NAME%" :: Install our CIPD packages. The CIPD client self-bootstraps. :: See "//cipd.bat" and "//cipd.ps1" for more information. -set CIPD_EXE=%WIN_TOOLS_ROOT_DIR%\cipd.bat -call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%WIN_TOOLS_PATH%" +set CIPD_EXE=%BOOTSTRAP_ROOT_DIR%\cipd.bat +call "%CIPD_EXE%" ensure -log-level warning -ensure-file "%~dp0%CIPD_MANIFEST%" -root "%BOOTSTRAP_PATH%" if errorlevel 1 goto :END :: This executes "win_tools.py" using the bundle's Python interpreter. -set WIN_TOOLS_PYTHON_BIN=%WIN_TOOLS_PATH%\python\bin\python.exe -call "%WIN_TOOLS_PYTHON_BIN%" "%~dp0win_tools.py" %WIN_TOOLS_EXTRA_ARGS% +set BOOTSTRAP_PYTHON_BIN=%BOOTSTRAP_PATH%\python3\bin\python3.exe +call "%BOOTSTRAP_PYTHON_BIN%" "%~dp0bootstrap.py" %BOOTSTRAP_EXTRA_ARGS% :END diff --git a/bootstrap_python3 b/bootstrap_python3 new file mode 100644 index 000000000..86e1a920b --- /dev/null +++ b/bootstrap_python3 @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +function bootstrap_python3 { + base_dir=$(dirname "${BASH_SOURCE[0]}") + + cd $base_dir + + if [ -e ".bleeding_edge" ]; then + CIPD_MANIFEST="bootstrap/manifest_bleeding_edge.txt" + else + CIPD_MANIFEST="bootstrap/manifest.txt" + fi + + PYTHON3_VERSION="$(awk 'match($0, /cpython3\/.*version:([.0-9a-zA-Z]+)/,a) { print a[1]; }' $CIPD_MANIFEST)" + if [ "X$PYTHON3_VERSION" == "X" ]; then + echo Could not extract Python 3 version from manifest. + return 1 + fi + + BOOTSTRAP_NAME="bootstrap-${PYTHON3_VERSION}_bin" + BOOTSTRAP_PATH="$BOOTSTRAP_NAME" + + # Install CIPD packages. The CIPD client self-bootstraps. + "cipd" ensure -log-level warning -ensure-file "${CIPD_MANIFEST}" \ + -root "$BOOTSTRAP_PATH" + + BOOTSTRAP_PYTHON_BIN="${BOOTSTRAP_PATH}/python3/bin/python3" + "$BOOTSTRAP_PYTHON_BIN" "bootstrap/bootstrap.py" --bootstrap-name $BOOTSTRAP_PATH + + cd - > /dev/null +} diff --git a/ensure_bootstrap b/ensure_bootstrap index 32e7c3c68..47e89940d 100755 --- a/ensure_bootstrap +++ b/ensure_bootstrap @@ -9,10 +9,6 @@ # # Unlike "update_depot_tools", this script works with the current # checkout, and will not update/sync the "depot_tools" repository. -# -# TODO: This duplicates logic in "update_depot_tools". Update that script to -# invoke this script instead of manually calling "cipd_bin_setup" and other -# operations. # Test if this script is running under a MinGW install. If it is, we will # hardcode the paths to Git where possible. @@ -28,6 +24,11 @@ else fi fi +if [ "X$DEPOT_TOOLS_BOOTSTRAP_PYTHON3" != "X0" ]; then + source "$base_dir/bootstrap_python3" + bootstrap_python3 +fi + # Sync CIPD-boostrapped packages. source "$base_dir/cipd_bin_setup.sh" cipd_bin_setup diff --git a/update_depot_tools b/update_depot_tools index 12e6a9605..71caed22b 100755 --- a/update_depot_tools +++ b/update_depot_tools @@ -118,10 +118,7 @@ if [ "X$DEPOT_TOOLS_UPDATE" != "X0" ]; then cd - > /dev/null fi - # Sync CIPD and CIPD client tools. - source "$base_dir/cipd_bin_setup.sh" - cipd_bin_setup - - find "$base_dir" | grep -i ".*\.pyc" | xargs rm -f; + # Ensure depot_tools is bootstrapped + $base_dir/ensure_bootstrap fi diff --git a/update_depot_tools.bat b/update_depot_tools.bat index 978623d72..97b18f4d1 100644 --- a/update_depot_tools.bat +++ b/update_depot_tools.bat @@ -24,7 +24,7 @@ IF EXIST "%DEPOT_TOOLS_DIR%.disable_auto_update" GOTO :EOF set GIT_URL=https://chromium.googlesource.com/chromium/tools/depot_tools.git :: Will download git and python. -call "%DEPOT_TOOLS_DIR%bootstrap\win\win_tools.bat" +call "%DEPOT_TOOLS_DIR%bootstrap\win_tools.bat" if errorlevel 1 goto :EOF :: Now clear errorlevel so it can be set by other programs later. set errorlevel=