From a44d67c6e809dcb6a30081bc906c3bfa7ab41ec4 Mon Sep 17 00:00:00 2001 From: Edward Lemur Date: Tue, 20 Aug 2019 00:52:42 +0000 Subject: [PATCH] depot_tools: Add python3 to win bootstrap. Change-Id: Id24a1dc88ed9766b15a5145cdb29145d0c436514 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1759277 Commit-Queue: Edward Lesmes Reviewed-by: Robbie Iannucci --- bootstrap/win/README.md | 42 ++++++++++++++-------- bootstrap/win/git-bash.template.sh | 2 +- bootstrap/win/manifest.txt | 3 ++ bootstrap/win/manifest_bleeding_edge.txt | 3 ++ bootstrap/win/python3.bleeding_edge.bat | 46 ++++++++++++++++++++++++ bootstrap/win/python3.new.bat | 46 ++++++++++++++++++++++++ bootstrap/win/win_tools.bat | 8 ++++- bootstrap/win/win_tools.py | 32 +++++++++++++---- 8 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 bootstrap/win/python3.bleeding_edge.bat create mode 100644 bootstrap/win/python3.new.bat diff --git a/bootstrap/win/README.md b/bootstrap/win/README.md index a286f3c46b..c19c57e0f9 100644 --- a/bootstrap/win/README.md +++ b/bootstrap/win/README.md @@ -12,7 +12,7 @@ ommodate this, and Python cleanup, we handle Python in two stages: fix-ups. ## Software bootstrapped - * Python (https://www.python.org/) + * Python 2 and 3 (https://www.python.org/) * Git for Windows (https://git-for-windows.github.io/) ## Mechanism @@ -60,10 +60,10 @@ than the rest of this README. After any modification to this script set, a test sequence should be run on a Windows bot. -The post-processing will regenerate "python.bat" to point to the current -Python instance. Any previous Python installations will stick around, but -new invocations will use the new instance. Old installations will die -off either due to processes terminating or systems restarting. When this +The post-processing will regenerate "python.bat" and "python3.bat" to point to +the current Python instance. Any previous Python installations will stick +around, but new invocations will use the new instance. Old installations will +die off either due to processes terminating or systems restarting. When this happens, they will be cleaned up by the post-processing script. Testing @@ -73,20 +73,25 @@ For each of the following test scenarios, run these commands and verify that they are working: ```bash -# Assert that `gclient` invocation will update (and do the update). +:: Assert that `gclient` invocation will update (and do the update). gclient version -# Assert that Python fundamentally works. -python -c "import psutil; print dir(psutil)" +:: Assert that Python fundamentally works. +python -c "import Queue; print dir(Queue)" -# Assert that Python scripts work from `cmd.exe`. +:: Assert that Python 3 fundamentally works. +python3 -c "import queue; print(dir(queue))" + +:: Assert that Python scripts work from `cmd.exe`. git map-branches -# Assert that `git bash` works. +:: Assert that `git bash` works. git bash ## (Within `git bash`) assert that Python fundamentally works. -python -c "import psutil; print dir(psutil)" +python -c "import Queue; print dir(Queue)" +## (Within `git bash`) assert that Python 3 fundamentally works. +python3 -c "import queue; print(dir(queue))" ## (Within `git bash`) assert that Python scripts work. git map-branches ``` @@ -98,9 +103,12 @@ Run this sequence through the following upgrade/downgrade procedures: - Run through test steps. - Test upgrade to bleeding edge (if it differs). - Run `python.bat` in another shell, keep it open + - Run `python3.bat` in another shell, keep it open - Add `.bleeding_edge` to `depot_tools` root. - Run through test steps. - - In the old `python.bat` shell, run `import psutil`, confirm that it + - In the old `python.bat` shell, run `import Queue`, confirm that it + works. + - In the old `python3.bat` shell, run `import queue`, confirm that it works. - Close the Python shell, run `gclient version`, ensure that old directory is cleaned. @@ -110,9 +118,12 @@ Run this sequence through the following upgrade/downgrade procedures: - Run through test steps. - Test downgrade to default (if it differs). - Run `python.bat` in another shell, keep it open + - Run `python3.bat` in another shell, keep it open - Delete `.bleeding_edge` from `depot_tools` root. - Run through test steps. - - In the old `python.bat` shell, run `import psutil`, confirm that it + - In the old `python.bat` shell, run `import Queue`, confirm that it + works. + - In the old `python3.bat` shell, run `import queue`, confirm that it works. - Close the Python shell, run `gclient version`, ensure that old directory is cleaned. @@ -120,9 +131,12 @@ Run this sequence through the following upgrade/downgrade procedures: - Clean `depot_tools` via: `git clean -x -f -d .` - Run `gclient version` to load defaults. - Run `python.bat` in another shell, keep it open + - Run `python3.bat` in another shell, keep it open - Add `.bleeding_edge` to `depot_tools` root. - Run through test steps. - - In the old `python.bat` shell, run `import psutil`, confirm that it + - In the old `python.bat` shell, run `import Queue`, confirm that it + works. + - In the old `python3.bat` shell, run `import queue`, confirm that it works. - Close the Python shell, run `gclient version`, ensure that old directory is cleaned. diff --git a/bootstrap/win/git-bash.template.sh b/bootstrap/win/git-bash.template.sh index 340856de49..b78e08f79a 100755 --- a/bootstrap/win/git-bash.template.sh +++ b/bootstrap/win/git-bash.template.sh @@ -2,7 +2,7 @@ export EDITOR=${EDITOR:=notepad} WIN_BASE=`dirname $0` UNIX_BASE=`cygpath "$WIN_BASE"` -export PATH="$PATH:$UNIX_BASE/${PYTHON_BIN_RELDIR_UNIX}:$UNIX_BASE/${PYTHON_BIN_RELDIR_UNIX}/Scripts" +export PATH="$PATH:$UNIX_BASE/${PYTHON_BIN_RELDIR_UNIX}:$UNIX_BASE/${PYTHON_BIN_RELDIR_UNIX}/Scripts:$UNIX_BASE/${PYTHON3_BIN_RELDIR_UNIX}:$UNIX_BASE/${PYTHON3_BIN_RELDIR_UNIX}/Scripts" export PYTHON_DIRECT=1 export PYTHONUNBUFFERED=1 if [[ $# > 0 ]]; then diff --git a/bootstrap/win/manifest.txt b/bootstrap/win/manifest.txt index 64f3e03563..170feb7eb3 100644 --- a/bootstrap/win/manifest.txt +++ b/bootstrap/win/manifest.txt @@ -16,5 +16,8 @@ $VerifiedPlatform windows-386 windows-amd64 @Subdir python infra/python/cpython/${platform} version:2.7.15.chromium14 +@Subdir python3 +infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1 + @Subdir git infra/git/windows-${arch} version:2.21.0.chromium16 diff --git a/bootstrap/win/manifest_bleeding_edge.txt b/bootstrap/win/manifest_bleeding_edge.txt index 64f3e03563..170feb7eb3 100644 --- a/bootstrap/win/manifest_bleeding_edge.txt +++ b/bootstrap/win/manifest_bleeding_edge.txt @@ -16,5 +16,8 @@ $VerifiedPlatform windows-386 windows-amd64 @Subdir python infra/python/cpython/${platform} version:2.7.15.chromium14 +@Subdir python3 +infra/python/cpython3/${os}-${arch=amd64} version:3.8.0b1.chromium.1 + @Subdir git infra/git/windows-${arch} version:2.21.0.chromium16 diff --git a/bootstrap/win/python3.bleeding_edge.bat b/bootstrap/win/python3.bleeding_edge.bat new file mode 100644 index 0000000000..6412fe464a --- /dev/null +++ b/bootstrap/win/python3.bleeding_edge.bat @@ -0,0 +1,46 @@ +@echo off +:: Copyright 2017 The Chromium Authors. All rights reserved. +:: Use of this source code is governed by a BSD-style license that can be +:: found in the LICENSE file. + +setlocal +set PYTHON_BAT_RUNNER=1 + +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +:: This file is automatically generated by "bootstrap\win\win_tools.py", and +:: should not be modified. +:: +:: The previous "::" block acts as a nop-sled. Each time a batch file executes +:: a command, it reloads itself and seeks to its previous execution offset to +:: begin execution. Updating this batch file is, therefore, risky, since any +:: running Python instance that is using the old batch file will reload the new +:: batch file once the Python command terminates and resume at some unknown +:: offset. +:: +:: With the sled in place, a previous instance will resume mid-label. We are +:: assuming that the offset of the Python invocation is greater than the +:: PYTHON_BAT_RUNNER set command, which is the case since the old instance has +:: a large PATH set block before the Python execution. Old instances will hit +:: the next block of code without PYTHON_BAT_RUNNER set. New instances will have +:: it set. +:: +:: We remedy this in the future by having the batch file load its core paths +:: from an external file with for/set, removing the need to modify "python.bat" +:: during upgrade. +:: +:: After all of the old batch files are believed to be replaced, we can remove +:: the PYTHON_BAT_RUNNER block and the sled. For this update, old instances +:: will resume past the end of the file and terminate. + +if not "%PYTHON_BAT_RUNNER%" == "1" goto :END + +for /f %%i in (%~dp0python3_bin_reldir.txt) do set PYTHON3_BIN_RELDIR=%%i +set PATH=%~dp0%PYTHON3_BIN_RELDIR%;%~dp0%PYTHON3_BIN_RELDIR%\Scripts;%PATH% +"%~dp0%PYTHON3_BIN_RELDIR%\python3.exe" %* + +:END diff --git a/bootstrap/win/python3.new.bat b/bootstrap/win/python3.new.bat new file mode 100644 index 0000000000..6412fe464a --- /dev/null +++ b/bootstrap/win/python3.new.bat @@ -0,0 +1,46 @@ +@echo off +:: Copyright 2017 The Chromium Authors. All rights reserved. +:: Use of this source code is governed by a BSD-style license that can be +:: found in the LICENSE file. + +setlocal +set PYTHON_BAT_RUNNER=1 + +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +:: This file is automatically generated by "bootstrap\win\win_tools.py", and +:: should not be modified. +:: +:: The previous "::" block acts as a nop-sled. Each time a batch file executes +:: a command, it reloads itself and seeks to its previous execution offset to +:: begin execution. Updating this batch file is, therefore, risky, since any +:: running Python instance that is using the old batch file will reload the new +:: batch file once the Python command terminates and resume at some unknown +:: offset. +:: +:: With the sled in place, a previous instance will resume mid-label. We are +:: assuming that the offset of the Python invocation is greater than the +:: PYTHON_BAT_RUNNER set command, which is the case since the old instance has +:: a large PATH set block before the Python execution. Old instances will hit +:: the next block of code without PYTHON_BAT_RUNNER set. New instances will have +:: it set. +:: +:: We remedy this in the future by having the batch file load its core paths +:: from an external file with for/set, removing the need to modify "python.bat" +:: during upgrade. +:: +:: After all of the old batch files are believed to be replaced, we can remove +:: the PYTHON_BAT_RUNNER block and the sled. For this update, old instances +:: will resume past the end of the file and terminate. + +if not "%PYTHON_BAT_RUNNER%" == "1" goto :END + +for /f %%i in (%~dp0python3_bin_reldir.txt) do set PYTHON3_BIN_RELDIR=%%i +set PATH=%~dp0%PYTHON3_BIN_RELDIR%;%~dp0%PYTHON3_BIN_RELDIR%\Scripts;%PATH% +"%~dp0%PYTHON3_BIN_RELDIR%\python3.exe" %* + +:END diff --git a/bootstrap/win/win_tools.bat b/bootstrap/win/win_tools.bat index 17844d2ff7..3f3174a196 100644 --- a/bootstrap/win/win_tools.bat +++ b/bootstrap/win/win_tools.bat @@ -36,13 +36,19 @@ if not exist "%WIN_TOOLS_ROOT_DIR%\.bleeding_edge" ( :: delayed expansion variables. for /F "tokens=*" %%A in (%~dp0%CIPD_MANIFEST%) do ( set LINE=%%A - if not "x!LINE:cpython=!" == "x!LINE!" set PYTHON_VERSION=!LINE:*version:=! + if not "x!LINE:cpython/=!" == "x!LINE!" set PYTHON_VERSION=!LINE:*version:=! + if not "x!LINE:cpython3/=!" == "x!LINE!" set PYTHON3_VERSION=!LINE:*version:=! ) if "%PYTHON_VERSION%" == "" ( @echo Could not extract Python version from manifest. set ERRORLEVEL=1 goto :END ) +if "%PYTHON3_VERSION%" == "" ( + @echo Could not extract Python version from manifest. + set ERRORLEVEL=1 + goto :END +) :: We will take the version string, replace "." with "_", and surround it with :: "win-tools-_bin" so that it matches "win_tools.py"'s cleanup diff --git a/bootstrap/win/win_tools.py b/bootstrap/win/win_tools.py index dc16d2bc71..db1a8991e0 100644 --- a/bootstrap/win/win_tools.py +++ b/bootstrap/win/win_tools.py @@ -38,7 +38,8 @@ STUBS = { # Accumulated template parameters for generated stubs. class Template(collections.namedtuple('Template', ( 'PYTHON_RELDIR', 'PYTHON_BIN_RELDIR', 'PYTHON_BIN_RELDIR_UNIX', - 'GIT_BIN_RELDIR', 'GIT_BIN_RELDIR_UNIX', 'GIT_PROGRAM', + 'PYTHON3_BIN_RELDIR', 'PYTHON3_BIN_RELDIR_UNIX', 'GIT_BIN_RELDIR', + 'GIT_BIN_RELDIR_UNIX', 'GIT_PROGRAM', ))): @classmethod @@ -164,12 +165,20 @@ def _toolchain_in_use(toolchain_path): os.path.join(toolchain_path, 'python', 'bin'), # CIPD toolchain_path, # Legacy ZIP distributions. ): - for component in ( - os.path.join(python_dir, 'python.exe'), - os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), - ): - if os.path.isfile(component) and _in_use(component): - return True + for component in ( + os.path.join(python_dir, 'python.exe'), + os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), + ): + if os.path.isfile(component) and _in_use(component): + return True + # Look for Pytho:n 3 files that may be in use. + python_dir = os.path.join(toolchain_path, 'python3', 'bin') + for component in ( + os.path.join(python_dir, 'python3.exe'), + os.path.join(python_dir, 'DLLs', 'unicodedata.pyd'), + ): + if os.path.isfile(component) and _in_use(component): + return True return False @@ -297,6 +306,9 @@ def main(argv): PYTHON_BIN_RELDIR=os.path.join(args.win_tools_name, 'python', 'bin'), PYTHON_BIN_RELDIR_UNIX=posixpath.join( args.win_tools_name, 'python', 'bin'), + PYTHON3_BIN_RELDIR=os.path.join(args.win_tools_name, 'python3', 'bin'), + PYTHON3_BIN_RELDIR_UNIX=posixpath.join( + args.win_tools_name, 'python3', 'bin'), GIT_BIN_RELDIR=os.path.join(args.win_tools_name, 'git'), GIT_BIN_RELDIR_UNIX=posixpath.join(args.win_tools_name, 'git')) @@ -320,15 +332,21 @@ def main(argv): maybe_update( template.PYTHON_BIN_RELDIR, os.path.join(ROOT_DIR, 'python_bin_reldir.txt')) + maybe_update( + template.PYTHON3_BIN_RELDIR, + os.path.join(ROOT_DIR, 'python3_bin_reldir.txt')) python_bat_template = ('python27.new.bat' if not args.bleeding_edge else 'python27.bleeding_edge.bat') + python3_bat_template = ('python3.new.bat' if not args.bleeding_edge + else 'python3.bleeding_edge.bat') # Re-evaluate and regenerate our root templated files. for src_name, dst_name in ( ('git-bash.template.sh', 'git-bash'), ('pylint.new.bat', 'pylint.bat'), (python_bat_template, 'python.bat'), + (python3_bat_template, 'python3.bat'), ): template.maybe_install(src_name, os.path.join(ROOT_DIR, dst_name))