From 607284d7190a11b77dba7567bc867d63329ea2ba Mon Sep 17 00:00:00 2001 From: Junji Watanabe Date: Thu, 20 Apr 2023 03:14:52 +0000 Subject: [PATCH] [siso] add autosiso wrappers Developers will be able to use `autosiso` command to invoke siso builds. This CL extracts reclient management logic from `ninja_reclient` to be reused in both Ninja and Siso builds. Bug: b/278675516 Change-Id: I3e64a3188db184f4d3f851063a0feef7d3a73d6d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4445366 Auto-Submit: Junji Watanabe Reviewed-by: Philipp Wollermann Commit-Queue: Joanna Wang Reviewed-by: Fumitoshi Ukai Reviewed-by: Takuto Ikuta Reviewed-by: Joanna Wang --- OWNERS | 3 + autosiso | 8 +++ autosiso.bat | 12 ++++ autosiso.py | 34 +++++++++++ ninja_reclient.py | 140 +++----------------------------------------- reclient_helper.py | 143 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 132 deletions(-) create mode 100755 autosiso create mode 100644 autosiso.bat create mode 100755 autosiso.py create mode 100644 reclient_helper.py diff --git a/OWNERS b/OWNERS index cbbf94e8f..d6b813056 100644 --- a/OWNERS +++ b/OWNERS @@ -11,12 +11,15 @@ per-file autoninja*=dpranke@google.com per-file gn*=dpranke@google.com +per-file autosiso*=file://BUILD_OWNERS per-file ninja*=dpranke@google.com per-file ninja*=thakis@chromium.org per-file ninja_reclient.py=file://BUILD_OWNERS per-file ninja_reclient.py=file://RECLIENT_OWNERS per-file ninjalog*=tikuta@chromium.org per-file post_build_ninja_summary.py=brucedawson@chromium.org +per-file reclient_helper.py=file://BUILD_OWNERS +per-file reclient_helper.py=file://RECLIENT_OWNERS per-file reclientreport*=file://RECLIENT_OWNERS per-file presubmit*.py=brucedawson@chromium.org diff --git a/autosiso b/autosiso new file mode 100755 index 000000000..7abd9f729 --- /dev/null +++ b/autosiso @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright 2023 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +base_dir=$(dirname "$0") +PYTHONDONTWRITEBYTECODE=1 exec python3 "$base_dir/autosiso.py" "$@" diff --git a/autosiso.bat b/autosiso.bat new file mode 100644 index 000000000..5cb658150 --- /dev/null +++ b/autosiso.bat @@ -0,0 +1,12 @@ +@echo off +:: Copyright 2023 The Chromium Authors +:: Use of this source code is governed by a BSD-style license that can be +:: found in the LICENSE file. +setlocal + +:: Ensure that "depot_tools" is somewhere in PATH so this tool can be used +:: standalone, but allow other PATH manipulations to take priority. +set PATH=%PATH%;%~dp0 + +:: Defer control. +python3 "%~dp0\autosiso.py" "%*" diff --git a/autosiso.py b/autosiso.py new file mode 100755 index 000000000..1dce46315 --- /dev/null +++ b/autosiso.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +""" +Developers invoke this script via autosiso or autosiso.bat to simply run +Siso builds. +""" + +import sys + +import reclient_helper +import siso + + +def main(argv): + with reclient_helper.build_context(argv) as ret_code: + if ret_code: + return ret_code + argv = [ + argv[0], + 'ninja', + # Do not authenticate when using Reproxy. + '-project=', + '-reapi_instance=', + ] + argv[1:] + return siso.main(argv) + + +if __name__ == '__main__': + try: + sys.exit(main(sys.argv)) + except KeyboardInterrupt: + sys.exit(1) diff --git a/ninja_reclient.py b/ninja_reclient.py index 1ed00d83a..bef10ab17 100755 --- a/ninja_reclient.py +++ b/ninja_reclient.py @@ -7,144 +7,20 @@ handles the client lifecycle safely. It will automatically start reproxy before running ninja and stop reproxy when ninja stops for any reason eg. build completes, keyboard interupt etc.""" -import hashlib -import os -import subprocess import sys import ninja -import gclient_paths - - -def find_reclient_bin_dir(): - tools_path = gclient_paths.GetBuildtoolsPath() - if not tools_path: - return None - - reclient_bin_dir = os.path.join(tools_path, 'reclient') - if os.path.isdir(reclient_bin_dir): - return reclient_bin_dir - return None - - -def find_reclient_cfg(): - tools_path = gclient_paths.GetBuildtoolsPath() - if not tools_path: - return None - - reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg') - if os.path.isfile(reclient_cfg): - return reclient_cfg - return None - - -def run(cmd_args): - if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': - print(' '.join(cmd_args)) - return subprocess.call(cmd_args) - - -def start_reproxy(reclient_cfg, reclient_bin_dir): - return run([ - os.path.join(reclient_bin_dir, 'bootstrap'), - '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy'), - '--cfg=' + reclient_cfg - ]) - - -def stop_reproxy(reclient_cfg, reclient_bin_dir): - return run([ - os.path.join(reclient_bin_dir, 'bootstrap'), '--shutdown', - '--cfg=' + reclient_cfg - ]) - - -def find_rel_ninja_out_dir(args): - # Ninja uses getopt_long, which allow to intermix non-option arguments. - # To leave non supported parameters untouched, we do not use getopt. - for index, arg in enumerate(args[1:]): - if arg == '-C': - # + 1 to get the next argument and +1 because we trimmed off args[0] - return args[index + 2] - if arg.startswith('-C'): - # Support -Cout/Default - return arg[2:] - return '.' - - -def set_reproxy_path_flags(out_dir, make_dirs=True): - """Helper to setup the logs and cache directories for reclient - - Creates the following directory structure if make_dirs is true: - out_dir/ - .reproxy_tmp/ - logs/ - cache/ - - The following env vars are set if not already set: - RBE_output_dir=out_dir/.reproxy_tmp/logs - RBE_proxy_log_dir=out_dir/.reproxy_tmp/logs - RBE_log_dir=out_dir/.reproxy_tmp/logs - RBE_cache_dir=out_dir/.reproxy_tmp/cache - *Nix Only: - RBE_server_address=unix://out_dir/.reproxy_tmp/reproxy.sock - Windows Only: - RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe - """ - tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp')) - log_dir = os.path.join(tmp_dir, 'logs') - cache_dir = os.path.join(tmp_dir, 'cache') - if make_dirs: - os.makedirs(tmp_dir, exist_ok=True) - os.makedirs(log_dir, exist_ok=True) - os.makedirs(cache_dir, exist_ok=True) - os.environ.setdefault("RBE_output_dir", log_dir) - os.environ.setdefault("RBE_proxy_log_dir", log_dir) - os.environ.setdefault("RBE_log_dir", log_dir) - os.environ.setdefault("RBE_cache_dir", cache_dir) - if sys.platform.startswith('win'): - pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest() - os.environ.setdefault("RBE_server_address", - "pipe://%s/reproxy.pipe" % pipe_dir) - else: - os.environ.setdefault("RBE_server_address", - "unix://%s/reproxy.sock" % tmp_dir) +import reclient_helper def main(argv): - # If use_remoteexec is set, but the reclient binaries or configs don't - # exist, display an error message and stop. Otherwise, the build will - # attempt to run with rewrapper wrapping actions, but will fail with - # possible non-obvious problems. - # As of January 2023, dev builds with reclient are not supported, so - # indicate that use_goma should be swapped for use_remoteexec. This - # message will be changed when dev builds are fully supported. - reclient_bin_dir = find_reclient_bin_dir() - reclient_cfg = find_reclient_cfg() - if reclient_bin_dir is None or reclient_cfg is None: - print(("Build is configured to use reclient but necessary binaries " - "or config files can't be found. Developer builds with " - "reclient are not yet supported. Try regenerating your " - "build with use_goma in place of use_remoteexec for now."), - file=sys.stderr) - return 1 - try: - set_reproxy_path_flags(find_rel_ninja_out_dir(argv)) - except OSError: - print("Error creating reproxy_tmp in output dir", file=sys.stderr) - return 1 - reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir) - if reproxy_ret_code != 0: - return reproxy_ret_code - try: - return ninja.main(argv) - except KeyboardInterrupt: - print("Caught User Interrupt", file=sys.stderr) - # Suppress python stack trace if ninja is interrupted - return 1 - finally: - print("Shutting down reproxy...", file=sys.stderr) - stop_reproxy(reclient_cfg, reclient_bin_dir) + with reclient_helper.build_context(argv) as ret_code: + if ret_code: + return ret_code + try: + return ninja.main(argv) + except KeyboardInterrupt: + return 1 if __name__ == '__main__': diff --git a/reclient_helper.py b/reclient_helper.py new file mode 100644 index 000000000..54f258d20 --- /dev/null +++ b/reclient_helper.py @@ -0,0 +1,143 @@ +# Copyright 2023 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""This helper provides a build context that handles +the reclient lifecycle safely. It will automatically start +reproxy before running ninja and stop reproxy when build stops +for any reason e.g. build completion, keyboard interrupt etc.""" + +import contextlib +import hashlib +import os +import subprocess +import sys + +import gclient_paths + + +def find_reclient_bin_dir(): + tools_path = gclient_paths.GetBuildtoolsPath() + if not tools_path: + return None + + reclient_bin_dir = os.path.join(tools_path, 'reclient') + if os.path.isdir(reclient_bin_dir): + return reclient_bin_dir + return None + + +def find_reclient_cfg(): + tools_path = gclient_paths.GetBuildtoolsPath() + if not tools_path: + return None + + reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg') + if os.path.isfile(reclient_cfg): + return reclient_cfg + return None + + +def run(cmd_args): + if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': + print(' '.join(cmd_args)) + return subprocess.call(cmd_args) + + +def start_reproxy(reclient_cfg, reclient_bin_dir): + return run([ + os.path.join(reclient_bin_dir, 'bootstrap'), + '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy'), + '--cfg=' + reclient_cfg + ]) + + +def stop_reproxy(reclient_cfg, reclient_bin_dir): + return run([ + os.path.join(reclient_bin_dir, 'bootstrap'), '--shutdown', + '--cfg=' + reclient_cfg + ]) + + +def find_ninja_out_dir(args): + # Ninja uses getopt_long, which allows to intermix non-option arguments. + # To leave non supported parameters untouched, we do not use getopt. + for index, arg in enumerate(args[1:]): + if arg == '-C': + # + 1 to get the next argument and +1 because we trimmed off args[0] + return args[index + 2] + if arg.startswith('-C'): + # Support -Cout/Default + return arg[2:] + return '.' + + +def set_reproxy_path_flags(out_dir, make_dirs=True): + """Helper to setup the logs and cache directories for reclient. + + Creates the following directory structure if make_dirs is true: + out_dir/ + .reproxy_tmp/ + logs/ + cache/ + + The following env vars are set if not already set: + RBE_output_dir=out_dir/.reproxy_tmp/logs + RBE_proxy_log_dir=out_dir/.reproxy_tmp/logs + RBE_log_dir=out_dir/.reproxy_tmp/logs + RBE_cache_dir=out_dir/.reproxy_tmp/cache + *Nix Only: + RBE_server_address=unix://out_dir/.reproxy_tmp/reproxy.sock + Windows Only: + RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe + """ + tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp')) + log_dir = os.path.join(tmp_dir, 'logs') + cache_dir = os.path.join(tmp_dir, 'cache') + if make_dirs: + os.makedirs(tmp_dir, exist_ok=True) + os.makedirs(log_dir, exist_ok=True) + os.makedirs(cache_dir, exist_ok=True) + os.environ.setdefault("RBE_output_dir", log_dir) + os.environ.setdefault("RBE_proxy_log_dir", log_dir) + os.environ.setdefault("RBE_log_dir", log_dir) + os.environ.setdefault("RBE_cache_dir", cache_dir) + if sys.platform.startswith('win'): + pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest() + os.environ.setdefault("RBE_server_address", + "pipe://%s/reproxy.pipe" % pipe_dir) + else: + os.environ.setdefault("RBE_server_address", + "unix://%s/reproxy.sock" % tmp_dir) + + +@contextlib.contextmanager +def build_context(argv): + # If use_remoteexec is set, but the reclient binaries or configs don't + # exist, display an error message and stop. Otherwise, the build will + # attempt to run with rewrapper wrapping actions, but will fail with + # possible non-obvious problems. + reclient_bin_dir = find_reclient_bin_dir() + reclient_cfg = find_reclient_cfg() + if reclient_bin_dir is None or reclient_cfg is None: + print(("Build is configured to use reclient but necessary binaries " + "or config files can't be found. Developer builds with " + "reclient are not yet supported. Try regenerating your " + "build with use_goma in place of use_remoteexec for now."), + file=sys.stderr) + yield 1 + return + try: + set_reproxy_path_flags(find_ninja_out_dir(argv)) + except OSError: + print("Error creating reproxy_tmp in output dir", file=sys.stderr) + yield 1 + return + reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir) + if reproxy_ret_code != 0: + yield reproxy_ret_code + return + try: + yield + finally: + print("Shutting down reproxy...", file=sys.stderr) + stop_reproxy(reclient_cfg, reclient_bin_dir)