From 9b4d1e485d37a44e3ebe73a06d21fc79c527ec96 Mon Sep 17 00:00:00 2001 From: Mohamed Heikal Date: Thu, 12 Dec 2024 10:28:46 -0800 Subject: [PATCH] Integrate autoninja.py with fast_local_dev_server.py - Starts the build server when the build starts. - Writes tty filename to env variable. - Tells the build server about the current build so it does not exit until autoninja does, even if idle. - Cancels pending tasks on Ctrl+c. Change-Id: I86bb9852bd0975f381b049b9ff21c38eef7cef9d Bug: 370589852 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5917985 Reviewed-by: Junji Watanabe Reviewed-by: Josip Sokcevic Reviewed-by: Fumitoshi Ukai Auto-Submit: Mohamed Heikal Commit-Queue: Mohamed Heikal --- OWNERS | 2 ++ android_build_server_helper.py | 64 ++++++++++++++++++++++++++++++++++ autoninja.py | 59 ++++++++++++++++++------------- siso.py | 18 ++++++++-- 4 files changed, 116 insertions(+), 27 deletions(-) create mode 100644 android_build_server_helper.py diff --git a/OWNERS b/OWNERS index 1dd452a619..9ffc683084 100644 --- a/OWNERS +++ b/OWNERS @@ -20,6 +20,8 @@ per-file ninja*=dpranke@google.com per-file ninja*=thakis@chromium.org per-file ninja*=file://BUILD_OWNERS per-file post_build_ninja_summary.py=file://BUILD_OWNERS +per-file android_build_server_helper.py=mheikal@chromium.org +per-file android_build_server_helper.py=agrieve@chromium.org # GN per-file gn*=dpranke@google.com diff --git a/android_build_server_helper.py b/android_build_server_helper.py new file mode 100644 index 0000000000..a0a1fce21d --- /dev/null +++ b/android_build_server_helper.py @@ -0,0 +1,64 @@ +# Copyright 2024 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import contextlib +import os +import sys +import signal +import subprocess + +import gclient_paths + + +def _register_build_id(local_dev_server_path, build_id): + subprocess.run([ + local_dev_server_path, '--register-build-id', build_id, '--builder-pid', + str(os.getpid()) + ]) + + +def _print_status(local_dev_server_path, build_id): + subprocess.run([local_dev_server_path, '--print-status', build_id]) + + +def _get_server_path(): + src_dir = gclient_paths.GetPrimarySolutionPath() + return os.path.join(src_dir, 'build/android/fast_local_dev_server.py') + + +def _set_signal_handler(local_dev_server_path, build_id): + original_sigint_handler = signal.getsignal(signal.SIGINT) + + def _kill_handler(signum, frame): + # Cancel the pending build tasks if user CTRL+c early. + print('Canceling pending build_server tasks', file=sys.stderr) + subprocess.run([local_dev_server_path, '--cancel-build', build_id]) + original_sigint_handler(signum, frame) + + signal.signal(signal.SIGINT, _kill_handler) + + +def _start_server(local_dev_server_path): + subprocess.Popen([local_dev_server_path, '--exit-on-idle', '--quiet'], + start_new_session=True) + + +def _set_tty_env(): + stdout_name = os.readlink('/proc/self/fd/1') + os.environ.setdefault("AUTONINJA_STDOUT_NAME", stdout_name) + + +@contextlib.contextmanager +def build_server_context(build_id, use_android_build_server=False): + if not use_android_build_server: + yield + return + _set_tty_env() + server_path = _get_server_path() + _start_server(server_path) + # Tell the build server about us. + _register_build_id(server_path, build_id) + _set_signal_handler(server_path, build_id) + yield + _print_status(server_path, build_id) diff --git a/autoninja.py b/autoninja.py index 3a958fbc3d..73fe37f0e5 100755 --- a/autoninja.py +++ b/autoninja.py @@ -15,7 +15,6 @@ settings. """ import importlib.util -import logging import multiprocessing import os import platform @@ -28,6 +27,7 @@ import time import uuid import warnings +import android_build_server_helper import build_telemetry import gclient_paths import gclient_utils @@ -226,6 +226,7 @@ def _main_inner(input_args, build_id, should_collect_logs=False): use_remoteexec = False use_reclient = _get_use_reclient_value(output_dir) use_siso = _get_use_siso_default(output_dir) + use_android_build_server = False # Attempt to auto-detect remote build acceleration. We support gn-based # builds, where we look for args.gn in the build tree, and cmake-based @@ -257,6 +258,9 @@ def _main_inner(input_args, build_id, should_collect_logs=False): if k == "use_reclient" and v == "false": use_reclient = False continue + if k == "android_static_analysis" and v == '"build_server"': + use_android_build_server = True + continue if use_reclient is None: use_reclient = use_remoteexec @@ -322,28 +326,33 @@ def _main_inner(input_args, build_id, should_collect_logs=False): file=sys.stderr, ) return 1 + # Build ID consistently used in other tools. e.g. Reclient, ninjalog. os.environ.setdefault("SISO_BUILD_ID", build_id) - if use_remoteexec: - if use_reclient and not t_specified: - return reclient_helper.run_siso( - [ - 'siso', - 'ninja', - # Do not authenticate when using Reproxy. - '-project=', - '-reapi_instance=', - ] + input_args[1:], - should_collect_logs) - return siso.main(["siso", "ninja"] + input_args[1:]) - if not project: - project = _siso_rbe_project() - if not t_specified and project and not offline: - print( - 'Missing "use_remoteexec=true". No remote execution', - file=sys.stderr, - ) - return siso.main(["siso", "ninja", "--offline"] + input_args[1:]) + with android_build_server_helper.build_server_context( + build_id, + use_android_build_server=use_android_build_server): + if use_remoteexec: + if use_reclient and not t_specified: + return reclient_helper.run_siso( + [ + 'siso', + 'ninja', + # Do not authenticate when using Reproxy. + '-project=', + '-reapi_instance=', + ] + input_args[1:], + should_collect_logs) + return siso.main(["siso", "ninja"] + input_args[1:]) + if not project: + project = _siso_rbe_project() + if not t_specified and project and not offline: + print( + 'Missing "use_remoteexec=true". No remote execution', + file=sys.stderr, + ) + return siso.main(["siso", "ninja", "--offline"] + + input_args[1:]) if os.path.exists(siso_marker): print( @@ -443,9 +452,11 @@ def _main_inner(input_args, build_id, should_collect_logs=False): # are being used. _print_cmd(ninja_args) - if use_reclient and not t_specified: - return reclient_helper.run_ninja(ninja_args, should_collect_logs) - return ninja.main(ninja_args) + with android_build_server_helper.build_server_context( + build_id, use_android_build_server=use_android_build_server): + if use_reclient and not t_specified: + return reclient_helper.run_ninja(ninja_args, should_collect_logs) + return ninja.main(ninja_args) def _upload_ninjalog(args, exit_code, build_duration): diff --git a/siso.py b/siso.py index 328386c182..ed6996d114 100644 --- a/siso.py +++ b/siso.py @@ -39,9 +39,21 @@ def checkOutdir(args): def main(args): - # Propagate signals to siso process so that it can run cleanup steps. - # Siso will be terminated immediately after the second Ctrl-C. - signal.signal(signal.SIGINT, lambda signum, frame: None) + # Do not raise KeyboardInterrupt on SIGINT so as to give siso time to run + # cleanup tasks. Siso will be terminated immediately after the second + # Ctrl-C. + original_sigint_handler = signal.getsignal(signal.SIGINT) + + def _ignore(signum, frame): + try: + # Call the original signal handler. + original_sigint_handler(signum, frame) + except KeyboardInterrupt: + # Do not reraise KeyboardInterrupt so as to not kill siso too early. + pass + + signal.signal(signal.SIGINT, _ignore) + if not sys.platform.startswith('win'): signal.signal(signal.SIGTERM, lambda signum, frame: None)