[siso] add autosiso wrappers
Developers will be able to use `autosiso` command to invoke siso builds. This CL extracts reclient management logic from `ninja_reclient` to be reused in both Ninja and Siso builds. Bug: b/278675516 Change-Id: I3e64a3188db184f4d3f851063a0feef7d3a73d6d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4445366 Auto-Submit: Junji Watanabe <jwata@google.com> Reviewed-by: Philipp Wollermann <philwo@google.com> Commit-Queue: Joanna Wang <jojwang@chromium.org> Reviewed-by: Fumitoshi Ukai <ukai@google.com> Reviewed-by: Takuto Ikuta <tikuta@chromium.org> Reviewed-by: Joanna Wang <jojwang@chromium.org>changes/66/4445366/11
Junji Watanabe
3 years ago
committed by
LUCI CQ
parent
0d6cb8fa70
commit
607284d719
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2023 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
base_dir=$(dirname "$0")
|
||||
PYTHONDONTWRITEBYTECODE=1 exec python3 "$base_dir/autosiso.py" "$@"
|
||||
@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
:: Copyright 2023 The Chromium Authors
|
||||
:: Use of this source code is governed by a BSD-style license that can be
|
||||
:: found in the LICENSE file.
|
||||
setlocal
|
||||
|
||||
:: Ensure that "depot_tools" is somewhere in PATH so this tool can be used
|
||||
:: standalone, but allow other PATH manipulations to take priority.
|
||||
set PATH=%PATH%;%~dp0
|
||||
|
||||
:: Defer control.
|
||||
python3 "%~dp0\autosiso.py" "%*"
|
||||
@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""
|
||||
Developers invoke this script via autosiso or autosiso.bat to simply run
|
||||
Siso builds.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import reclient_helper
|
||||
import siso
|
||||
|
||||
|
||||
def main(argv):
|
||||
with reclient_helper.build_context(argv) as ret_code:
|
||||
if ret_code:
|
||||
return ret_code
|
||||
argv = [
|
||||
argv[0],
|
||||
'ninja',
|
||||
# Do not authenticate when using Reproxy.
|
||||
'-project=',
|
||||
'-reapi_instance=',
|
||||
] + argv[1:]
|
||||
return siso.main(argv)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
sys.exit(main(sys.argv))
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(1)
|
||||
@ -0,0 +1,143 @@
|
||||
# Copyright 2023 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""This helper provides a build context that handles
|
||||
the reclient lifecycle safely. It will automatically start
|
||||
reproxy before running ninja and stop reproxy when build stops
|
||||
for any reason e.g. build completion, keyboard interrupt etc."""
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import gclient_paths
|
||||
|
||||
|
||||
def find_reclient_bin_dir():
|
||||
tools_path = gclient_paths.GetBuildtoolsPath()
|
||||
if not tools_path:
|
||||
return None
|
||||
|
||||
reclient_bin_dir = os.path.join(tools_path, 'reclient')
|
||||
if os.path.isdir(reclient_bin_dir):
|
||||
return reclient_bin_dir
|
||||
return None
|
||||
|
||||
|
||||
def find_reclient_cfg():
|
||||
tools_path = gclient_paths.GetBuildtoolsPath()
|
||||
if not tools_path:
|
||||
return None
|
||||
|
||||
reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg')
|
||||
if os.path.isfile(reclient_cfg):
|
||||
return reclient_cfg
|
||||
return None
|
||||
|
||||
|
||||
def run(cmd_args):
|
||||
if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1':
|
||||
print(' '.join(cmd_args))
|
||||
return subprocess.call(cmd_args)
|
||||
|
||||
|
||||
def start_reproxy(reclient_cfg, reclient_bin_dir):
|
||||
return run([
|
||||
os.path.join(reclient_bin_dir, 'bootstrap'),
|
||||
'--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy'),
|
||||
'--cfg=' + reclient_cfg
|
||||
])
|
||||
|
||||
|
||||
def stop_reproxy(reclient_cfg, reclient_bin_dir):
|
||||
return run([
|
||||
os.path.join(reclient_bin_dir, 'bootstrap'), '--shutdown',
|
||||
'--cfg=' + reclient_cfg
|
||||
])
|
||||
|
||||
|
||||
def find_ninja_out_dir(args):
|
||||
# Ninja uses getopt_long, which allows to intermix non-option arguments.
|
||||
# To leave non supported parameters untouched, we do not use getopt.
|
||||
for index, arg in enumerate(args[1:]):
|
||||
if arg == '-C':
|
||||
# + 1 to get the next argument and +1 because we trimmed off args[0]
|
||||
return args[index + 2]
|
||||
if arg.startswith('-C'):
|
||||
# Support -Cout/Default
|
||||
return arg[2:]
|
||||
return '.'
|
||||
|
||||
|
||||
def set_reproxy_path_flags(out_dir, make_dirs=True):
|
||||
"""Helper to setup the logs and cache directories for reclient.
|
||||
|
||||
Creates the following directory structure if make_dirs is true:
|
||||
out_dir/
|
||||
.reproxy_tmp/
|
||||
logs/
|
||||
cache/
|
||||
|
||||
The following env vars are set if not already set:
|
||||
RBE_output_dir=out_dir/.reproxy_tmp/logs
|
||||
RBE_proxy_log_dir=out_dir/.reproxy_tmp/logs
|
||||
RBE_log_dir=out_dir/.reproxy_tmp/logs
|
||||
RBE_cache_dir=out_dir/.reproxy_tmp/cache
|
||||
*Nix Only:
|
||||
RBE_server_address=unix://out_dir/.reproxy_tmp/reproxy.sock
|
||||
Windows Only:
|
||||
RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe
|
||||
"""
|
||||
tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp'))
|
||||
log_dir = os.path.join(tmp_dir, 'logs')
|
||||
cache_dir = os.path.join(tmp_dir, 'cache')
|
||||
if make_dirs:
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
os.environ.setdefault("RBE_output_dir", log_dir)
|
||||
os.environ.setdefault("RBE_proxy_log_dir", log_dir)
|
||||
os.environ.setdefault("RBE_log_dir", log_dir)
|
||||
os.environ.setdefault("RBE_cache_dir", cache_dir)
|
||||
if sys.platform.startswith('win'):
|
||||
pipe_dir = hashlib.md5(tmp_dir.encode()).hexdigest()
|
||||
os.environ.setdefault("RBE_server_address",
|
||||
"pipe://%s/reproxy.pipe" % pipe_dir)
|
||||
else:
|
||||
os.environ.setdefault("RBE_server_address",
|
||||
"unix://%s/reproxy.sock" % tmp_dir)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def build_context(argv):
|
||||
# If use_remoteexec is set, but the reclient binaries or configs don't
|
||||
# exist, display an error message and stop. Otherwise, the build will
|
||||
# attempt to run with rewrapper wrapping actions, but will fail with
|
||||
# possible non-obvious problems.
|
||||
reclient_bin_dir = find_reclient_bin_dir()
|
||||
reclient_cfg = find_reclient_cfg()
|
||||
if reclient_bin_dir is None or reclient_cfg is None:
|
||||
print(("Build is configured to use reclient but necessary binaries "
|
||||
"or config files can't be found. Developer builds with "
|
||||
"reclient are not yet supported. Try regenerating your "
|
||||
"build with use_goma in place of use_remoteexec for now."),
|
||||
file=sys.stderr)
|
||||
yield 1
|
||||
return
|
||||
try:
|
||||
set_reproxy_path_flags(find_ninja_out_dir(argv))
|
||||
except OSError:
|
||||
print("Error creating reproxy_tmp in output dir", file=sys.stderr)
|
||||
yield 1
|
||||
return
|
||||
reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir)
|
||||
if reproxy_ret_code != 0:
|
||||
yield reproxy_ret_code
|
||||
return
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
print("Shutting down reproxy...", file=sys.stderr)
|
||||
stop_reproxy(reclient_cfg, reclient_bin_dir)
|
||||
Loading…
Reference in New Issue