Create reclient specific ninja wrapper to properly handle reproxy lifecyle
This is required as it is impossible to catch a keyboard interrupt in a windows batch file and we dont want zombie reproxy instances running on developer's machines for performance and metric collection reasons. Bug: b/264405266 Change-Id: I00f036c8f14451cdb1b99a5cad1c2af03dd57d57 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4171506 Reviewed-by: Dirk Pranke <dpranke@google.com> Auto-Submit: Ben Segall <bentekkie@google.com> Commit-Queue: Ben Segall <bentekkie@google.com>changes/06/4171506/13
Ben Segall
3 years ago
committed by
LUCI CQ
parent
7c6ebe7fd1
commit
eb2866e654
@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""This script is a wrapper around the ninja.py script that also
|
||||
handles the client lifecycle safely. It will automatically start
|
||||
reproxy before running ninja and stop reproxy when ninja stops
|
||||
for any reason eg. build completes, keyboard interupt etc."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import ninja
|
||||
import gclient_paths
|
||||
|
||||
|
||||
def find_reclient_bin_dir():
|
||||
tools_path = gclient_paths.GetBuildtoolsPath()
|
||||
if not tools_path:
|
||||
return None
|
||||
|
||||
reclient_bin_dir = os.path.join(tools_path, 'reclient')
|
||||
if os.path.isdir(reclient_bin_dir):
|
||||
return reclient_bin_dir
|
||||
return None
|
||||
|
||||
|
||||
def find_reclient_cfg():
|
||||
tools_path = gclient_paths.GetBuildtoolsPath()
|
||||
if not tools_path:
|
||||
return None
|
||||
|
||||
reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg')
|
||||
if os.path.isfile(reclient_cfg):
|
||||
return reclient_cfg
|
||||
return None
|
||||
|
||||
|
||||
def run(cmd_args):
|
||||
if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1':
|
||||
print(' '.join(cmd_args))
|
||||
return subprocess.call(cmd_args)
|
||||
|
||||
|
||||
def start_reproxy(reclient_cfg, reclient_bin_dir):
|
||||
return run([
|
||||
os.path.join(reclient_bin_dir, 'bootstrap'),
|
||||
'--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy'),
|
||||
'--cfg=' + reclient_cfg
|
||||
])
|
||||
|
||||
|
||||
def stop_reproxy(reclient_cfg, reclient_bin_dir):
|
||||
return run([
|
||||
os.path.join(reclient_bin_dir, 'bootstrap'), '--shutdown',
|
||||
'--cfg=' + reclient_cfg
|
||||
])
|
||||
|
||||
|
||||
def main(argv):
|
||||
# If use_remoteexec is set, but the reclient binaries or configs don't
|
||||
# exist, display an error message and stop. Otherwise, the build will
|
||||
# attempt to run with rewrapper wrapping actions, but will fail with
|
||||
# possible non-obvious problems.
|
||||
# As of January 2023, dev builds with reclient are not supported, so
|
||||
# indicate that use_goma should be swapped for use_remoteexec. This
|
||||
# message will be changed when dev builds are fully supported.
|
||||
reclient_bin_dir = find_reclient_bin_dir()
|
||||
reclient_cfg = find_reclient_cfg()
|
||||
if reclient_bin_dir is None or reclient_cfg is None:
|
||||
print(("Build is configured to use reclient but necessary binaries "
|
||||
"or config files can't be found. Developer builds with "
|
||||
"reclient are not yet supported. Try regenerating your "
|
||||
"build with use_goma in place of use_remoteexec for now."),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir)
|
||||
if reproxy_ret_code != 0:
|
||||
return reproxy_ret_code
|
||||
try:
|
||||
return ninja.main(argv)
|
||||
except KeyboardInterrupt:
|
||||
# Suppress python stack trace if ninja is interrupted
|
||||
return 1
|
||||
finally:
|
||||
stop_reproxy(reclient_cfg, reclient_bin_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
@ -1,3 +1,4 @@
|
||||
per-file autoninja_test.py=brucedawson@chromium.org
|
||||
per-file autoninja_test.py=tikuta@chromium.org
|
||||
per-file ninjalog_uploader_test.py=tikuta@chromium.org
|
||||
per-file ninja_reclient_test.py=file://GOMA_OWNERS
|
||||
|
||||
@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2023 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
import os
|
||||
import os.path
|
||||
import sys
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, ROOT_DIR)
|
||||
|
||||
import ninja_reclient
|
||||
from testing_support import trial_dir
|
||||
|
||||
|
||||
def write(filename, content):
|
||||
"""Writes the content of a file and create the directories as needed."""
|
||||
filename = os.path.abspath(filename)
|
||||
dirname = os.path.dirname(filename)
|
||||
if not os.path.isdir(dirname):
|
||||
os.makedirs(dirname)
|
||||
with open(filename, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
class NinjaReclientTest(trial_dir.TestCase):
|
||||
def setUp(self):
|
||||
super(NinjaReclientTest, self).setUp()
|
||||
self.previous_dir = os.getcwd()
|
||||
os.chdir(self.root_dir)
|
||||
|
||||
def tearDown(self):
|
||||
os.chdir(self.previous_dir)
|
||||
super(NinjaReclientTest, self).tearDown()
|
||||
|
||||
@unittest.mock.patch('subprocess.call', return_value=0)
|
||||
@unittest.mock.patch('ninja.main', return_value=0)
|
||||
def test_ninja_reclient(self, mock_ninja, mock_call):
|
||||
reclient_bin_dir = os.path.join('src', 'buildtools', 'reclient')
|
||||
reclient_cfg = os.path.join('src', 'buildtools', 'reclient_cfgs',
|
||||
'reproxy.cfg')
|
||||
write('.gclient', '')
|
||||
write('.gclient_entries', 'entries = {"buildtools": "..."}')
|
||||
write(os.path.join(reclient_bin_dir, 'version.txt'), '0.0')
|
||||
write(reclient_cfg, '0.0')
|
||||
argv = ["ninja_reclient.py", "-C", "out/a", "chrome"]
|
||||
|
||||
self.assertEqual(0, ninja_reclient.main(argv))
|
||||
|
||||
mock_ninja.assert_called_once_with(argv)
|
||||
mock_call.assert_has_calls([
|
||||
unittest.mock.call([
|
||||
os.path.join(self.root_dir, reclient_bin_dir,
|
||||
'bootstrap'), "--re_proxy=" +
|
||||
os.path.join(self.root_dir, reclient_bin_dir, 'reproxy'),
|
||||
"--cfg=" + os.path.join(self.root_dir, reclient_cfg)
|
||||
]),
|
||||
unittest.mock.call([
|
||||
os.path.join(self.root_dir, reclient_bin_dir, 'bootstrap'),
|
||||
"--shutdown", "--cfg=" + os.path.join(self.root_dir, reclient_cfg)
|
||||
]),
|
||||
])
|
||||
|
||||
@unittest.mock.patch('subprocess.call', return_value=0)
|
||||
@unittest.mock.patch('ninja.main', side_effect=KeyboardInterrupt())
|
||||
def test_ninja_reclient_ninja_interrupted(self, mock_ninja, mock_call):
|
||||
reclient_bin_dir = os.path.join('src', 'buildtools', 'reclient')
|
||||
reclient_cfg = os.path.join('src', 'buildtools', 'reclient_cfgs',
|
||||
'reproxy.cfg')
|
||||
write('.gclient', '')
|
||||
write('.gclient_entries', 'entries = {"buildtools": "..."}')
|
||||
write(os.path.join(reclient_bin_dir, 'version.txt'), '0.0')
|
||||
write(reclient_cfg, '0.0')
|
||||
argv = ["ninja_reclient.py", "-C", "out/a", "chrome"]
|
||||
|
||||
self.assertEqual(1, ninja_reclient.main(argv))
|
||||
|
||||
mock_ninja.assert_called_once_with(argv)
|
||||
mock_call.assert_has_calls([
|
||||
unittest.mock.call([
|
||||
os.path.join(self.root_dir, reclient_bin_dir,
|
||||
'bootstrap'), "--re_proxy=" +
|
||||
os.path.join(self.root_dir, reclient_bin_dir, 'reproxy'),
|
||||
"--cfg=" + os.path.join(self.root_dir, reclient_cfg)
|
||||
]),
|
||||
unittest.mock.call([
|
||||
os.path.join(self.root_dir, reclient_bin_dir, 'bootstrap'),
|
||||
"--shutdown", "--cfg=" + os.path.join(self.root_dir, reclient_cfg)
|
||||
]),
|
||||
])
|
||||
|
||||
@unittest.mock.patch('subprocess.call', return_value=0)
|
||||
@unittest.mock.patch('ninja.main', return_value=0)
|
||||
def test_ninja_reclient_cfg_not_found(self, mock_ninja, mock_call):
|
||||
write('.gclient', '')
|
||||
write('.gclient_entries', 'entries = {"buildtools": "..."}')
|
||||
write(os.path.join('src', 'buildtools', 'reclient', 'version.txt'), '0.0')
|
||||
argv = ["ninja_reclient.py", "-C", "out/a", "chrome"]
|
||||
|
||||
self.assertEqual(1, ninja_reclient.main(argv))
|
||||
|
||||
mock_ninja.assert_not_called()
|
||||
mock_call.assert_not_called()
|
||||
|
||||
@unittest.mock.patch('subprocess.call', return_value=0)
|
||||
@unittest.mock.patch('ninja.main', return_value=0)
|
||||
def test_ninja_reclient_bins_not_found(self, mock_ninja, mock_call):
|
||||
write('.gclient', '')
|
||||
write('.gclient_entries', 'entries = {"buildtools": "..."}')
|
||||
write(os.path.join('src', 'buildtools', 'reclient_cfgs', 'reproxy.cfg'),
|
||||
'0.0')
|
||||
argv = ["ninja_reclient.py", "-C", "out/a", "chrome"]
|
||||
|
||||
self.assertEqual(1, ninja_reclient.main(argv))
|
||||
|
||||
mock_ninja.assert_not_called()
|
||||
mock_call.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue