aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply black and `git cl format` for ninja related python files

Browse Source 
This is made by
$ black --line-length 79 *ninja*.py
$ git cl format

Change-Id: Ic446898a5461ae536542f6312cae2ce126dfe82a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5035265
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Junji Watanabe <jwata@google.com>
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Philipp Wollermann <philwo@chromium.org>
Reviewed-by: Fumitoshi Ukai <ukai@google.com>
changes/65/5035265/9
Takuto Ikuta 1 year ago committed by LUCI CQ
parent 0b98e7c063
commit df3e577855
6 changed files with 299 additions and 248 deletions
Show Stats Download Patch File Download Diff File

169
autoninja.py
Unescape Escape View File

@ -27,7 +27,7 @@ import ninja
import ninja_reclient
import siso
if sys.platform in ['darwin', 'linux']:
if sys.platform in ["darwin", "linux"]:
import resource
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@ -39,13 +39,13 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
# pylint: disable=line-too-long
# [1] https://learn.microsoft.com/en-us/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way # noqa
# [2] https://web.archive.org/web/20150815000000*/https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/set.mspx # noqa
_UNSAFE_FOR_CMD = set('^<>&|()%')
_UNSAFE_FOR_CMD = set("^<>&|()%")
_ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"'))
def _quote_for_cmd(arg):
# First, escape the arg so that CommandLineToArgvW will parse it properly.
if arg == '' or ' ' in arg or '"' in arg:
if arg == "" or " " in arg or '"' in arg:
quote_re = re.compile(r'(\\*)"')
arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))
@ -53,13 +53,13 @@ def _quote_for_cmd(arg):
# double quotes; if it does, quote everything (including the double
# quotes) for safety.
if any(a in _UNSAFE_FOR_CMD for a in arg):
arg = ''.join('^' + a if a in _ALL_META_CHARS else a for a in arg)
arg = "".join("^" + a if a in _ALL_META_CHARS else a for a in arg)
return arg
def _print_cmd(cmd):
shell_quoter = shlex.quote
if sys.platform.startswith('win'):
if sys.platform.startswith("win"):
shell_quoter = _quote_for_cmd
print(*[shell_quoter(arg) for arg in cmd], file=sys.stderr)
@ -70,14 +70,14 @@ def _gn_lines(output_dir, path):
import directives as needed.
"""
import_re = re.compile(r'\s*import\("(.*)"\)')
with open(path, encoding='utf-8') as f:
with open(path, encoding="utf-8") as f:
for line in f:
match = import_re.match(line)
if match:
raw_import_path = match.groups()[0]
if raw_import_path[:2] == "//":
import_path = os.path.normpath(
os.path.join(output_dir, '..', '..',
os.path.join(output_dir, "..", "..",
raw_import_path[2:]))
else:
import_path = os.path.normpath(
@ -93,9 +93,9 @@ def main(args):
t_specified = False
j_specified = False
offline = False
output_dir = '.'
output_dir = "."
input_args = args
summarize_build = os.environ.get('NINJA_SUMMARIZE_BUILD') == '1'
summarize_build = os.environ.get("NINJA_SUMMARIZE_BUILD") == "1"
# On Windows the autoninja.bat script passes along the arguments enclosed in
# double quotes. This prevents multiple levels of parsing of the special '^'
# characters needed when compiling a single file but means that this script
@ -103,29 +103,31 @@ def main(args):
# separated by spaces. When this case is detected we need to do argument
# splitting ourselves. This means that arguments containing actual spaces
# are not supported by autoninja, but that is not a real limitation.
if (sys.platform.startswith('win') and len(args) == 2
and input_args[1].count(' ') > 0):
if (sys.platform.startswith("win") and len(args) == 2
and input_args[1].count(" ") > 0):
input_args = args[:1] + args[1].split()
# Ninja uses getopt_long, which allow to intermix non-option arguments.
# To leave non supported parameters untouched, we do not use getopt.
for index, arg in enumerate(input_args[1:]):
if arg.startswith('-j'):
if arg.startswith("-j"):
j_specified = True
if arg.startswith('-t'):
if arg.startswith("-t"):
t_specified = True
if arg == '-C':
if arg == "-C":
# + 1 to get the next argument and +1 because we trimmed off
# input_args[0]
output_dir = input_args[index + 2]
elif arg.startswith('-C'):
elif arg.startswith("-C"):
# Support -Cout/Default
output_dir = arg[2:]
elif arg in ('-o', '--offline'):
elif arg in ("-o", "--offline"):
offline = True
elif arg == '-h':
print('autoninja: Use -o/--offline to temporary disable goma.',
file=sys.stderr)
elif arg == "-h":
print(
"autoninja: Use -o/--offline to temporary disable goma.",
file=sys.stderr,
)
print(file=sys.stderr)
use_goma = False
@ -135,8 +137,8 @@ def main(args):
# Attempt to auto-detect remote build acceleration. We support gn-based
# builds, where we look for args.gn in the build tree, and cmake-based
# builds where we look for rules.ninja.
if os.path.exists(os.path.join(output_dir, 'args.gn')):
for line in _gn_lines(output_dir, os.path.join(output_dir, 'args.gn')):
if os.path.exists(os.path.join(output_dir, "args.gn")):
for line in _gn_lines(output_dir, os.path.join(output_dir, "args.gn")):
# use_goma, or use_remoteexec will activate build
# acceleration.
#
@ -145,62 +147,70 @@ def main(args):
# use_goma=false# use_goma=true This comment is ignored
#
# Anything after a comment is not consider a valid argument.
line_without_comment = line.split('#')[0]
if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
line_without_comment = line.split("#")[0]
if re.search(r"(^|\s)(use_goma)\s*=\s*true($|\s)",
line_without_comment):
use_goma = True
continue
if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)',
line_without_comment):
if re.search(
r"(^|\s)(use_remoteexec)\s*=\s*true($|\s)",
line_without_comment,
):
use_remoteexec = True
continue
if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)',
if re.search(r"(^|\s)(use_siso)\s*=\s*true($|\s)",
line_without_comment):
use_siso = True
continue
siso_marker = os.path.join(output_dir, '.siso_deps')
siso_marker = os.path.join(output_dir, ".siso_deps")
if use_siso:
# autosiso generates a .ninja_log file so the mere existence of a
# .ninja_log file doesn't imply that a ninja build was done. However
# if there is a .ninja_log but no .siso_deps then that implies a
# ninja build.
ninja_marker = os.path.join(output_dir, '.ninja_log')
ninja_marker = os.path.join(output_dir, ".ninja_log")
if os.path.exists(ninja_marker) and not os.path.exists(siso_marker):
print('Run gn clean before switching from ninja to siso in %s' %
output_dir,
file=sys.stderr)
print(
"Run gn clean before switching from ninja to siso in %s" %
output_dir,
file=sys.stderr,
)
return 1
if use_goma:
print('Siso does not support Goma.', file=sys.stderr)
print('Do not use use_siso=true and use_goma=true',
file=sys.stderr)
print("Siso does not support Goma.", file=sys.stderr)
print(
"Do not use use_siso=true and use_goma=true",
file=sys.stderr,
)
return 1
if use_remoteexec:
return autosiso.main(['autosiso'] + input_args[1:])
return siso.main(['siso', 'ninja', '--offline'] + input_args[1:])
return autosiso.main(["autosiso"] + input_args[1:])
return siso.main(["siso", "ninja", "--offline"] + input_args[1:])
if os.path.exists(siso_marker):
print('Run gn clean before switching from siso to ninja in %s' %
output_dir,
file=sys.stderr)
print(
"Run gn clean before switching from siso to ninja in %s" %
output_dir,
file=sys.stderr,
)
return 1
else:
for relative_path in [
'', # GN keeps them in the root of output_dir
'CMakeFiles'
"", # GN keeps them in the root of output_dir
"CMakeFiles",
]:
path = os.path.join(output_dir, relative_path, 'rules.ninja')
path = os.path.join(output_dir, relative_path, "rules.ninja")
if os.path.exists(path):
with open(path, encoding='utf-8') as file_handle:
with open(path, encoding="utf-8") as file_handle:
for line in file_handle:
if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
if re.match(r"^\s*command\s*=\s*\S+gomacc", line):
use_goma = True
break
# Strip -o/--offline so ninja doesn't see them.
input_args = [arg for arg in input_args if arg not in ('-o', '--offline')]
input_args = [arg for arg in input_args if arg not in ("-o", "--offline")]
# If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1"
# (case-insensitive) then gomacc will use the local compiler instead of
@ -210,43 +220,46 @@ def main(args):
# non-goma build because an extra process is created for each compile step.
# Checking this environment variable ensures that autoninja uses an
# appropriate -j value in this situation.
goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
goma_disabled_env = os.environ.get("GOMA_DISABLED", "0").lower()
if offline or goma_disabled_env in ["true", "t", "yes", "y", "1"]:
use_goma = False
if use_goma:
gomacc_file = 'gomacc.exe' if sys.platform.startswith(
'win') else 'gomacc'
goma_dir = os.environ.get('GOMA_DIR',
os.path.join(SCRIPT_DIR, '.cipd_bin'))
gomacc_file = ("gomacc.exe"
if sys.platform.startswith("win") else "gomacc")
goma_dir = os.environ.get("GOMA_DIR",
os.path.join(SCRIPT_DIR, ".cipd_bin"))
gomacc_path = os.path.join(goma_dir, gomacc_file)
# Don't invoke gomacc if it doesn't exist.
if os.path.exists(gomacc_path):
# Check to make sure that goma is running. If not, don't start the
# build.
status = subprocess.call([gomacc_path, 'port'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
status = subprocess.call(
[gomacc_path, "port"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
)
if status == 1:
print(
'Goma is not running. Use "goma_ctl ensure_start" to start '
'it.',
file=sys.stderr)
if sys.platform.startswith('win'):
"it.",
file=sys.stderr,
)
if sys.platform.startswith("win"):
# Set an exit code of 1 in the batch file.
print('cmd "/c exit 1"')
else:
# Set an exit code of 1 by executing 'false' in the bash
# script.
print('false')
print("false")
sys.exit(1)
# A large build (with or without goma) tends to hog all system resources.
# Depending on the operating system, we might have mechanisms available
# to run at a lower priority, which improves this situation.
if os.environ.get('NINJA_BUILD_IN_BACKGROUND') == '1':
if sys.platform in ['darwin', 'linux']:
if os.environ.get("NINJA_BUILD_IN_BACKGROUND") == "1":
if sys.platform in ["darwin", "linux"]:
# nice-level 10 is usually considered a good default for background
# tasks. The niceness is inherited by child processes, so we can
# just set it here for us and it'll apply to the build tool we
@ -255,8 +268,8 @@ def main(args):
# Tell goma or reclient to do local compiles.
if offline:
os.environ['RBE_remote_disabled'] = '1'
os.environ['GOMA_DISABLED'] = '1'
os.environ["RBE_remote_disabled"] = "1"
os.environ["GOMA_DISABLED"] = "1"
# On macOS and most Linux distributions, the default limit of open file
# descriptors is too low (256 and 1024, respectively).
@ -264,7 +277,7 @@ def main(args):
# Check whether the limit can be raised to a large enough value. If yes,
# use `ulimit -n .... &&` as a prefix to increase the limit when running
# ninja.
if sys.platform in ['darwin', 'linux']:
if sys.platform in ["darwin", "linux"]:
# Increase the number of allowed open file descriptors to the maximum.
fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if fileno_limit < hard_limit:
@ -278,54 +291,54 @@ def main(args):
# Call ninja.py so that it can find ninja binary installed by DEPS or one in
# PATH.
ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py')
ninja_path = os.path.join(SCRIPT_DIR, "ninja.py")
# If using remoteexec, use ninja_reclient.py which wraps ninja.py with
# starting and stopping reproxy.
if use_remoteexec:
ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py')
ninja_path = os.path.join(SCRIPT_DIR, "ninja_reclient.py")
args = [sys.executable, ninja_path] + input_args[1:]
num_cores = multiprocessing.cpu_count()
if not j_specified and not t_specified:
if not offline and (use_goma or use_remoteexec):
args.append('-j')
args.append("-j")
default_core_multiplier = 80
if platform.machine() in ('x86_64', 'AMD64'):
if platform.machine() in ("x86_64", "AMD64"):
# Assume simultaneous multithreading and therefore half as many
# cores as logical processors.
num_cores //= 2
core_multiplier = int(
os.environ.get('NINJA_CORE_MULTIPLIER',
os.environ.get("NINJA_CORE_MULTIPLIER",
default_core_multiplier))
j_value = num_cores * core_multiplier
core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value))
core_limit = int(os.environ.get("NINJA_CORE_LIMIT", j_value))
j_value = min(j_value, core_limit)
# On Windows, a -j higher than 1000 doesn't improve build times.
# On macOS, ninja is limited to at most FD_SETSIZE (1024) open file
# descriptors.
if sys.platform in ['darwin', 'win32']:
if sys.platform in ["darwin", "win32"]:
j_value = min(j_value, 1000)
# Use a j value that reliably works with the open file descriptors
# limit.
if sys.platform in ['darwin', 'linux']:
if sys.platform in ["darwin", "linux"]:
j_value = min(j_value, int(fileno_limit * 0.8))
args.append('%d' % j_value)
args.append("%d" % j_value)
else:
j_value = num_cores
# Ninja defaults to |num_cores + 2|
j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
args.append('-j')
args.append('%d' % j_value)
j_value += int(os.environ.get("NINJA_CORE_ADDITION", "2"))
args.append("-j")
args.append("%d" % j_value)
if summarize_build:
# Enable statistics collection in Ninja.
args += ['-d', 'stats']
args += ["-d", "stats"]
# Print the command-line to reassure the user that the right settings
# are being used.
_print_cmd(args)
@ -335,7 +348,7 @@ def main(args):
return ninja.main(args[1:])
if __name__ == '__main__':
if __name__ == "__main__":
try:
sys.exit(main(sys.argv))
except KeyboardInterrupt:

39
ninja.py
Unescape Escape View File

@ -15,14 +15,14 @@ import gclient_paths
def findNinjaInPath():
env_path = os.getenv('PATH')
env_path = os.getenv("PATH")
if not env_path:
return
exe = 'ninja'
if sys.platform in ('win32', 'cygwin'):
exe += '.exe'
exe = "ninja"
if sys.platform in ("win32", "cygwin"):
exe += ".exe"
for bin_dir in env_path.split(os.pathsep):
if bin_dir.rstrip(os.sep).endswith('depot_tools'):
if bin_dir.rstrip(os.sep).endswith("depot_tools"):
# skip depot_tools to avoid calling ninja.py infinitely.
continue
ninja_path = os.path.join(bin_dir, exe)
@ -37,12 +37,13 @@ def fallback(ninja_args):
return subprocess.call([ninja_path] + ninja_args)
print(
'depot_tools/ninja.py: Could not find Ninja in the third_party of '
'the current project, nor in your PATH.\n'
'Please take one of the following actions to install Ninja:\n'
'- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n'
'- Otherwise, add Ninja to your PATH *after* depot_tools.',
file=sys.stderr)
"depot_tools/ninja.py: Could not find Ninja in the third_party of "
"the current project, nor in your PATH.\n"
"Please take one of the following actions to install Ninja:\n"
"- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n"
"- Otherwise, add Ninja to your PATH *after* depot_tools.",
file=sys.stderr,
)
return 1
@ -53,14 +54,14 @@ def main(args):
# detected, we need to split the argument. This means that arguments
# containing actual spaces are not supported by ninja.bat, but that is not a
# real limitation.
if (sys.platform.startswith('win') and len(args) == 2):
if sys.platform.startswith("win") and len(args) == 2:
args = args[:1] + args[1].split()
# macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly.
# https://openradar.appspot.com/radar?id=5608755232243712
#
# Removing those environment variables to avoid affecting clang's behaviors.
if sys.platform == 'darwin':
if sys.platform == "darwin":
os.environ.pop("CPATH", None)
os.environ.pop("LIBRARY_PATH", None)
os.environ.pop("SDKROOT", None)
@ -70,21 +71,25 @@ def main(args):
gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd())
gclient_src_root_path = None
if gclient_root_path:
gclient_src_root_path = os.path.join(gclient_root_path, 'src')
gclient_src_root_path = os.path.join(gclient_root_path, "src")
for base_path in set(
[primary_solution_path, gclient_root_path, gclient_src_root_path]):
if not base_path:
continue
ninja_path = os.path.join(base_path, 'third_party', 'ninja',
'ninja' + gclient_paths.GetExeSuffix())
ninja_path = os.path.join(
base_path,
"third_party",
"ninja",
"ninja" + gclient_paths.GetExeSuffix(),
)
if os.path.isfile(ninja_path):
return subprocess.call([ninja_path] + args[1:])
return fallback(args[1:])
if __name__ == '__main__':
if __name__ == "__main__":
try:
sys.exit(main(sys.argv))
except KeyboardInterrupt:

4
ninja_reclient.py
Unescape Escape View File

@ -14,7 +14,7 @@ import reclient_helper
def main(argv):
with reclient_helper.build_context(argv, 'ninja_reclient') as ret_code:
with reclient_helper.build_context(argv, "ninja_reclient") as ret_code:
if ret_code:
return ret_code
try:
@ -24,5 +24,5 @@ def main(argv):
return 1
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main(sys.argv))

131
ninjalog_uploader.py
Unescape Escape View File

@ -28,22 +28,35 @@ import time
import urllib.request
# These build configs affect build performance.
ALLOWLISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug',
'is_component_build', 'enable_nacl', 'host_os',
'host_cpu', 'target_os', 'target_cpu',
'blink_symbol_level', 'is_java_debug',
'treat_warnings_as_errors', 'disable_android_lint',
'use_errorprone_java_compiler', 'incremental_install',
'android_static_analysis')
ALLOWLISTED_CONFIGS = (
"symbol_level",
"use_goma",
"is_debug",
"is_component_build",
"enable_nacl",
"host_os",
"host_cpu",
"target_os",
"target_cpu",
"blink_symbol_level",
"is_java_debug",
"treat_warnings_as_errors",
"disable_android_lint",
"use_errorprone_java_compiler",
"incremental_install",
"android_static_analysis",
)
def IsGoogler():
"""Check whether this user is Googler or not."""
p = subprocess.run('goma_auth info',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
shell=True)
p = subprocess.run(
"goma_auth info",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
shell=True,
)
if p.returncode != 0:
return False
lines = p.stdout.splitlines()
@ -51,7 +64,7 @@ def IsGoogler():
return False
l = lines[0]
# |l| will be like 'Login as <user>@google.com' for googler using goma.
return l.startswith('Login as ') and l.endswith('@google.com')
return l.startswith("Login as ") and l.endswith("@google.com")
def ParseGNArgs(gn_args):
@ -63,10 +76,10 @@ def ParseGNArgs(gn_args):
key = config["name"]
if key not in ALLOWLISTED_CONFIGS:
continue
if 'current' in config:
build_configs[key] = config['current']['value']
if "current" in config:
build_configs[key] = config["current"]["value"]
else:
build_configs[key] = config['default']['value']
build_configs[key] = config["default"]["value"]
return build_configs
@ -79,8 +92,8 @@ def GetBuildTargetFromCommandLine(cmdline):
# Skipping all args that involve these flags, and taking all remaining args
# as targets.
onearg_flags = ('-C', '-d', '-f', '-j', '-k', '-l', '-p', '-t', '-w')
zeroarg_flags = ('--version', '-n', '-v')
onearg_flags = ("-C", "-d", "-f", "-j", "-k", "-l", "-p", "-t", "-w")
zeroarg_flags = ("--version", "-n", "-v")
targets = []
@ -90,12 +103,12 @@ def GetBuildTargetFromCommandLine(cmdline):
idx += 2
continue
if (arg[:2] in onearg_flags or arg in zeroarg_flags):
if arg[:2] in onearg_flags or arg in zeroarg_flags:
idx += 1
continue
# A target doesn't start with '-'.
if arg.startswith('-'):
if arg.startswith("-"):
idx += 1
continue
@ -114,12 +127,12 @@ def GetJflag(cmdline):
"""Parse cmdline to get flag value for -j"""
for i in range(len(cmdline)):
if (cmdline[i] == '-j' and i + 1 < len(cmdline)
if (cmdline[i] == "-j" and i + 1 < len(cmdline)
and cmdline[i + 1].isdigit()):
return int(cmdline[i + 1])
if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()):
return int(cmdline[i][len('-j'):])
if cmdline[i].startswith("-j") and cmdline[i][len("-j"):].isdigit():
return int(cmdline[i][len("-j"):])
def GetMetadata(cmdline, ninjalog):
@ -136,10 +149,10 @@ def GetMetadata(cmdline, ninjalog):
build_configs = {}
try:
args = ['gn', 'args', build_dir, '--list', '--short', '--json']
if sys.platform == 'win32':
args = ["gn", "args", build_dir, "--list", "--short", "--json"]
if sys.platform == "win32":
# gn in PATH is bat file in windows environment (except cygwin).
args = ['cmd', '/c'] + args
args = ["cmd", "/c"] + args
gn_args = subprocess.check_output(args)
build_configs = ParseGNArgs(gn_args)
@ -152,15 +165,15 @@ def GetMetadata(cmdline, ninjalog):
build_configs[k] = str(build_configs[k])
metadata = {
'platform': platform.system(),
'cpu_core': multiprocessing.cpu_count(),
'build_configs': build_configs,
'targets': GetBuildTargetFromCommandLine(cmdline),
"platform": platform.system(),
"cpu_core": multiprocessing.cpu_count(),
"build_configs": build_configs,
"targets": GetBuildTargetFromCommandLine(cmdline),
}
jflag = GetJflag(cmdline)
if jflag is not None:
metadata['jobs'] = jflag
metadata["jobs"] = jflag
return metadata
@ -168,36 +181,40 @@ def GetMetadata(cmdline, ninjalog):
def GetNinjalog(cmdline):
"""GetNinjalog returns the path to ninjalog from cmdline."""
# ninjalog is in current working directory by default.
ninjalog_dir = '.'
ninjalog_dir = "."
i = 0
while i < len(cmdline):
cmd = cmdline[i]
i += 1
if cmd == '-C' and i < len(cmdline):
if cmd == "-C" and i < len(cmdline):
ninjalog_dir = cmdline[i]
i += 1
continue
if cmd.startswith('-C') and len(cmd) > len('-C'):
ninjalog_dir = cmd[len('-C'):]
if cmd.startswith("-C") and len(cmd) > len("-C"):
ninjalog_dir = cmd[len("-C"):]
return os.path.join(ninjalog_dir, '.ninja_log')
return os.path.join(ninjalog_dir, ".ninja_log")
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--server',
default='chromium-build-stats.appspot.com',
help='server to upload ninjalog file.')
parser.add_argument('--ninjalog', help='ninjalog file to upload.')
parser.add_argument('--verbose',
action='store_true',
help='Enable verbose logging.')
parser.add_argument('--cmdline',
required=True,
nargs=argparse.REMAINDER,
help='command line args passed to ninja.')
parser.add_argument(
"--server",
default="chromium-build-stats.appspot.com",
help="server to upload ninjalog file.",
)
parser.add_argument("--ninjalog", help="ninjalog file to upload.")
parser.add_argument("--verbose",
action="store_true",
help="Enable verbose logging.")
parser.add_argument(
"--cmdline",
required=True,
nargs=argparse.REMAINDER,
help="command line args passed to ninja.",
)
args = parser.parse_args()
@ -224,27 +241,29 @@ def main():
output = io.BytesIO()
with open(ninjalog) as f:
with gzip.GzipFile(fileobj=output, mode='wb') as g:
with gzip.GzipFile(fileobj=output, mode="wb") as g:
g.write(f.read().encode())
g.write(b'# end of ninja log\n')
g.write(b"# end of ninja log\n")
metadata = GetMetadata(args.cmdline, ninjalog)
logging.info('send metadata: %s', json.dumps(metadata))
logging.info("send metadata: %s", json.dumps(metadata))
g.write(json.dumps(metadata).encode())
resp = urllib.request.urlopen(
urllib.request.Request('https://' + args.server + '/upload_ninja_log/',
data=output.getvalue(),
headers={'Content-Encoding': 'gzip'}))
urllib.request.Request(
"https://" + args.server + "/upload_ninja_log/",
data=output.getvalue(),
headers={"Content-Encoding": "gzip"},
))
if resp.status != 200:
logging.warning("unexpected status code for response: %s", resp.status)
return 1
logging.info('response header: %s', resp.headers)
logging.info('response content: %s', resp.read())
logging.info("response header: %s", resp.headers)
logging.info("response content: %s", resp.read())
return 0
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())

67
ninjalog_uploader_wrapper.py
Unescape Escape View File

@ -13,43 +13,43 @@ import ninjalog_uploader
import subprocess2
THIS_DIR = os.path.dirname(__file__)
UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py')
CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg')
UPLOADER = os.path.join(THIS_DIR, "ninjalog_uploader.py")
CONFIG = os.path.join(THIS_DIR, "ninjalog.cfg")
VERSION = 3
def LoadConfig():
if os.path.isfile(CONFIG):
with open(CONFIG, 'r') as f:
with open(CONFIG, "r") as f:
try:
config = json.load(f)
except Exception:
# Set default value when failed to load config.
config = {
'is-googler': ninjalog_uploader.IsGoogler(),
'countdown': 10,
'version': VERSION,
"is-googler": ninjalog_uploader.IsGoogler(),
"countdown": 10,
"version": VERSION,
}
if config['version'] == VERSION:
config['countdown'] = max(0, config['countdown'] - 1)
if config["version"] == VERSION:
config["countdown"] = max(0, config["countdown"] - 1)
return config
return {
'is-googler': ninjalog_uploader.IsGoogler(),
'countdown': 10,
'version': VERSION,
"is-googler": ninjalog_uploader.IsGoogler(),
"countdown": 10,
"version": VERSION,
}
def SaveConfig(config):
with open(CONFIG, 'w') as f:
with open(CONFIG, "w") as f:
json.dump(config, f)
def ShowMessage(countdown):
allowlisted = '\n'.join(
[' * %s' % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS])
allowlisted = "\n".join(
[" * %s" % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS])
print("""
Your ninjalog will be uploaded to build stats server. The uploaded log will be
used to analyze user side build performance.
@ -78,27 +78,32 @@ You can find a more detailed explanation in
or
https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md
""" % (allowlisted, countdown, __file__, __file__,
os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md"))))
""" % (
allowlisted,
countdown,
__file__,
__file__,
os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md")),
))
def main():
config = LoadConfig()
if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
config['opt-in'] = True
config['countdown'] = 0
if len(sys.argv) == 2 and sys.argv[1] == "opt-in":
config["opt-in"] = True
config["countdown"] = 0
SaveConfig(config)
print('ninjalog upload is opted in.')
print("ninjalog upload is opted in.")
return 0
if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
config['opt-in'] = False
if len(sys.argv) == 2 and sys.argv[1] == "opt-out":
config["opt-in"] = False
SaveConfig(config)
print('ninjalog upload is opted out.')
print("ninjalog upload is opted out.")
return 0
if 'opt-in' in config and not config['opt-in']:
if "opt-in" in config and not config["opt-in"]:
# Upload is opted out.
return 0
@ -121,13 +126,15 @@ def main():
# Run upload script without wait.
devnull = open(os.devnull, "w")
creationflags = 0
if platform.system() == 'Windows':
if platform.system() == "Windows":
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:],
stdout=devnull,
stderr=devnull,
creationflags=creationflags)
subprocess2.Popen(
[sys.executable, UPLOADER] + sys.argv[1:],
stdout=devnull,
stderr=devnull,
creationflags=creationflags,
)
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())

137
post_build_ninja_summary.py
Unescape Escape View File

@ -94,9 +94,9 @@ class Target:
"""
# Allow for modest floating-point errors
epsilon = 0.000002
if (self.weighted_duration > self.Duration() + epsilon):
print('%s > %s?' % (self.weighted_duration, self.Duration()))
assert (self.weighted_duration <= self.Duration() + epsilon)
if self.weighted_duration > self.Duration() + epsilon:
print("%s > %s?" % (self.weighted_duration, self.Duration()))
assert self.weighted_duration <= self.Duration() + epsilon
return self.weighted_duration
def DescribeTargets(self):
@ -104,10 +104,10 @@ class Target:
# Some build steps generate dozens of outputs - handle them sanely.
# The max_length was chosen so that it can fit most of the long
# single-target names, while minimizing word wrapping.
result = ', '.join(self.targets)
result = ", ".join(self.targets)
max_length = 65
if len(result) > max_length:
result = result[:max_length] + '...'
result = result[:max_length] + "..."
return result
@ -121,12 +121,12 @@ def ReadTargets(log, show_all):
# targets.
if not header:
return []
assert header == '# ninja log v5\n', ('unrecognized ninja log version %r' %
assert header == "# ninja log v5\n", ("unrecognized ninja log version %r" %
header)
targets_dict = {}
last_end_seen = 0.0
for line in log:
parts = line.strip().split('\t')
parts = line.strip().split("\t")
if len(parts) != 5:
# If ninja.exe is rudely halted then the .ninja_log file may be
# corrupt. Silently continue.
@ -170,12 +170,12 @@ def GetExtension(target, extra_patterns):
steps by type."""
for output in target.targets:
if extra_patterns:
for fn_pattern in extra_patterns.split(';'):
if fnmatch.fnmatch(output, '*' + fn_pattern + '*'):
for fn_pattern in extra_patterns.split(";"):
if fnmatch.fnmatch(output, "*" + fn_pattern + "*"):
return fn_pattern
# Not a true extension, but a good grouping.
if output.endswith('type_mappings'):
extension = 'type_mappings'
if output.endswith("type_mappings"):
extension = "type_mappings"
break
# Capture two extensions if present. For example: file.javac.jar should
@ -185,26 +185,26 @@ def GetExtension(target, extra_patterns):
extension = ext2 + ext1 # Preserve the order in the file name.
if len(extension) == 0:
extension = '(no extension found)'
extension = "(no extension found)"
if ext1 in ['.pdb', '.dll', '.exe']:
extension = 'PEFile (linking)'
if ext1 in [".pdb", ".dll", ".exe"]:
extension = "PEFile (linking)"
# Make sure that .dll and .exe are grouped together and that the
# .dll.lib files don't cause these to be listed as libraries
break
if ext1 in ['.so', '.TOC']:
extension = '.so (linking)'
if ext1 in [".so", ".TOC"]:
extension = ".so (linking)"
# Attempt to identify linking, avoid identifying as '.TOC'
break
# Make sure .obj files don't get categorized as mojo files
if ext1 in ['.obj', '.o']:
if ext1 in [".obj", ".o"]:
break
# Jars are the canonical output of java targets.
if ext1 == '.jar':
if ext1 == ".jar":
break
# Normalize all mojo related outputs to 'mojo'.
if output.count('.mojom') > 0:
extension = 'mojo'
if output.count(".mojom") > 0:
extension = "mojo"
break
return extension
@ -229,8 +229,8 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
if target.end > latest:
latest = target.end
total_cpu_time += target.Duration()
task_start_stop_times.append((target.start, 'start', target))
task_start_stop_times.append((target.end, 'stop', target))
task_start_stop_times.append((target.start, "start", target))
task_start_stop_times.append((target.end, "stop", target))
length = latest - earliest
weighted_total = 0.0
@ -256,10 +256,10 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
if num_running > 0:
# Update the total weighted time up to this moment.
last_weighted_time += (time - last_time) / float(num_running)
if action_name == 'start':
if action_name == "start":
# Record the total weighted task time when this task starts.
running_tasks[target] = last_weighted_time
if action_name == 'stop':
if action_name == "stop":
# Record the change in the total weighted task time while this task
# ran.
weighted_duration = last_weighted_time - running_tasks[target]
@ -267,24 +267,26 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
weighted_total += weighted_duration
del running_tasks[target]
last_time = time
assert (len(running_tasks) == 0)
assert len(running_tasks) == 0
# Warn if the sum of weighted times is off by more than half a second.
if abs(length - weighted_total) > 500:
print('Warning: Possible corrupt ninja log, results may be '
'untrustworthy. Length = %.3f, weighted total = %.3f' %
print("Warning: Possible corrupt ninja log, results may be "
"untrustworthy. Length = %.3f, weighted total = %.3f" %
(length, weighted_total))
# Print the slowest build steps:
print(' Longest build steps:')
print(" Longest build steps:")
if elapsed_time_sorting:
entries.sort(key=lambda x: x.Duration())
else:
entries.sort(key=lambda x: x.WeightedDuration())
for target in entries[-long_count:]:
print(' %8.1f weighted s to build %s (%.1f s elapsed time)' %
(target.WeightedDuration(), target.DescribeTargets(),
target.Duration()))
print(" %8.1f weighted s to build %s (%.1f s elapsed time)" % (
target.WeightedDuration(),
target.DescribeTargets(),
target.Duration(),
))
# Sum up the time by file extension/type of the output file
count_by_ext = {}
@ -293,13 +295,13 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
# Scan through all of the targets to build up per-extension statistics.
for target in entries:
extension = GetExtension(target, extra_step_types)
time_by_ext[extension] = time_by_ext.get(extension,
0) + target.Duration()
weighted_time_by_ext[extension] = weighted_time_by_ext.get(
extension, 0) + target.WeightedDuration()
time_by_ext[extension] = (time_by_ext.get(extension, 0) +
target.Duration())
weighted_time_by_ext[extension] = (
weighted_time_by_ext.get(extension, 0) + target.WeightedDuration())
count_by_ext[extension] = count_by_ext.get(extension, 0) + 1
print(' Time by build-step type:')
print(" Time by build-step type:")
# Copy to a list with extension name and total time swapped, to (time, ext)
if elapsed_time_sorting:
weighted_time_by_ext_sorted = sorted(
@ -309,34 +311,39 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
(y, x) for (x, y) in weighted_time_by_ext.items())
# Print the slowest build target types:
for time, extension in weighted_time_by_ext_sorted[-long_ext_count:]:
print(
' %8.1f s weighted time to generate %d %s files '
'(%1.1f s elapsed time sum)' %
(time, count_by_ext[extension], extension, time_by_ext[extension]))
print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx '
'parallelism)' %
print(" %8.1f s weighted time to generate %d %s files "
"(%1.1f s elapsed time sum)" % (
time,
count_by_ext[extension],
extension,
time_by_ext[extension],
))
print(" %.1f s weighted time (%.1f s elapsed time sum, %1.1fx "
"parallelism)" %
(length, total_cpu_time, total_cpu_time * 1.0 / length))
print(' %d build steps completed, average of %1.2f/s' %
print(" %d build steps completed, average of %1.2f/s" %
(len(entries), len(entries) / (length)))
def main():
log_file = '.ninja_log'
metrics_file = 'siso_metrics.json'
log_file = ".ninja_log"
metrics_file = "siso_metrics.json"
parser = argparse.ArgumentParser()
parser.add_argument('-C', dest='build_directory', help='Build directory.')
parser.add_argument("-C", dest="build_directory", help="Build directory.")
parser.add_argument(
'-s',
'--step-types',
help='semicolon separated fnmatch patterns for build-step grouping')
"-s",
"--step-types",
help="semicolon separated fnmatch patterns for build-step grouping",
)
parser.add_argument(
'-e',
'--elapsed_time_sorting',
"-e",
"--elapsed_time_sorting",
default=False,
action='store_true',
help='Sort output by elapsed time instead of weighted time')
parser.add_argument('--log-file',
action="store_true",
help="Sort output by elapsed time instead of weighted time",
)
parser.add_argument("--log-file",
help="specific ninja log file to analyze.")
args, _extra_args = parser.parse_known_args()
if args.build_directory:
@ -348,34 +355,34 @@ def main():
# Offer a convenient way to add extra step types automatically,
# including when this script is run by autoninja. get() returns None if
# the variable isn't set.
args.step_types = os.environ.get('chromium_step_types')
args.step_types = os.environ.get("chromium_step_types")
if args.step_types:
# Make room for the extra build types.
global long_ext_count
long_ext_count += len(args.step_types.split(';'))
long_ext_count += len(args.step_types.split(";"))
if os.path.exists(metrics_file):
# Automatically handle summarizing siso builds.
cmd = ['siso.bat' if 'win32' in sys.platform else 'siso']
cmd.extend(['metrics', 'summary'])
cmd = ["siso.bat" if "win32" in sys.platform else "siso"]
cmd.extend(["metrics", "summary"])
if args.build_directory:
cmd.extend(['-C', args.build_directory])
cmd.extend(["-C", args.build_directory])
if args.step_types:
cmd.extend(['--step_types', args.step_types])
cmd.extend(["--step_types", args.step_types])
if args.elapsed_time_sorting:
cmd.append('--elapsed_time_sorting')
cmd.append("--elapsed_time_sorting")
subprocess.run(cmd)
else:
try:
with open(log_file, 'r') as log:
with open(log_file, "r") as log:
entries = ReadTargets(log, False)
if entries:
SummarizeEntries(entries, args.step_types,
args.elapsed_time_sorting)
except IOError:
print('Log file %r not found, no build summary created.' % log_file)
print("Log file %r not found, no build summary created." % log_file)
return errno.ENOENT
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())

Write Preview
Loading…
Cancel
Save