diff --git a/autoninja.py b/autoninja.py index af8417c55..417fa977e 100755 --- a/autoninja.py +++ b/autoninja.py @@ -27,7 +27,7 @@ import ninja import ninja_reclient import siso -if sys.platform in ['darwin', 'linux']: +if sys.platform in ["darwin", "linux"]: import resource SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -39,13 +39,13 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) # pylint: disable=line-too-long # [1] https://learn.microsoft.com/en-us/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way # noqa # [2] https://web.archive.org/web/20150815000000*/https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/set.mspx # noqa -_UNSAFE_FOR_CMD = set('^<>&|()%') +_UNSAFE_FOR_CMD = set("^<>&|()%") _ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"')) def _quote_for_cmd(arg): # First, escape the arg so that CommandLineToArgvW will parse it properly. - if arg == '' or ' ' in arg or '"' in arg: + if arg == "" or " " in arg or '"' in arg: quote_re = re.compile(r'(\\*)"') arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) @@ -53,13 +53,13 @@ def _quote_for_cmd(arg): # double quotes; if it does, quote everything (including the double # quotes) for safety. if any(a in _UNSAFE_FOR_CMD for a in arg): - arg = ''.join('^' + a if a in _ALL_META_CHARS else a for a in arg) + arg = "".join("^" + a if a in _ALL_META_CHARS else a for a in arg) return arg def _print_cmd(cmd): shell_quoter = shlex.quote - if sys.platform.startswith('win'): + if sys.platform.startswith("win"): shell_quoter = _quote_for_cmd print(*[shell_quoter(arg) for arg in cmd], file=sys.stderr) @@ -70,14 +70,14 @@ def _gn_lines(output_dir, path): import directives as needed. """ import_re = re.compile(r'\s*import\("(.*)"\)') - with open(path, encoding='utf-8') as f: + with open(path, encoding="utf-8") as f: for line in f: match = import_re.match(line) if match: raw_import_path = match.groups()[0] if raw_import_path[:2] == "//": import_path = os.path.normpath( - os.path.join(output_dir, '..', '..', + os.path.join(output_dir, "..", "..", raw_import_path[2:])) else: import_path = os.path.normpath( @@ -93,9 +93,9 @@ def main(args): t_specified = False j_specified = False offline = False - output_dir = '.' + output_dir = "." input_args = args - summarize_build = os.environ.get('NINJA_SUMMARIZE_BUILD') == '1' + summarize_build = os.environ.get("NINJA_SUMMARIZE_BUILD") == "1" # On Windows the autoninja.bat script passes along the arguments enclosed in # double quotes. This prevents multiple levels of parsing of the special '^' # characters needed when compiling a single file but means that this script @@ -103,29 +103,31 @@ def main(args): # separated by spaces. When this case is detected we need to do argument # splitting ourselves. This means that arguments containing actual spaces # are not supported by autoninja, but that is not a real limitation. - if (sys.platform.startswith('win') and len(args) == 2 - and input_args[1].count(' ') > 0): + if (sys.platform.startswith("win") and len(args) == 2 + and input_args[1].count(" ") > 0): input_args = args[:1] + args[1].split() # Ninja uses getopt_long, which allow to intermix non-option arguments. # To leave non supported parameters untouched, we do not use getopt. for index, arg in enumerate(input_args[1:]): - if arg.startswith('-j'): + if arg.startswith("-j"): j_specified = True - if arg.startswith('-t'): + if arg.startswith("-t"): t_specified = True - if arg == '-C': + if arg == "-C": # + 1 to get the next argument and +1 because we trimmed off # input_args[0] output_dir = input_args[index + 2] - elif arg.startswith('-C'): + elif arg.startswith("-C"): # Support -Cout/Default output_dir = arg[2:] - elif arg in ('-o', '--offline'): + elif arg in ("-o", "--offline"): offline = True - elif arg == '-h': - print('autoninja: Use -o/--offline to temporary disable goma.', - file=sys.stderr) + elif arg == "-h": + print( + "autoninja: Use -o/--offline to temporary disable goma.", + file=sys.stderr, + ) print(file=sys.stderr) use_goma = False @@ -135,8 +137,8 @@ def main(args): # Attempt to auto-detect remote build acceleration. We support gn-based # builds, where we look for args.gn in the build tree, and cmake-based # builds where we look for rules.ninja. - if os.path.exists(os.path.join(output_dir, 'args.gn')): - for line in _gn_lines(output_dir, os.path.join(output_dir, 'args.gn')): + if os.path.exists(os.path.join(output_dir, "args.gn")): + for line in _gn_lines(output_dir, os.path.join(output_dir, "args.gn")): # use_goma, or use_remoteexec will activate build # acceleration. # @@ -145,62 +147,70 @@ def main(args): # use_goma=false# use_goma=true This comment is ignored # # Anything after a comment is not consider a valid argument. - line_without_comment = line.split('#')[0] - if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', + line_without_comment = line.split("#")[0] + if re.search(r"(^|\s)(use_goma)\s*=\s*true($|\s)", line_without_comment): use_goma = True continue - if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)', - line_without_comment): + if re.search( + r"(^|\s)(use_remoteexec)\s*=\s*true($|\s)", + line_without_comment, + ): use_remoteexec = True continue - if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)', + if re.search(r"(^|\s)(use_siso)\s*=\s*true($|\s)", line_without_comment): use_siso = True continue - siso_marker = os.path.join(output_dir, '.siso_deps') + siso_marker = os.path.join(output_dir, ".siso_deps") if use_siso: # autosiso generates a .ninja_log file so the mere existence of a # .ninja_log file doesn't imply that a ninja build was done. However # if there is a .ninja_log but no .siso_deps then that implies a # ninja build. - ninja_marker = os.path.join(output_dir, '.ninja_log') + ninja_marker = os.path.join(output_dir, ".ninja_log") if os.path.exists(ninja_marker) and not os.path.exists(siso_marker): - print('Run gn clean before switching from ninja to siso in %s' % - output_dir, - file=sys.stderr) + print( + "Run gn clean before switching from ninja to siso in %s" % + output_dir, + file=sys.stderr, + ) return 1 if use_goma: - print('Siso does not support Goma.', file=sys.stderr) - print('Do not use use_siso=true and use_goma=true', - file=sys.stderr) + print("Siso does not support Goma.", file=sys.stderr) + print( + "Do not use use_siso=true and use_goma=true", + file=sys.stderr, + ) return 1 if use_remoteexec: - return autosiso.main(['autosiso'] + input_args[1:]) - return siso.main(['siso', 'ninja', '--offline'] + input_args[1:]) + return autosiso.main(["autosiso"] + input_args[1:]) + return siso.main(["siso", "ninja", "--offline"] + input_args[1:]) if os.path.exists(siso_marker): - print('Run gn clean before switching from siso to ninja in %s' % - output_dir, - file=sys.stderr) + print( + "Run gn clean before switching from siso to ninja in %s" % + output_dir, + file=sys.stderr, + ) return 1 else: for relative_path in [ - '', # GN keeps them in the root of output_dir - 'CMakeFiles' + "", # GN keeps them in the root of output_dir + "CMakeFiles", ]: - path = os.path.join(output_dir, relative_path, 'rules.ninja') + path = os.path.join(output_dir, relative_path, "rules.ninja") if os.path.exists(path): - with open(path, encoding='utf-8') as file_handle: + with open(path, encoding="utf-8") as file_handle: for line in file_handle: - if re.match(r'^\s*command\s*=\s*\S+gomacc', line): + if re.match(r"^\s*command\s*=\s*\S+gomacc", line): use_goma = True break # Strip -o/--offline so ninja doesn't see them. - input_args = [arg for arg in input_args if arg not in ('-o', '--offline')] + input_args = [arg for arg in input_args if arg not in ("-o", "--offline")] # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" # (case-insensitive) then gomacc will use the local compiler instead of @@ -210,43 +220,46 @@ def main(args): # non-goma build because an extra process is created for each compile step. # Checking this environment variable ensures that autoninja uses an # appropriate -j value in this situation. - goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() - if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: + goma_disabled_env = os.environ.get("GOMA_DISABLED", "0").lower() + if offline or goma_disabled_env in ["true", "t", "yes", "y", "1"]: use_goma = False if use_goma: - gomacc_file = 'gomacc.exe' if sys.platform.startswith( - 'win') else 'gomacc' - goma_dir = os.environ.get('GOMA_DIR', - os.path.join(SCRIPT_DIR, '.cipd_bin')) + gomacc_file = ("gomacc.exe" + if sys.platform.startswith("win") else "gomacc") + goma_dir = os.environ.get("GOMA_DIR", + os.path.join(SCRIPT_DIR, ".cipd_bin")) gomacc_path = os.path.join(goma_dir, gomacc_file) # Don't invoke gomacc if it doesn't exist. if os.path.exists(gomacc_path): # Check to make sure that goma is running. If not, don't start the # build. - status = subprocess.call([gomacc_path, 'port'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False) + status = subprocess.call( + [gomacc_path, "port"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) if status == 1: print( 'Goma is not running. Use "goma_ctl ensure_start" to start ' - 'it.', - file=sys.stderr) - if sys.platform.startswith('win'): + "it.", + file=sys.stderr, + ) + if sys.platform.startswith("win"): # Set an exit code of 1 in the batch file. print('cmd "/c exit 1"') else: # Set an exit code of 1 by executing 'false' in the bash # script. - print('false') + print("false") sys.exit(1) # A large build (with or without goma) tends to hog all system resources. # Depending on the operating system, we might have mechanisms available # to run at a lower priority, which improves this situation. - if os.environ.get('NINJA_BUILD_IN_BACKGROUND') == '1': - if sys.platform in ['darwin', 'linux']: + if os.environ.get("NINJA_BUILD_IN_BACKGROUND") == "1": + if sys.platform in ["darwin", "linux"]: # nice-level 10 is usually considered a good default for background # tasks. The niceness is inherited by child processes, so we can # just set it here for us and it'll apply to the build tool we @@ -255,8 +268,8 @@ def main(args): # Tell goma or reclient to do local compiles. if offline: - os.environ['RBE_remote_disabled'] = '1' - os.environ['GOMA_DISABLED'] = '1' + os.environ["RBE_remote_disabled"] = "1" + os.environ["GOMA_DISABLED"] = "1" # On macOS and most Linux distributions, the default limit of open file # descriptors is too low (256 and 1024, respectively). @@ -264,7 +277,7 @@ def main(args): # Check whether the limit can be raised to a large enough value. If yes, # use `ulimit -n .... &&` as a prefix to increase the limit when running # ninja. - if sys.platform in ['darwin', 'linux']: + if sys.platform in ["darwin", "linux"]: # Increase the number of allowed open file descriptors to the maximum. fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) if fileno_limit < hard_limit: @@ -278,54 +291,54 @@ def main(args): # Call ninja.py so that it can find ninja binary installed by DEPS or one in # PATH. - ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py') + ninja_path = os.path.join(SCRIPT_DIR, "ninja.py") # If using remoteexec, use ninja_reclient.py which wraps ninja.py with # starting and stopping reproxy. if use_remoteexec: - ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py') + ninja_path = os.path.join(SCRIPT_DIR, "ninja_reclient.py") args = [sys.executable, ninja_path] + input_args[1:] num_cores = multiprocessing.cpu_count() if not j_specified and not t_specified: if not offline and (use_goma or use_remoteexec): - args.append('-j') + args.append("-j") default_core_multiplier = 80 - if platform.machine() in ('x86_64', 'AMD64'): + if platform.machine() in ("x86_64", "AMD64"): # Assume simultaneous multithreading and therefore half as many # cores as logical processors. num_cores //= 2 core_multiplier = int( - os.environ.get('NINJA_CORE_MULTIPLIER', + os.environ.get("NINJA_CORE_MULTIPLIER", default_core_multiplier)) j_value = num_cores * core_multiplier - core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value)) + core_limit = int(os.environ.get("NINJA_CORE_LIMIT", j_value)) j_value = min(j_value, core_limit) # On Windows, a -j higher than 1000 doesn't improve build times. # On macOS, ninja is limited to at most FD_SETSIZE (1024) open file # descriptors. - if sys.platform in ['darwin', 'win32']: + if sys.platform in ["darwin", "win32"]: j_value = min(j_value, 1000) # Use a j value that reliably works with the open file descriptors # limit. - if sys.platform in ['darwin', 'linux']: + if sys.platform in ["darwin", "linux"]: j_value = min(j_value, int(fileno_limit * 0.8)) - args.append('%d' % j_value) + args.append("%d" % j_value) else: j_value = num_cores # Ninja defaults to |num_cores + 2| - j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) - args.append('-j') - args.append('%d' % j_value) + j_value += int(os.environ.get("NINJA_CORE_ADDITION", "2")) + args.append("-j") + args.append("%d" % j_value) if summarize_build: # Enable statistics collection in Ninja. - args += ['-d', 'stats'] + args += ["-d", "stats"] # Print the command-line to reassure the user that the right settings # are being used. _print_cmd(args) @@ -335,7 +348,7 @@ def main(args): return ninja.main(args[1:]) -if __name__ == '__main__': +if __name__ == "__main__": try: sys.exit(main(sys.argv)) except KeyboardInterrupt: diff --git a/ninja.py b/ninja.py index f6dffe164..7dd0dbd69 100755 --- a/ninja.py +++ b/ninja.py @@ -15,14 +15,14 @@ import gclient_paths def findNinjaInPath(): - env_path = os.getenv('PATH') + env_path = os.getenv("PATH") if not env_path: return - exe = 'ninja' - if sys.platform in ('win32', 'cygwin'): - exe += '.exe' + exe = "ninja" + if sys.platform in ("win32", "cygwin"): + exe += ".exe" for bin_dir in env_path.split(os.pathsep): - if bin_dir.rstrip(os.sep).endswith('depot_tools'): + if bin_dir.rstrip(os.sep).endswith("depot_tools"): # skip depot_tools to avoid calling ninja.py infinitely. continue ninja_path = os.path.join(bin_dir, exe) @@ -37,12 +37,13 @@ def fallback(ninja_args): return subprocess.call([ninja_path] + ninja_args) print( - 'depot_tools/ninja.py: Could not find Ninja in the third_party of ' - 'the current project, nor in your PATH.\n' - 'Please take one of the following actions to install Ninja:\n' - '- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n' - '- Otherwise, add Ninja to your PATH *after* depot_tools.', - file=sys.stderr) + "depot_tools/ninja.py: Could not find Ninja in the third_party of " + "the current project, nor in your PATH.\n" + "Please take one of the following actions to install Ninja:\n" + "- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n" + "- Otherwise, add Ninja to your PATH *after* depot_tools.", + file=sys.stderr, + ) return 1 @@ -53,14 +54,14 @@ def main(args): # detected, we need to split the argument. This means that arguments # containing actual spaces are not supported by ninja.bat, but that is not a # real limitation. - if (sys.platform.startswith('win') and len(args) == 2): + if sys.platform.startswith("win") and len(args) == 2: args = args[:1] + args[1].split() # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. # https://openradar.appspot.com/radar?id=5608755232243712 # # Removing those environment variables to avoid affecting clang's behaviors. - if sys.platform == 'darwin': + if sys.platform == "darwin": os.environ.pop("CPATH", None) os.environ.pop("LIBRARY_PATH", None) os.environ.pop("SDKROOT", None) @@ -70,21 +71,25 @@ def main(args): gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd()) gclient_src_root_path = None if gclient_root_path: - gclient_src_root_path = os.path.join(gclient_root_path, 'src') + gclient_src_root_path = os.path.join(gclient_root_path, "src") for base_path in set( [primary_solution_path, gclient_root_path, gclient_src_root_path]): if not base_path: continue - ninja_path = os.path.join(base_path, 'third_party', 'ninja', - 'ninja' + gclient_paths.GetExeSuffix()) + ninja_path = os.path.join( + base_path, + "third_party", + "ninja", + "ninja" + gclient_paths.GetExeSuffix(), + ) if os.path.isfile(ninja_path): return subprocess.call([ninja_path] + args[1:]) return fallback(args[1:]) -if __name__ == '__main__': +if __name__ == "__main__": try: sys.exit(main(sys.argv)) except KeyboardInterrupt: diff --git a/ninja_reclient.py b/ninja_reclient.py index 446898ccd..0b03fb848 100755 --- a/ninja_reclient.py +++ b/ninja_reclient.py @@ -14,7 +14,7 @@ import reclient_helper def main(argv): - with reclient_helper.build_context(argv, 'ninja_reclient') as ret_code: + with reclient_helper.build_context(argv, "ninja_reclient") as ret_code: if ret_code: return ret_code try: @@ -24,5 +24,5 @@ def main(argv): return 1 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py index 7649846ea..80f72698b 100755 --- a/ninjalog_uploader.py +++ b/ninjalog_uploader.py @@ -28,22 +28,35 @@ import time import urllib.request # These build configs affect build performance. -ALLOWLISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug', - 'is_component_build', 'enable_nacl', 'host_os', - 'host_cpu', 'target_os', 'target_cpu', - 'blink_symbol_level', 'is_java_debug', - 'treat_warnings_as_errors', 'disable_android_lint', - 'use_errorprone_java_compiler', 'incremental_install', - 'android_static_analysis') +ALLOWLISTED_CONFIGS = ( + "symbol_level", + "use_goma", + "is_debug", + "is_component_build", + "enable_nacl", + "host_os", + "host_cpu", + "target_os", + "target_cpu", + "blink_symbol_level", + "is_java_debug", + "treat_warnings_as_errors", + "disable_android_lint", + "use_errorprone_java_compiler", + "incremental_install", + "android_static_analysis", +) def IsGoogler(): """Check whether this user is Googler or not.""" - p = subprocess.run('goma_auth info', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True) + p = subprocess.run( + "goma_auth info", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True, + ) if p.returncode != 0: return False lines = p.stdout.splitlines() @@ -51,7 +64,7 @@ def IsGoogler(): return False l = lines[0] # |l| will be like 'Login as @google.com' for googler using goma. - return l.startswith('Login as ') and l.endswith('@google.com') + return l.startswith("Login as ") and l.endswith("@google.com") def ParseGNArgs(gn_args): @@ -63,10 +76,10 @@ def ParseGNArgs(gn_args): key = config["name"] if key not in ALLOWLISTED_CONFIGS: continue - if 'current' in config: - build_configs[key] = config['current']['value'] + if "current" in config: + build_configs[key] = config["current"]["value"] else: - build_configs[key] = config['default']['value'] + build_configs[key] = config["default"]["value"] return build_configs @@ -79,8 +92,8 @@ def GetBuildTargetFromCommandLine(cmdline): # Skipping all args that involve these flags, and taking all remaining args # as targets. - onearg_flags = ('-C', '-d', '-f', '-j', '-k', '-l', '-p', '-t', '-w') - zeroarg_flags = ('--version', '-n', '-v') + onearg_flags = ("-C", "-d", "-f", "-j", "-k", "-l", "-p", "-t", "-w") + zeroarg_flags = ("--version", "-n", "-v") targets = [] @@ -90,12 +103,12 @@ def GetBuildTargetFromCommandLine(cmdline): idx += 2 continue - if (arg[:2] in onearg_flags or arg in zeroarg_flags): + if arg[:2] in onearg_flags or arg in zeroarg_flags: idx += 1 continue # A target doesn't start with '-'. - if arg.startswith('-'): + if arg.startswith("-"): idx += 1 continue @@ -114,12 +127,12 @@ def GetJflag(cmdline): """Parse cmdline to get flag value for -j""" for i in range(len(cmdline)): - if (cmdline[i] == '-j' and i + 1 < len(cmdline) + if (cmdline[i] == "-j" and i + 1 < len(cmdline) and cmdline[i + 1].isdigit()): return int(cmdline[i + 1]) - if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()): - return int(cmdline[i][len('-j'):]) + if cmdline[i].startswith("-j") and cmdline[i][len("-j"):].isdigit(): + return int(cmdline[i][len("-j"):]) def GetMetadata(cmdline, ninjalog): @@ -136,10 +149,10 @@ def GetMetadata(cmdline, ninjalog): build_configs = {} try: - args = ['gn', 'args', build_dir, '--list', '--short', '--json'] - if sys.platform == 'win32': + args = ["gn", "args", build_dir, "--list", "--short", "--json"] + if sys.platform == "win32": # gn in PATH is bat file in windows environment (except cygwin). - args = ['cmd', '/c'] + args + args = ["cmd", "/c"] + args gn_args = subprocess.check_output(args) build_configs = ParseGNArgs(gn_args) @@ -152,15 +165,15 @@ def GetMetadata(cmdline, ninjalog): build_configs[k] = str(build_configs[k]) metadata = { - 'platform': platform.system(), - 'cpu_core': multiprocessing.cpu_count(), - 'build_configs': build_configs, - 'targets': GetBuildTargetFromCommandLine(cmdline), + "platform": platform.system(), + "cpu_core": multiprocessing.cpu_count(), + "build_configs": build_configs, + "targets": GetBuildTargetFromCommandLine(cmdline), } jflag = GetJflag(cmdline) if jflag is not None: - metadata['jobs'] = jflag + metadata["jobs"] = jflag return metadata @@ -168,36 +181,40 @@ def GetMetadata(cmdline, ninjalog): def GetNinjalog(cmdline): """GetNinjalog returns the path to ninjalog from cmdline.""" # ninjalog is in current working directory by default. - ninjalog_dir = '.' + ninjalog_dir = "." i = 0 while i < len(cmdline): cmd = cmdline[i] i += 1 - if cmd == '-C' and i < len(cmdline): + if cmd == "-C" and i < len(cmdline): ninjalog_dir = cmdline[i] i += 1 continue - if cmd.startswith('-C') and len(cmd) > len('-C'): - ninjalog_dir = cmd[len('-C'):] + if cmd.startswith("-C") and len(cmd) > len("-C"): + ninjalog_dir = cmd[len("-C"):] - return os.path.join(ninjalog_dir, '.ninja_log') + return os.path.join(ninjalog_dir, ".ninja_log") def main(): parser = argparse.ArgumentParser() - parser.add_argument('--server', - default='chromium-build-stats.appspot.com', - help='server to upload ninjalog file.') - parser.add_argument('--ninjalog', help='ninjalog file to upload.') - parser.add_argument('--verbose', - action='store_true', - help='Enable verbose logging.') - parser.add_argument('--cmdline', - required=True, - nargs=argparse.REMAINDER, - help='command line args passed to ninja.') + parser.add_argument( + "--server", + default="chromium-build-stats.appspot.com", + help="server to upload ninjalog file.", + ) + parser.add_argument("--ninjalog", help="ninjalog file to upload.") + parser.add_argument("--verbose", + action="store_true", + help="Enable verbose logging.") + parser.add_argument( + "--cmdline", + required=True, + nargs=argparse.REMAINDER, + help="command line args passed to ninja.", + ) args = parser.parse_args() @@ -224,27 +241,29 @@ def main(): output = io.BytesIO() with open(ninjalog) as f: - with gzip.GzipFile(fileobj=output, mode='wb') as g: + with gzip.GzipFile(fileobj=output, mode="wb") as g: g.write(f.read().encode()) - g.write(b'# end of ninja log\n') + g.write(b"# end of ninja log\n") metadata = GetMetadata(args.cmdline, ninjalog) - logging.info('send metadata: %s', json.dumps(metadata)) + logging.info("send metadata: %s", json.dumps(metadata)) g.write(json.dumps(metadata).encode()) resp = urllib.request.urlopen( - urllib.request.Request('https://' + args.server + '/upload_ninja_log/', - data=output.getvalue(), - headers={'Content-Encoding': 'gzip'})) + urllib.request.Request( + "https://" + args.server + "/upload_ninja_log/", + data=output.getvalue(), + headers={"Content-Encoding": "gzip"}, + )) if resp.status != 200: logging.warning("unexpected status code for response: %s", resp.status) return 1 - logging.info('response header: %s', resp.headers) - logging.info('response content: %s', resp.read()) + logging.info("response header: %s", resp.headers) + logging.info("response content: %s", resp.read()) return 0 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py index 7a06826e1..f17a05363 100755 --- a/ninjalog_uploader_wrapper.py +++ b/ninjalog_uploader_wrapper.py @@ -13,43 +13,43 @@ import ninjalog_uploader import subprocess2 THIS_DIR = os.path.dirname(__file__) -UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py') -CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg') +UPLOADER = os.path.join(THIS_DIR, "ninjalog_uploader.py") +CONFIG = os.path.join(THIS_DIR, "ninjalog.cfg") VERSION = 3 def LoadConfig(): if os.path.isfile(CONFIG): - with open(CONFIG, 'r') as f: + with open(CONFIG, "r") as f: try: config = json.load(f) except Exception: # Set default value when failed to load config. config = { - 'is-googler': ninjalog_uploader.IsGoogler(), - 'countdown': 10, - 'version': VERSION, + "is-googler": ninjalog_uploader.IsGoogler(), + "countdown": 10, + "version": VERSION, } - if config['version'] == VERSION: - config['countdown'] = max(0, config['countdown'] - 1) + if config["version"] == VERSION: + config["countdown"] = max(0, config["countdown"] - 1) return config return { - 'is-googler': ninjalog_uploader.IsGoogler(), - 'countdown': 10, - 'version': VERSION, + "is-googler": ninjalog_uploader.IsGoogler(), + "countdown": 10, + "version": VERSION, } def SaveConfig(config): - with open(CONFIG, 'w') as f: + with open(CONFIG, "w") as f: json.dump(config, f) def ShowMessage(countdown): - allowlisted = '\n'.join( - [' * %s' % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS]) + allowlisted = "\n".join( + [" * %s" % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS]) print(""" Your ninjalog will be uploaded to build stats server. The uploaded log will be used to analyze user side build performance. @@ -78,27 +78,32 @@ You can find a more detailed explanation in or https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md -""" % (allowlisted, countdown, __file__, __file__, - os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md")))) +""" % ( + allowlisted, + countdown, + __file__, + __file__, + os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md")), + )) def main(): config = LoadConfig() - if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': - config['opt-in'] = True - config['countdown'] = 0 + if len(sys.argv) == 2 and sys.argv[1] == "opt-in": + config["opt-in"] = True + config["countdown"] = 0 SaveConfig(config) - print('ninjalog upload is opted in.') + print("ninjalog upload is opted in.") return 0 - if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': - config['opt-in'] = False + if len(sys.argv) == 2 and sys.argv[1] == "opt-out": + config["opt-in"] = False SaveConfig(config) - print('ninjalog upload is opted out.') + print("ninjalog upload is opted out.") return 0 - if 'opt-in' in config and not config['opt-in']: + if "opt-in" in config and not config["opt-in"]: # Upload is opted out. return 0 @@ -121,13 +126,15 @@ def main(): # Run upload script without wait. devnull = open(os.devnull, "w") creationflags = 0 - if platform.system() == 'Windows': + if platform.system() == "Windows": creationflags = subprocess.CREATE_NEW_PROCESS_GROUP - subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:], - stdout=devnull, - stderr=devnull, - creationflags=creationflags) + subprocess2.Popen( + [sys.executable, UPLOADER] + sys.argv[1:], + stdout=devnull, + stderr=devnull, + creationflags=creationflags, + ) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/post_build_ninja_summary.py b/post_build_ninja_summary.py index e05cb9841..fe7219d22 100755 --- a/post_build_ninja_summary.py +++ b/post_build_ninja_summary.py @@ -94,9 +94,9 @@ class Target: """ # Allow for modest floating-point errors epsilon = 0.000002 - if (self.weighted_duration > self.Duration() + epsilon): - print('%s > %s?' % (self.weighted_duration, self.Duration())) - assert (self.weighted_duration <= self.Duration() + epsilon) + if self.weighted_duration > self.Duration() + epsilon: + print("%s > %s?" % (self.weighted_duration, self.Duration())) + assert self.weighted_duration <= self.Duration() + epsilon return self.weighted_duration def DescribeTargets(self): @@ -104,10 +104,10 @@ class Target: # Some build steps generate dozens of outputs - handle them sanely. # The max_length was chosen so that it can fit most of the long # single-target names, while minimizing word wrapping. - result = ', '.join(self.targets) + result = ", ".join(self.targets) max_length = 65 if len(result) > max_length: - result = result[:max_length] + '...' + result = result[:max_length] + "..." return result @@ -121,12 +121,12 @@ def ReadTargets(log, show_all): # targets. if not header: return [] - assert header == '# ninja log v5\n', ('unrecognized ninja log version %r' % + assert header == "# ninja log v5\n", ("unrecognized ninja log version %r" % header) targets_dict = {} last_end_seen = 0.0 for line in log: - parts = line.strip().split('\t') + parts = line.strip().split("\t") if len(parts) != 5: # If ninja.exe is rudely halted then the .ninja_log file may be # corrupt. Silently continue. @@ -170,12 +170,12 @@ def GetExtension(target, extra_patterns): steps by type.""" for output in target.targets: if extra_patterns: - for fn_pattern in extra_patterns.split(';'): - if fnmatch.fnmatch(output, '*' + fn_pattern + '*'): + for fn_pattern in extra_patterns.split(";"): + if fnmatch.fnmatch(output, "*" + fn_pattern + "*"): return fn_pattern # Not a true extension, but a good grouping. - if output.endswith('type_mappings'): - extension = 'type_mappings' + if output.endswith("type_mappings"): + extension = "type_mappings" break # Capture two extensions if present. For example: file.javac.jar should @@ -185,26 +185,26 @@ def GetExtension(target, extra_patterns): extension = ext2 + ext1 # Preserve the order in the file name. if len(extension) == 0: - extension = '(no extension found)' + extension = "(no extension found)" - if ext1 in ['.pdb', '.dll', '.exe']: - extension = 'PEFile (linking)' + if ext1 in [".pdb", ".dll", ".exe"]: + extension = "PEFile (linking)" # Make sure that .dll and .exe are grouped together and that the # .dll.lib files don't cause these to be listed as libraries break - if ext1 in ['.so', '.TOC']: - extension = '.so (linking)' + if ext1 in [".so", ".TOC"]: + extension = ".so (linking)" # Attempt to identify linking, avoid identifying as '.TOC' break # Make sure .obj files don't get categorized as mojo files - if ext1 in ['.obj', '.o']: + if ext1 in [".obj", ".o"]: break # Jars are the canonical output of java targets. - if ext1 == '.jar': + if ext1 == ".jar": break # Normalize all mojo related outputs to 'mojo'. - if output.count('.mojom') > 0: - extension = 'mojo' + if output.count(".mojom") > 0: + extension = "mojo" break return extension @@ -229,8 +229,8 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): if target.end > latest: latest = target.end total_cpu_time += target.Duration() - task_start_stop_times.append((target.start, 'start', target)) - task_start_stop_times.append((target.end, 'stop', target)) + task_start_stop_times.append((target.start, "start", target)) + task_start_stop_times.append((target.end, "stop", target)) length = latest - earliest weighted_total = 0.0 @@ -256,10 +256,10 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): if num_running > 0: # Update the total weighted time up to this moment. last_weighted_time += (time - last_time) / float(num_running) - if action_name == 'start': + if action_name == "start": # Record the total weighted task time when this task starts. running_tasks[target] = last_weighted_time - if action_name == 'stop': + if action_name == "stop": # Record the change in the total weighted task time while this task # ran. weighted_duration = last_weighted_time - running_tasks[target] @@ -267,24 +267,26 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): weighted_total += weighted_duration del running_tasks[target] last_time = time - assert (len(running_tasks) == 0) + assert len(running_tasks) == 0 # Warn if the sum of weighted times is off by more than half a second. if abs(length - weighted_total) > 500: - print('Warning: Possible corrupt ninja log, results may be ' - 'untrustworthy. Length = %.3f, weighted total = %.3f' % + print("Warning: Possible corrupt ninja log, results may be " + "untrustworthy. Length = %.3f, weighted total = %.3f" % (length, weighted_total)) # Print the slowest build steps: - print(' Longest build steps:') + print(" Longest build steps:") if elapsed_time_sorting: entries.sort(key=lambda x: x.Duration()) else: entries.sort(key=lambda x: x.WeightedDuration()) for target in entries[-long_count:]: - print(' %8.1f weighted s to build %s (%.1f s elapsed time)' % - (target.WeightedDuration(), target.DescribeTargets(), - target.Duration())) + print(" %8.1f weighted s to build %s (%.1f s elapsed time)" % ( + target.WeightedDuration(), + target.DescribeTargets(), + target.Duration(), + )) # Sum up the time by file extension/type of the output file count_by_ext = {} @@ -293,13 +295,13 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): # Scan through all of the targets to build up per-extension statistics. for target in entries: extension = GetExtension(target, extra_step_types) - time_by_ext[extension] = time_by_ext.get(extension, - 0) + target.Duration() - weighted_time_by_ext[extension] = weighted_time_by_ext.get( - extension, 0) + target.WeightedDuration() + time_by_ext[extension] = (time_by_ext.get(extension, 0) + + target.Duration()) + weighted_time_by_ext[extension] = ( + weighted_time_by_ext.get(extension, 0) + target.WeightedDuration()) count_by_ext[extension] = count_by_ext.get(extension, 0) + 1 - print(' Time by build-step type:') + print(" Time by build-step type:") # Copy to a list with extension name and total time swapped, to (time, ext) if elapsed_time_sorting: weighted_time_by_ext_sorted = sorted( @@ -309,34 +311,39 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting): (y, x) for (x, y) in weighted_time_by_ext.items()) # Print the slowest build target types: for time, extension in weighted_time_by_ext_sorted[-long_ext_count:]: - print( - ' %8.1f s weighted time to generate %d %s files ' - '(%1.1f s elapsed time sum)' % - (time, count_by_ext[extension], extension, time_by_ext[extension])) - - print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx ' - 'parallelism)' % + print(" %8.1f s weighted time to generate %d %s files " + "(%1.1f s elapsed time sum)" % ( + time, + count_by_ext[extension], + extension, + time_by_ext[extension], + )) + + print(" %.1f s weighted time (%.1f s elapsed time sum, %1.1fx " + "parallelism)" % (length, total_cpu_time, total_cpu_time * 1.0 / length)) - print(' %d build steps completed, average of %1.2f/s' % + print(" %d build steps completed, average of %1.2f/s" % (len(entries), len(entries) / (length))) def main(): - log_file = '.ninja_log' - metrics_file = 'siso_metrics.json' + log_file = ".ninja_log" + metrics_file = "siso_metrics.json" parser = argparse.ArgumentParser() - parser.add_argument('-C', dest='build_directory', help='Build directory.') + parser.add_argument("-C", dest="build_directory", help="Build directory.") parser.add_argument( - '-s', - '--step-types', - help='semicolon separated fnmatch patterns for build-step grouping') + "-s", + "--step-types", + help="semicolon separated fnmatch patterns for build-step grouping", + ) parser.add_argument( - '-e', - '--elapsed_time_sorting', + "-e", + "--elapsed_time_sorting", default=False, - action='store_true', - help='Sort output by elapsed time instead of weighted time') - parser.add_argument('--log-file', + action="store_true", + help="Sort output by elapsed time instead of weighted time", + ) + parser.add_argument("--log-file", help="specific ninja log file to analyze.") args, _extra_args = parser.parse_known_args() if args.build_directory: @@ -348,34 +355,34 @@ def main(): # Offer a convenient way to add extra step types automatically, # including when this script is run by autoninja. get() returns None if # the variable isn't set. - args.step_types = os.environ.get('chromium_step_types') + args.step_types = os.environ.get("chromium_step_types") if args.step_types: # Make room for the extra build types. global long_ext_count - long_ext_count += len(args.step_types.split(';')) + long_ext_count += len(args.step_types.split(";")) if os.path.exists(metrics_file): # Automatically handle summarizing siso builds. - cmd = ['siso.bat' if 'win32' in sys.platform else 'siso'] - cmd.extend(['metrics', 'summary']) + cmd = ["siso.bat" if "win32" in sys.platform else "siso"] + cmd.extend(["metrics", "summary"]) if args.build_directory: - cmd.extend(['-C', args.build_directory]) + cmd.extend(["-C", args.build_directory]) if args.step_types: - cmd.extend(['--step_types', args.step_types]) + cmd.extend(["--step_types", args.step_types]) if args.elapsed_time_sorting: - cmd.append('--elapsed_time_sorting') + cmd.append("--elapsed_time_sorting") subprocess.run(cmd) else: try: - with open(log_file, 'r') as log: + with open(log_file, "r") as log: entries = ReadTargets(log, False) if entries: SummarizeEntries(entries, args.step_types, args.elapsed_time_sorting) except IOError: - print('Log file %r not found, no build summary created.' % log_file) + print("Log file %r not found, no build summary created." % log_file) return errno.ENOENT -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main())