depot_tools/autoninja.py

#!/usr/bin/env vpython
# Copyright (c) 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
This script (intended to be invoked by autoninja or autoninja.bat) detects
whether a build is accelerated using a service like goma. If so, it runs with a
large -j value, and otherwise it chooses a small one. This auto-adjustment
makes using remote build acceleration simpler and safer, and avoids errors that
can cause slow goma builds or swap-storms on unaccelerated builds.
"""

from __future__ import print_function

import multiprocessing
import os
import re
import subprocess
import sys

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))

# The -t tools are incompatible with -j
t_specified = False
j_specified = False
offline = False
output_dir = '.'
input_args = sys.argv
# On Windows the autoninja.bat script passes along the arguments enclosed in
# double quotes. This prevents multiple levels of parsing of the special '^'
# characters needed when compiling a single file but means that this script gets
# called with a single argument containing all of the actual arguments,
# separated by spaces. When this case is detected we need to do argument
# splitting ourselves. This means that arguments containing actual spaces are
# not supported by autoninja, but that is not a real limitation.
if (sys.platform.startswith('win') and len(sys.argv) == 2 and
    input_args[1].count(' ') > 0):
  input_args = sys.argv[:1] + sys.argv[1].split()

# Ninja uses getopt_long, which allow to intermix non-option arguments.
# To leave non supported parameters untouched, we do not use getopt.
for index, arg in enumerate(input_args[1:]):
  if arg.startswith('-j'):
    j_specified = True
  if arg.startswith('-t'):
    t_specified = True
  if arg == '-C':
    # + 1 to get the next argument and +1 because we trimmed off input_args[0]
    output_dir = input_args[index + 2]
  elif arg.startswith('-C'):
    # Support -Cout/Default
    output_dir = arg[2:]
  elif arg == '-o' or arg == '--offline':
    offline = True
  elif arg == '-h':
    print('autoninja: Use -o/--offline to temporary disable goma.',
          file=sys.stderr)
    print(file=sys.stderr)

# Strip -o/--offline so ninja doesn't see them.
input_args = [ arg for arg in input_args if arg != '-o' and arg != '--offline']

use_goma = False
use_rbe = False

# Currently get reclient binary and config dirs relative to output_dir.  If
# they exist and using rbe, then automatically call bootstrap to start
# reproxy.  This works under the current assumption that the output
# directory is two levels up from chromium/src.
reclient_bin_dir = os.path.join(
  output_dir, '..', '..', 'buildtools', 'reclient')
reclient_cfg = os.path.join(
  output_dir, '..', '..', 'buildtools', 'reclient_cfgs', 'reproxy.cfg')

# Attempt to auto-detect remote build acceleration. We support gn-based
# builds, where we look for args.gn in the build tree, and cmake-based builds
# where we look for rules.ninja.
if os.path.exists(os.path.join(output_dir, 'args.gn')):
  with open(os.path.join(output_dir, 'args.gn')) as file_handle:
    for line in file_handle:
      # Either use_goma or use_rbe activate build acceleration.
      #
      # This test can match multi-argument lines. Examples of this are:
      # is_debug=false use_goma=true is_official_build=false
      # use_goma=false# use_goma=true This comment is ignored
      #
      # Anything after a comment is not consider a valid argument.
      line_without_comment = line.split('#')[0]
      if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
                   line_without_comment):
        use_goma = True
        continue
      if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)',
                   line_without_comment):
        use_rbe = True
        continue
else:
  for relative_path in [
      '',  # GN keeps them in the root of output_dir
      'CMakeFiles'
  ]:
    path = os.path.join(output_dir, relative_path, 'rules.ninja')
    if os.path.exists(path):
      with open(path) as file_handle:
        for line in file_handle:
          if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
            use_goma = True
            break

# If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" (case-insensitive)
# then gomacc will use the local compiler instead of doing a goma compile. This
# is convenient if you want to briefly disable goma. It avoids having to rebuild
# the world when transitioning between goma/non-goma builds. However, it is not
# as fast as doing a "normal" non-goma build because an extra process is created
# for each compile step. Checking this environment variable ensures that
# autoninja uses an appropriate -j value in this situation.
goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
  use_goma = False

if use_goma:
  gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc'
  goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin'))
  gomacc_path = os.path.join(goma_dir, gomacc_file)
  # Don't invoke gomacc if it doesn't exist.
  if os.path.exists(gomacc_path):
    # Check to make sure that goma is running. If not, don't start the build.
    status = subprocess.call([gomacc_path, 'port'], stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, shell=False)
    if status == 1:
      print('Goma is not running. Use "goma_ctl ensure_start" to start it.',
            file=sys.stderr)
      if sys.platform.startswith('win'):
        # Set an exit code of 1 in the batch file.
        print('cmd "/c exit 1"')
      else:
        # Set an exit code of 1 by executing 'false' in the bash script.
        print('false')
      sys.exit(1)

# Specify ninja.exe on Windows so that ninja.bat can call autoninja and not
# be called back.
ninja_exe = 'ninja.exe' if sys.platform.startswith('win') else 'ninja'
ninja_exe_path = os.path.join(SCRIPT_DIR, ninja_exe)

# A large build (with or without goma) tends to hog all system resources.
# Launching the ninja process with 'nice' priorities improves this situation.
prefix_args = []
if (sys.platform.startswith('linux')
    and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'):
  # nice -10 is process priority 10 lower than default 0
  # ionice -c 3 is IO priority IDLE
  prefix_args = ['nice'] + ['-10']


# Use absolute path for ninja path,
# or fail to execute ninja if depot_tools is not in PATH.
args = prefix_args + [ninja_exe_path] + input_args[1:]

num_cores = multiprocessing.cpu_count()
if not j_specified and not t_specified:
  if use_goma or use_rbe:
    args.append('-j')
    core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40'))
    j_value = num_cores * core_multiplier

    if sys.platform.startswith('win'):
      # On windows, j value higher than 1000 does not improve build performance.
      j_value = min(j_value, 1000)
    elif sys.platform == 'darwin':
      # On Mac, j value higher than 500 causes 'Too many open files' error
      # (crbug.com/936864).
      j_value = min(j_value, 500)

    args.append('%d' % j_value)
  else:
    j_value = num_cores
    # Ninja defaults to |num_cores + 2|
    j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
    args.append('-j')
    args.append('%d' % j_value)

# On Windows, fully quote the path so that the command processor doesn't think
# the whole output is the command.
# On Linux and Mac, if people put depot_tools in directories with ' ',
# shell would misunderstand ' ' as a path separation.
# TODO(yyanagisawa): provide proper quoting for Windows.
# see https://cs.chromium.org/chromium/src/tools/mb/mb.py
for i in range(len(args)):
  if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]:
    args[i] = '"%s"' % args[i].replace('"', '\\"')

if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
  args += ['-d', 'stats']

# If using rbe and the necessary environment variables are set, also start
# reproxy (via bootstrap) before running ninja.
if (not offline and use_rbe and os.path.exists(reclient_bin_dir)
    and os.path.exists(reclient_cfg)):
  bootstrap = os.path.join(reclient_bin_dir, 'bootstrap')
  setup_args = [
    bootstrap,
    '--cfg=' + reclient_cfg,
    '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy')]

  teardown_args = [bootstrap, '--cfg=' + reclient_cfg, '--shutdown']

  cmd_sep = '\n' if sys.platform.startswith('win') else '&&'
  args = setup_args + [cmd_sep] + args + [cmd_sep] + teardown_args

if offline and not sys.platform.startswith('win'):
  # Tell goma or reclient to do local compiles. On Windows these environment
  # variables are set by the wrapper batch file.
  print('RBE_remote_disabled=1 GOMA_DISABLED=1 ' + ' '.join(args))
else:
  print(' '.join(args))