diff --git a/.gitignore b/.gitignore index c4eb6b38eb..a26db3ad1e 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ testing_support/google_appengine # Ignore the monitoring config. It is unique for each user. /metrics.cfg + +# Ignore the ninjalog upload config. +/ninjalog.cfg \ No newline at end of file diff --git a/OWNERS b/OWNERS index 50fa902be8..a5dcf2cec7 100644 --- a/OWNERS +++ b/OWNERS @@ -16,3 +16,6 @@ per-file ninja*=thakis@chromium.org per-file ninja*=scottmg@chromium.org per-file autoninja*=brucedawson@chromium.org + +per-file ninjalog_uploader.py=tikuta@chromium.org +per-file ninjalog_uploader_wrapper.py=tikuta@chromium.org diff --git a/autoninja b/autoninja index 2bcee51c60..3847c3dc30 100755 --- a/autoninja +++ b/autoninja @@ -16,9 +16,11 @@ if eval "$command"; then if [ "$NINJA_SUMMARIZE_BUILD" == "1" ]; then python "$(dirname -- "$0")/post_build_ninja_summary.py" "$@" fi + + # Collect ninjalog from googler. + "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command exit fi # Return an error code of 1 so that if a developer types: # "autoninja chrome && chrome" then chrome won't run if the build fails. exit 1 - diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py new file mode 100755 index 0000000000..8cbc09f2ff --- /dev/null +++ b/ninjalog_uploader.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +This is script to upload ninja_log from googler. + +Server side implementation is in +https://cs.chromium.org/chromium/infra/go/src/infra/appengine/chromium_build_stats/ + +Uploaded ninjalog is stored in BigQuery table having following schema. +https://cs.chromium.org/chromium/infra/go/src/infra/appengine/chromium_build_stats/ninjaproto/ninjalog.proto + +The log will be used to analyze user side build performance. +""" + +import argparse +import cStringIO +import gzip +import json +import logging +import multiprocessing +import os +import platform +import socket +import sys + +from third_party import httplib2 + +def IsGoogler(server): + """Check whether this script run inside corp network.""" + try: + h = httplib2.Http() + _, content = h.request('https://'+server+'/should-upload', 'GET') + return content == 'Success' + except httplib2.HttpLib2Error: + return False + +def GetMetadata(cmdline, ninjalog): + """Get metadata for uploaded ninjalog.""" + + # TODO(tikuta): Support build_configs from args.gn. + + build_dir = os.path.dirname(ninjalog) + metadata = { + 'platform': platform.system(), + 'cwd': build_dir, + 'hostname': socket.gethostname(), + 'cpu_core': multiprocessing.cpu_count(), + 'cmdline': cmdline, + } + + return metadata + +def GetNinjalog(cmdline): + """GetNinjalog returns the path to ninjalog from cmdline. + + >>> GetNinjalog(['ninja']) + './.ninja_log' + >>> GetNinjalog(['ninja', '-C', 'out/Release']) + 'out/Release/.ninja_log' + >>> GetNinjalog(['ninja', '-Cout/Release']) + 'out/Release/.ninja_log' + >>> GetNinjalog(['ninja', '-C']) + './.ninja_log' + >>> GetNinjalog(['ninja', '-C', 'out/Release', '-C', 'out/Debug']) + 'out/Debug/.ninja_log' + """ + # ninjalog is in current working directory by default. + ninjalog_dir = '.' + + i = 0 + while i < len(cmdline): + cmd = cmdline[i] + i += 1 + if cmd == '-C' and i < len(cmdline): + ninjalog_dir = cmdline[i] + i += 1 + continue + + if cmd.startswith('-C') and len(cmd) > len('-C'): + ninjalog_dir = cmd[len('-C'):] + + return os.path.join(ninjalog_dir, '.ninja_log') + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--server', + default='chromium-build-stats.appspot.com', + help='server to upload ninjalog file.') + parser.add_argument('--ninjalog', help='ninjalog file to upload.') + parser.add_argument('--verbose', action='store_true', + help='Enable verbose logging.') + parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER, + help='command line args passed to ninja.') + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.INFO) + else: + # Disable logging. + logging.disable(logging.CRITICAL) + + if not IsGoogler(args.server): + return 0 + + + ninjalog = args.ninjalog or GetNinjalog(args.cmdline) + if not os.path.isfile(ninjalog): + logging.warn("ninjalog is not found in %s", ninjalog) + return 1 + + output = cStringIO.StringIO() + + with open(ninjalog) as f: + with gzip.GzipFile(fileobj=output, mode='wb') as g: + g.write(f.read()) + g.write('# end of ninja log\n') + + metadata = GetMetadata(args.cmdline, ninjalog) + logging.info('send metadata: %s', metadata) + g.write(json.dumps(metadata)) + + h = httplib2.Http() + resp_headers, content = h.request( + 'https://'+args.server+'/upload_ninja_log/', 'POST', + body=output.getvalue(), headers={'Content-Encoding': 'gzip'}) + + if resp_headers.status != 200: + logging.warn("unexpected status code for response: %s", + resp_headers.status) + return 1 + + logging.info('response header: %s', resp_headers) + logging.info('response content: %s', content) + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py new file mode 100755 index 0000000000..7228886016 --- /dev/null +++ b/ninjalog_uploader_wrapper.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import subprocess +import json +import sys + +from third_party import httplib2 + +import ninjalog_uploader + +THIS_DIR = os.path.dirname(__file__) +UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py') +CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg') +VERSION = 1 + + +def LoadConfig(): + if os.path.isfile(CONFIG): + with open(CONFIG, 'rb') as f: + config = json.load(f) + if config['version'] == VERSION: + config['countdown'] -= 1 + return config + + return { + 'is-googler': ninjalog_uploader.IsGoogler( + 'chromium-build-stats.appspot.com'), + 'countdown': 10, + 'version': VERSION, + } + + +def SaveConfig(config): + with open(CONFIG, 'wb') as f: + json.dump(config, f) + + +def ShowMessage(countdown): + print """ +Your ninjalog will be uploaded to build stats server. Uploaded log will be used +to analyze user side build performance. + +The following information will be uploaded with ninjalog. +* OS (e.g. Win, Mac or Linux) +* build directory (e.g. /home/foo/chromium/src/out/Release) +* hostname +* number of cpu cores of building machine +* cmdline passed to ninja (e.g. ninja -C out/Default -j1024 chrome) +* build config (e.g. use_goma=true, is_component_build=true, etc) + +Uploading ninjalog will be started after you run autoninja another %d time. + +If you don't want to upload ninjalog, please run following command. +$ %s opt-out + +If you allow upload ninjalog from next autoninja run, please run the following +command. +$ %s opt-in + +If you have question about this, please send mail to infra-dev@chromium.org + +""" % (countdown, __file__, __file__) + + +def main(): + config = LoadConfig() + + if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': + config['opt-in'] = True + config['countdown'] = 0 + SaveConfig(config) + print('ninjalog upload is opted in.') + return 0 + + if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': + config['opt-in'] = False + SaveConfig(config) + print('ninjalog upload is opted out.') + return 0 + + SaveConfig(config) + + if 'opt-in' in config and not config['opt-in']: + # Upload is opted out. + return 0 + + if not config.get("is-googler", False): + # Not googler. + return 0 + + if config.get("countdown", 0) > 0: + # Need to show message. + ShowMessage(config["countdown"]) + return 0 + + if len(sys.argv) == 1: + # dry-run for debugging. + print("upload ninjalog dry-run") + return 0 + + # Run upload script without wait. + devnull = open(os.devnull, "w") + subprocess.Popen([sys.executable, UPLOADER] + sys.argv[1:], + stdout=devnull, stderr=devnull) + + +if __name__ == '__main__': + sys.exit(main())