introduce ninjalog_uploader to autoninja

This CL introduces ninjalog_uploader to autoninja for posix environment.
This is to collect build statistics from googler to find user side build performance bottleneck.

ninjalog_uploader_wrapper is used to notify user and manage small config of upload script.

Bug: 900161
Change-Id: I48ac8cd1d52f64e8fdafaec43636d2d79ef9040d
Reviewed-on: https://chromium-review.googlesource.com/c/1345255
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Shinya Kawanaka <shinyak@chromium.org>
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>

.gitignore

@@ -82,3 +82,6 @@ testing_support/google_appengine
 
 # Ignore the monitoring config. It is unique for each user.
 /metrics.cfg
+
+# Ignore the ninjalog upload config.
+/ninjalog.cfg

OWNERS

@@ -16,3 +16,6 @@ per-file ninja*=thakis@chromium.org
 per-file ninja*=scottmg@chromium.org
 
 per-file autoninja*=brucedawson@chromium.org
+
+per-file ninjalog_uploader.py=tikuta@chromium.org
+per-file ninjalog_uploader_wrapper.py=tikuta@chromium.org

autoninja

@@ -16,9 +16,11 @@ if eval "$command"; then
   if [ "$NINJA_SUMMARIZE_BUILD" == "1" ]; then
     python "$(dirname -- "$0")/post_build_ninja_summary.py" "$@"
   fi
+
+  # Collect ninjalog from googler.
+  "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
   exit
 fi
 # Return an error code of 1 so that if a developer types:
 # "autoninja chrome && chrome" then chrome won't run if the build fails.
 exit 1
-

ninjalog_uploader.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This is script to upload ninja_log from googler.
+
+Server side implementation is in
+https://cs.chromium.org/chromium/infra/go/src/infra/appengine/chromium_build_stats/
+
+Uploaded ninjalog is stored in BigQuery table having following schema.
+https://cs.chromium.org/chromium/infra/go/src/infra/appengine/chromium_build_stats/ninjaproto/ninjalog.proto
+
+The log will be used to analyze user side build performance.
+"""
+
+import argparse
+import cStringIO
+import gzip
+import json
+import logging
+import multiprocessing
+import os
+import platform
+import socket
+import sys
+
+from third_party import httplib2
+
+def IsGoogler(server):
+    """Check whether this script run inside corp network."""
+    try:
+        h = httplib2.Http()
+        _, content = h.request('https://'+server+'/should-upload', 'GET')
+        return content == 'Success'
+    except httplib2.HttpLib2Error:
+        return False
+
+def GetMetadata(cmdline, ninjalog):
+    """Get metadata for uploaded ninjalog."""
+
+    # TODO(tikuta): Support build_configs from args.gn.
+
+    build_dir = os.path.dirname(ninjalog)
+    metadata = {
+        'platform': platform.system(),
+        'cwd': build_dir,
+        'hostname': socket.gethostname(),
+        'cpu_core': multiprocessing.cpu_count(),
+        'cmdline': cmdline,
+    }
+
+    return metadata
+
+def GetNinjalog(cmdline):
+    """GetNinjalog returns the path to ninjalog from cmdline.
+
+    >>> GetNinjalog(['ninja'])
+    './.ninja_log'
+    >>> GetNinjalog(['ninja', '-C', 'out/Release'])
+    'out/Release/.ninja_log'
+    >>> GetNinjalog(['ninja', '-Cout/Release'])
+    'out/Release/.ninja_log'
+    >>> GetNinjalog(['ninja', '-C'])
+    './.ninja_log'
+    >>> GetNinjalog(['ninja', '-C', 'out/Release', '-C', 'out/Debug'])
+    'out/Debug/.ninja_log'
+    """
+    # ninjalog is in current working directory by default.
+    ninjalog_dir = '.'
+
+    i = 0
+    while i < len(cmdline):
+        cmd = cmdline[i]
+        i += 1
+        if cmd == '-C' and i < len(cmdline):
+            ninjalog_dir = cmdline[i]
+            i += 1
+            continue
+
+        if cmd.startswith('-C') and len(cmd) > len('-C'):
+            ninjalog_dir = cmd[len('-C'):]
+
+    return os.path.join(ninjalog_dir, '.ninja_log')
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--server',
+                        default='chromium-build-stats.appspot.com',
+                        help='server to upload ninjalog file.')
+    parser.add_argument('--ninjalog', help='ninjalog file to upload.')
+    parser.add_argument('--verbose', action='store_true',
+                        help='Enable verbose logging.')
+    parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER,
+                        help='command line args passed to ninja.')
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.INFO)
+    else:
+        # Disable logging.
+        logging.disable(logging.CRITICAL)
+
+    if not IsGoogler(args.server):
+        return 0
+
+
+    ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
+    if not os.path.isfile(ninjalog):
+        logging.warn("ninjalog is not found in %s", ninjalog)
+        return 1
+
+    output = cStringIO.StringIO()
+
+    with open(ninjalog) as f:
+        with gzip.GzipFile(fileobj=output, mode='wb') as g:
+            g.write(f.read())
+            g.write('# end of ninja log\n')
+
+            metadata = GetMetadata(args.cmdline, ninjalog)
+            logging.info('send metadata: %s', metadata)
+            g.write(json.dumps(metadata))
+
+    h = httplib2.Http()
+    resp_headers, content = h.request(
+        'https://'+args.server+'/upload_ninja_log/', 'POST',
+        body=output.getvalue(), headers={'Content-Encoding': 'gzip'})
+
+    if resp_headers.status != 200:
+        logging.warn("unexpected status code for response: %s",
+                     resp_headers.status)
+        return 1
+
+    logging.info('response header: %s', resp_headers)
+    logging.info('response content: %s', content)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

ninjalog_uploader_wrapper.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import subprocess
+import json
+import sys
+
+from third_party import httplib2
+
+import ninjalog_uploader
+
+THIS_DIR = os.path.dirname(__file__)
+UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py')
+CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg')
+VERSION = 1
+
+
+def LoadConfig():
+    if os.path.isfile(CONFIG):
+        with open(CONFIG, 'rb') as f:
+            config = json.load(f)
+            if config['version'] == VERSION:
+                config['countdown'] -= 1
+                return config
+
+    return {
+        'is-googler': ninjalog_uploader.IsGoogler(
+            'chromium-build-stats.appspot.com'),
+        'countdown': 10,
+        'version': VERSION,
+    }
+
+
+def SaveConfig(config):
+    with open(CONFIG, 'wb') as f:
+        json.dump(config, f)
+
+
+def ShowMessage(countdown):
+    print """
+Your ninjalog will be uploaded to build stats server. Uploaded log will be used
+to analyze user side build performance.
+
+The following information will be uploaded with ninjalog.
+* OS (e.g. Win, Mac or Linux)
+* build directory (e.g. /home/foo/chromium/src/out/Release)
+* hostname
+* number of cpu cores of building machine
+* cmdline passed to ninja (e.g. ninja -C out/Default -j1024 chrome)
+* build config (e.g. use_goma=true, is_component_build=true, etc)
+
+Uploading ninjalog will be started after you run autoninja another %d time.
+
+If you don't want to upload ninjalog, please run following command.
+$ %s opt-out
+
+If you allow upload ninjalog from next autoninja run, please run the following
+command.
+$ %s opt-in
+
+If you have question about this, please send mail to infra-dev@chromium.org
+
+""" % (countdown, __file__, __file__)
+
+
+def main():
+    config = LoadConfig()
+
+    if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
+        config['opt-in'] = True
+        config['countdown'] = 0
+        SaveConfig(config)
+        print('ninjalog upload is opted in.')
+        return 0
+
+    if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
+        config['opt-in'] = False
+        SaveConfig(config)
+        print('ninjalog upload is opted out.')
+        return 0
+
+    SaveConfig(config)
+
+    if 'opt-in' in config and not config['opt-in']:
+        # Upload is opted out.
+        return 0
+
+    if not config.get("is-googler", False):
+        # Not googler.
+        return 0
+
+    if config.get("countdown", 0) > 0:
+        # Need to show message.
+        ShowMessage(config["countdown"])
+        return 0
+
+    if len(sys.argv) == 1:
+        # dry-run for debugging.
+        print("upload ninjalog dry-run")
+        return 0
+
+    # Run upload script without wait.
+    devnull = open(os.devnull, "w")
+    subprocess.Popen([sys.executable, UPLOADER] + sys.argv[1:],
+                     stdout=devnull, stderr=devnull)
+
+
+if __name__ == '__main__':
+    sys.exit(main())