diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py index e678e1606..a56d26fc1 100755 --- a/ninjalog_uploader.py +++ b/ninjalog_uploader.py @@ -2,7 +2,6 @@ # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """ This is script to upload ninja_log from googler. @@ -32,79 +31,80 @@ import httplib2 # These build configs affect build performance a lot. # TODO(tikuta): Add 'blink_symbol_level', 'closure_compile' and # 'use_jumbo_build'. -WHITELISTED_CONFIGS = ( - 'symbol_level', 'use_goma', 'is_debug', 'is_component_build', 'enable_nacl', - 'host_os', 'host_cpu', 'target_os', 'target_cpu' -) +WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug', + 'is_component_build', 'enable_nacl', 'host_os', + 'host_cpu', 'target_os', 'target_cpu') + def IsGoogler(server): - """Check whether this script run inside corp network.""" - try: - h = httplib2.Http() - _, content = h.request('https://'+server+'/should-upload', 'GET') - return content == 'Success' - except httplib2.HttpLib2Error: - return False + """Check whether this script run inside corp network.""" + try: + h = httplib2.Http() + _, content = h.request('https://' + server + '/should-upload', 'GET') + return content == 'Success' + except httplib2.HttpLib2Error: + return False + def ParseGNArgs(gn_args): - """Parse gn_args as json and return config dictionary.""" - configs = json.loads(gn_args) - build_configs = {} + """Parse gn_args as json and return config dictionary.""" + configs = json.loads(gn_args) + build_configs = {} + + for config in configs: + key = config["name"] + if key not in WHITELISTED_CONFIGS: + continue + if 'current' in config: + build_configs[key] = config['current']['value'] + else: + build_configs[key] = config['default']['value'] - for config in configs: - key = config["name"] - if key not in WHITELISTED_CONFIGS: - continue - if 'current' in config: - build_configs[key] = config['current']['value'] - else: - build_configs[key] = config['default']['value'] + return build_configs - return build_configs def GetBuildTargetFromCommandLine(cmdline): - """Get build targets from commandline.""" + """Get build targets from commandline.""" + + # Skip argv0. + idx = 1 - # Skip argv0. - idx = 1 + # Skipping all args that involve these flags, and taking all remaining args + # as targets. + onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w') + zeroarg_flags = ('--version', '-n', '-v') - # Skipping all args that involve these flags, and taking all remaining args - # as targets. - onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w') - zeroarg_flags = ('--version', '-n', '-v') + targets = [] - targets = [] + while idx < len(cmdline): + if cmdline[idx] in onearg_flags: + idx += 2 + continue - while idx < len(cmdline): - if cmdline[idx] in onearg_flags: - idx += 2 - continue + if (cmdline[idx][:2] in onearg_flags or cmdline[idx] in zeroarg_flags): + idx += 1 + continue - if (cmdline[idx][:2] in onearg_flags or - cmdline[idx] in zeroarg_flags): - idx += 1 - continue + targets.append(cmdline[idx]) + idx += 1 - targets.append(cmdline[idx]) - idx += 1 + return targets - return targets def GetJflag(cmdline): - """Parse cmdline to get flag value for -j""" + """Parse cmdline to get flag value for -j""" - for i in range(len(cmdline)): - if (cmdline[i] == '-j' and i + 1 < len(cmdline) and - cmdline[i+1].isdigit()): - return int(cmdline[i+1]) + for i in range(len(cmdline)): + if (cmdline[i] == '-j' and i + 1 < len(cmdline) + and cmdline[i + 1].isdigit()): + return int(cmdline[i + 1]) - if (cmdline[i].startswith('-j') and - cmdline[i][len('-j'):].isdigit()): - return int(cmdline[i][len('-j'):]) + if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()): + return int(cmdline[i][len('-j'):]) def GetMetadata(cmdline, ninjalog): - """Get metadata for uploaded ninjalog. + """Get metadata for uploaded ninjalog. Returned metadata has schema defined in https://cs.chromium.org?q="type+Metadata+struct+%7B"+file:%5Einfra/go/src/infra/appengine/chromium_build_stats/ninjalog/ @@ -112,116 +112,122 @@ def GetMetadata(cmdline, ninjalog): TODO(tikuta): Collect GOMA_* env var. """ - build_dir = os.path.dirname(ninjalog) + build_dir = os.path.dirname(ninjalog) - build_configs = {} + build_configs = {} - try: - args = ['gn', 'args', build_dir, '--list', '--short', '--json'] - if sys.platform == 'win32': - # gn in PATH is bat file in windows environment (except cygwin). - args = ['cmd', '/c'] + args + try: + args = ['gn', 'args', build_dir, '--list', '--short', '--json'] + if sys.platform == 'win32': + # gn in PATH is bat file in windows environment (except cygwin). + args = ['cmd', '/c'] + args - gn_args = subprocess.check_output(args) - build_configs = ParseGNArgs(gn_args) - except subprocess.CalledProcessError as e: - logging.error("Failed to call gn %s", e) - build_configs = {} + gn_args = subprocess.check_output(args) + build_configs = ParseGNArgs(gn_args) + except subprocess.CalledProcessError as e: + logging.error("Failed to call gn %s", e) + build_configs = {} - # Stringify config. - for k in build_configs: - build_configs[k] = str(build_configs[k]) + # Stringify config. + for k in build_configs: + build_configs[k] = str(build_configs[k]) - metadata = { - 'platform': platform.system(), - 'cpu_core': multiprocessing.cpu_count(), - 'build_configs': build_configs, - 'targets': GetBuildTargetFromCommandLine(cmdline), - } + metadata = { + 'platform': platform.system(), + 'cpu_core': multiprocessing.cpu_count(), + 'build_configs': build_configs, + 'targets': GetBuildTargetFromCommandLine(cmdline), + } - jflag = GetJflag(cmdline) - if jflag is not None: - metadata['jobs'] = jflag + jflag = GetJflag(cmdline) + if jflag is not None: + metadata['jobs'] = jflag + + return metadata - return metadata def GetNinjalog(cmdline): - """GetNinjalog returns the path to ninjalog from cmdline.""" - # ninjalog is in current working directory by default. - ninjalog_dir = '.' + """GetNinjalog returns the path to ninjalog from cmdline.""" + # ninjalog is in current working directory by default. + ninjalog_dir = '.' - i = 0 - while i < len(cmdline): - cmd = cmdline[i] - i += 1 - if cmd == '-C' and i < len(cmdline): - ninjalog_dir = cmdline[i] - i += 1 - continue + i = 0 + while i < len(cmdline): + cmd = cmdline[i] + i += 1 + if cmd == '-C' and i < len(cmdline): + ninjalog_dir = cmdline[i] + i += 1 + continue - if cmd.startswith('-C') and len(cmd) > len('-C'): - ninjalog_dir = cmd[len('-C'):] + if cmd.startswith('-C') and len(cmd) > len('-C'): + ninjalog_dir = cmd[len('-C'):] - return os.path.join(ninjalog_dir, '.ninja_log') + return os.path.join(ninjalog_dir, '.ninja_log') -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--server', - default='chromium-build-stats.appspot.com', - help='server to upload ninjalog file.') - parser.add_argument('--ninjalog', help='ninjalog file to upload.') - parser.add_argument('--verbose', action='store_true', - help='Enable verbose logging.') - parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER, - help='command line args passed to ninja.') - - args = parser.parse_args() - - if args.verbose: - logging.basicConfig(level=logging.INFO) - else: - # Disable logging. - logging.disable(logging.CRITICAL) - if not IsGoogler(args.server): - return 0 +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--server', + default='chromium-build-stats.appspot.com', + help='server to upload ninjalog file.') + parser.add_argument('--ninjalog', help='ninjalog file to upload.') + parser.add_argument('--verbose', + action='store_true', + help='Enable verbose logging.') + parser.add_argument('--cmdline', + required=True, + nargs=argparse.REMAINDER, + help='command line args passed to ninja.') + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.INFO) + else: + # Disable logging. + logging.disable(logging.CRITICAL) + + if not IsGoogler(args.server): + return 0 + ninjalog = args.ninjalog or GetNinjalog(args.cmdline) + if not os.path.isfile(ninjalog): + logging.warn("ninjalog is not found in %s", ninjalog) + return 1 - ninjalog = args.ninjalog or GetNinjalog(args.cmdline) - if not os.path.isfile(ninjalog): - logging.warn("ninjalog is not found in %s", ninjalog) - return 1 + # We assume that each ninja invocation interval takes at least 2 seconds. + # This is not to have duplicate entry in server when current build is no-op. + if os.stat(ninjalog).st_mtime < time.time() - 2: + logging.info("ninjalog is not updated recently %s", ninjalog) + return 0 - # We assume that each ninja invocation interval takes at least 2 seconds. - # This is not to have duplicate entry in server when current build is no-op. - if os.stat(ninjalog).st_mtime < time.time() - 2: - logging.info("ninjalog is not updated recently %s", ninjalog) - return 0 + output = cStringIO.StringIO() - output = cStringIO.StringIO() + with open(ninjalog) as f: + with gzip.GzipFile(fileobj=output, mode='wb') as g: + g.write(f.read()) + g.write('# end of ninja log\n') - with open(ninjalog) as f: - with gzip.GzipFile(fileobj=output, mode='wb') as g: - g.write(f.read()) - g.write('# end of ninja log\n') + metadata = GetMetadata(args.cmdline, ninjalog) + logging.info('send metadata: %s', json.dumps(metadata)) + g.write(json.dumps(metadata)) - metadata = GetMetadata(args.cmdline, ninjalog) - logging.info('send metadata: %s', json.dumps(metadata)) - g.write(json.dumps(metadata)) + h = httplib2.Http() + resp_headers, content = h.request('https://' + args.server + + '/upload_ninja_log/', + 'POST', + body=output.getvalue(), + headers={'Content-Encoding': 'gzip'}) - h = httplib2.Http() - resp_headers, content = h.request( - 'https://'+args.server+'/upload_ninja_log/', 'POST', - body=output.getvalue(), headers={'Content-Encoding': 'gzip'}) + if resp_headers.status != 200: + logging.warn("unexpected status code for response: %s", resp_headers.status) + return 1 - if resp_headers.status != 200: - logging.warn("unexpected status code for response: %s", - resp_headers.status) - return 1 + logging.info('response header: %s', resp_headers) + logging.info('response content: %s', content) + return 0 - logging.info('response header: %s', resp_headers) - logging.info('response content: %s', content) - return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py index 8cdb5d93d..ae45d2515 100755 --- a/ninjalog_uploader_wrapper.py +++ b/ninjalog_uploader_wrapper.py @@ -19,30 +19,30 @@ VERSION = 2 def LoadConfig(): - if os.path.isfile(CONFIG): - with open(CONFIG, 'rb') as f: - config = json.load(f) - if config['version'] == VERSION: - config['countdown'] = max(0, config['countdown'] - 1) - return config + if os.path.isfile(CONFIG): + with open(CONFIG, 'rb') as f: + config = json.load(f) + if config['version'] == VERSION: + config['countdown'] = max(0, config['countdown'] - 1) + return config - return { - 'is-googler': ninjalog_uploader.IsGoogler( - 'chromium-build-stats.appspot.com'), - 'countdown': 10, - 'version': VERSION, - } + return { + 'is-googler': + ninjalog_uploader.IsGoogler('chromium-build-stats.appspot.com'), + 'countdown': 10, + 'version': VERSION, + } def SaveConfig(config): - with open(CONFIG, 'wb') as f: - json.dump(config, f) + with open(CONFIG, 'wb') as f: + json.dump(config, f) def ShowMessage(countdown): - whitelisted = '\n'.join([' * %s' % config for config in - ninjalog_uploader.WHITELISTED_CONFIGS]) - print(""" + whitelisted = '\n'.join( + [' * %s' % config for config in ninjalog_uploader.WHITELISTED_CONFIGS]) + print(""" Your ninjalog will be uploaded to build stats server. The uploaded log will be used to analyze user side build performance. @@ -73,46 +73,47 @@ You can find a more detailed explanation in def main(): - config = LoadConfig() - - if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': - config['opt-in'] = True - config['countdown'] = 0 - SaveConfig(config) - print('ninjalog upload is opted in.') - return 0 - - if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': - config['opt-in'] = False - SaveConfig(config) - print('ninjalog upload is opted out.') - return 0 - - if 'opt-in' in config and not config['opt-in']: - # Upload is opted out. - return 0 - - if not config.get("is-googler", False): - # Not googler. - return 0 - - if config.get("countdown", 0) > 0: - # Need to show message. - ShowMessage(config["countdown"]) - # Only save config if something has meaningfully changed. - SaveConfig(config) - return 0 - - if len(sys.argv) == 1: - # dry-run for debugging. - print("upload ninjalog dry-run") - return 0 - - # Run upload script without wait. - devnull = open(os.devnull, "w") - subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:], - stdout=devnull, stderr=devnull) + config = LoadConfig() + + if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': + config['opt-in'] = True + config['countdown'] = 0 + SaveConfig(config) + print('ninjalog upload is opted in.') + return 0 + + if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': + config['opt-in'] = False + SaveConfig(config) + print('ninjalog upload is opted out.') + return 0 + + if 'opt-in' in config and not config['opt-in']: + # Upload is opted out. + return 0 + + if not config.get("is-googler", False): + # Not googler. + return 0 + + if config.get("countdown", 0) > 0: + # Need to show message. + ShowMessage(config["countdown"]) + # Only save config if something has meaningfully changed. + SaveConfig(config) + return 0 + + if len(sys.argv) == 1: + # dry-run for debugging. + print("upload ninjalog dry-run") + return 0 + + # Run upload script without wait. + devnull = open(os.devnull, "w") + subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:], + stdout=devnull, + stderr=devnull) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main())