aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply yapf for ninjalog*.py

Browse Source 
This is preparation to fix the issue.

Bug: 992282
Change-Id: I64b237670bf460ff765df3eaf069b0e0bdcfb31d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2237268
Commit-Queue: Ye Kuang <yekuang@google.com>
Reviewed-by: Ye Kuang <yekuang@google.com>
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
changes/68/2237268/3
Takuto Ikuta 5 years ago committed by LUCI CQ
parent 37e562110f
commit a2e91dbb61
2 changed files with 204 additions and 197 deletions
Show Stats Download Patch File Download Diff File

286
ninjalog_uploader.py
Unescape Escape View File

@ -2,7 +2,6 @@
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
This is script to upload ninja_log from googler.
@ -32,79 +31,80 @@ import httplib2
# These build configs affect build performance a lot.
# TODO(tikuta): Add 'blink_symbol_level', 'closure_compile' and
# 'use_jumbo_build'.
WHITELISTED_CONFIGS = (
'symbol_level', 'use_goma', 'is_debug', 'is_component_build', 'enable_nacl',
'host_os', 'host_cpu', 'target_os', 'target_cpu'
)
WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug',
'is_component_build', 'enable_nacl', 'host_os',
'host_cpu', 'target_os', 'target_cpu')
def IsGoogler(server):
"""Check whether this script run inside corp network."""
try:
h = httplib2.Http()
_, content = h.request('https://'+server+'/should-upload', 'GET')
return content == 'Success'
except httplib2.HttpLib2Error:
return False
"""Check whether this script run inside corp network."""
try:
h = httplib2.Http()
_, content = h.request('https://' + server + '/should-upload', 'GET')
return content == 'Success'
except httplib2.HttpLib2Error:
return False
def ParseGNArgs(gn_args):
"""Parse gn_args as json and return config dictionary."""
configs = json.loads(gn_args)
build_configs = {}
"""Parse gn_args as json and return config dictionary."""
configs = json.loads(gn_args)
build_configs = {}
for config in configs:
key = config["name"]
if key not in WHITELISTED_CONFIGS:
continue
if 'current' in config:
build_configs[key] = config['current']['value']
else:
build_configs[key] = config['default']['value']
for config in configs:
key = config["name"]
if key not in WHITELISTED_CONFIGS:
continue
if 'current' in config:
build_configs[key] = config['current']['value']
else:
build_configs[key] = config['default']['value']
return build_configs
return build_configs
def GetBuildTargetFromCommandLine(cmdline):
"""Get build targets from commandline."""
"""Get build targets from commandline."""
# Skip argv0.
idx = 1
# Skip argv0.
idx = 1
# Skipping all args that involve these flags, and taking all remaining args
# as targets.
onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
zeroarg_flags = ('--version', '-n', '-v')
# Skipping all args that involve these flags, and taking all remaining args
# as targets.
onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
zeroarg_flags = ('--version', '-n', '-v')
targets = []
targets = []
while idx < len(cmdline):
if cmdline[idx] in onearg_flags:
idx += 2
continue
while idx < len(cmdline):
if cmdline[idx] in onearg_flags:
idx += 2
continue
if (cmdline[idx][:2] in onearg_flags or cmdline[idx] in zeroarg_flags):
idx += 1
continue
if (cmdline[idx][:2] in onearg_flags or
cmdline[idx] in zeroarg_flags):
idx += 1
continue
targets.append(cmdline[idx])
idx += 1
targets.append(cmdline[idx])
idx += 1
return targets
return targets
def GetJflag(cmdline):
"""Parse cmdline to get flag value for -j"""
"""Parse cmdline to get flag value for -j"""
for i in range(len(cmdline)):
if (cmdline[i] == '-j' and i + 1 < len(cmdline) and
cmdline[i+1].isdigit()):
return int(cmdline[i+1])
for i in range(len(cmdline)):
if (cmdline[i] == '-j' and i + 1 < len(cmdline)
and cmdline[i + 1].isdigit()):
return int(cmdline[i + 1])
if (cmdline[i].startswith('-j') and
cmdline[i][len('-j'):].isdigit()):
return int(cmdline[i][len('-j'):])
if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()):
return int(cmdline[i][len('-j'):])
def GetMetadata(cmdline, ninjalog):
"""Get metadata for uploaded ninjalog.
"""Get metadata for uploaded ninjalog.
Returned metadata has schema defined in
https://cs.chromium.org?q="type+Metadata+struct+%7B"+file:%5Einfra/go/src/infra/appengine/chromium_build_stats/ninjalog/
@ -112,116 +112,122 @@ def GetMetadata(cmdline, ninjalog):
TODO(tikuta): Collect GOMA_* env var.
"""
build_dir = os.path.dirname(ninjalog)
build_dir = os.path.dirname(ninjalog)
build_configs = {}
build_configs = {}
try:
args = ['gn', 'args', build_dir, '--list', '--short', '--json']
if sys.platform == 'win32':
# gn in PATH is bat file in windows environment (except cygwin).
args = ['cmd', '/c'] + args
try:
args = ['gn', 'args', build_dir, '--list', '--short', '--json']
if sys.platform == 'win32':
# gn in PATH is bat file in windows environment (except cygwin).
args = ['cmd', '/c'] + args
gn_args = subprocess.check_output(args)
build_configs = ParseGNArgs(gn_args)
except subprocess.CalledProcessError as e:
logging.error("Failed to call gn %s", e)
build_configs = {}
gn_args = subprocess.check_output(args)
build_configs = ParseGNArgs(gn_args)
except subprocess.CalledProcessError as e:
logging.error("Failed to call gn %s", e)
build_configs = {}
# Stringify config.
for k in build_configs:
build_configs[k] = str(build_configs[k])
# Stringify config.
for k in build_configs:
build_configs[k] = str(build_configs[k])
metadata = {
'platform': platform.system(),
'cpu_core': multiprocessing.cpu_count(),
'build_configs': build_configs,
'targets': GetBuildTargetFromCommandLine(cmdline),
}
metadata = {
'platform': platform.system(),
'cpu_core': multiprocessing.cpu_count(),
'build_configs': build_configs,
'targets': GetBuildTargetFromCommandLine(cmdline),
}
jflag = GetJflag(cmdline)
if jflag is not None:
metadata['jobs'] = jflag
jflag = GetJflag(cmdline)
if jflag is not None:
metadata['jobs'] = jflag
return metadata
return metadata
def GetNinjalog(cmdline):
"""GetNinjalog returns the path to ninjalog from cmdline."""
# ninjalog is in current working directory by default.
ninjalog_dir = '.'
"""GetNinjalog returns the path to ninjalog from cmdline."""
# ninjalog is in current working directory by default.
ninjalog_dir = '.'
i = 0
while i < len(cmdline):
cmd = cmdline[i]
i += 1
if cmd == '-C' and i < len(cmdline):
ninjalog_dir = cmdline[i]
i += 1
continue
i = 0
while i < len(cmdline):
cmd = cmdline[i]
i += 1
if cmd == '-C' and i < len(cmdline):
ninjalog_dir = cmdline[i]
i += 1
continue
if cmd.startswith('-C') and len(cmd) > len('-C'):
ninjalog_dir = cmd[len('-C'):]
if cmd.startswith('-C') and len(cmd) > len('-C'):
ninjalog_dir = cmd[len('-C'):]
return os.path.join(ninjalog_dir, '.ninja_log')
return os.path.join(ninjalog_dir, '.ninja_log')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--server',
default='chromium-build-stats.appspot.com',
help='server to upload ninjalog file.')
parser.add_argument('--ninjalog', help='ninjalog file to upload.')
parser.add_argument('--verbose', action='store_true',
help='Enable verbose logging.')
parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER,
help='command line args passed to ninja.')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO)
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if not IsGoogler(args.server):
return 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--server',
default='chromium-build-stats.appspot.com',
help='server to upload ninjalog file.')
parser.add_argument('--ninjalog', help='ninjalog file to upload.')
parser.add_argument('--verbose',
action='store_true',
help='Enable verbose logging.')
parser.add_argument('--cmdline',
required=True,
nargs=argparse.REMAINDER,
help='command line args passed to ninja.')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO)
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if not IsGoogler(args.server):
return 0
ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
if not os.path.isfile(ninjalog):
logging.warn("ninjalog is not found in %s", ninjalog)
return 1
ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
if not os.path.isfile(ninjalog):
logging.warn("ninjalog is not found in %s", ninjalog)
return 1
# We assume that each ninja invocation interval takes at least 2 seconds.
# This is not to have duplicate entry in server when current build is no-op.
if os.stat(ninjalog).st_mtime < time.time() - 2:
logging.info("ninjalog is not updated recently %s", ninjalog)
return 0
# We assume that each ninja invocation interval takes at least 2 seconds.
# This is not to have duplicate entry in server when current build is no-op.
if os.stat(ninjalog).st_mtime < time.time() - 2:
logging.info("ninjalog is not updated recently %s", ninjalog)
return 0
output = cStringIO.StringIO()
output = cStringIO.StringIO()
with open(ninjalog) as f:
with gzip.GzipFile(fileobj=output, mode='wb') as g:
g.write(f.read())
g.write('# end of ninja log\n')
with open(ninjalog) as f:
with gzip.GzipFile(fileobj=output, mode='wb') as g:
g.write(f.read())
g.write('# end of ninja log\n')
metadata = GetMetadata(args.cmdline, ninjalog)
logging.info('send metadata: %s', json.dumps(metadata))
g.write(json.dumps(metadata))
metadata = GetMetadata(args.cmdline, ninjalog)
logging.info('send metadata: %s', json.dumps(metadata))
g.write(json.dumps(metadata))
h = httplib2.Http()
resp_headers, content = h.request('https://' + args.server +
'/upload_ninja_log/',
'POST',
body=output.getvalue(),
headers={'Content-Encoding': 'gzip'})
h = httplib2.Http()
resp_headers, content = h.request(
'https://'+args.server+'/upload_ninja_log/', 'POST',
body=output.getvalue(), headers={'Content-Encoding': 'gzip'})
if resp_headers.status != 200:
logging.warn("unexpected status code for response: %s", resp_headers.status)
return 1
if resp_headers.status != 200:
logging.warn("unexpected status code for response: %s",
resp_headers.status)
return 1
logging.info('response header: %s', resp_headers)
logging.info('response content: %s', content)
return 0
logging.info('response header: %s', resp_headers)
logging.info('response content: %s', content)
return 0
if __name__ == '__main__':
sys.exit(main())
sys.exit(main())

115
ninjalog_uploader_wrapper.py
Unescape Escape View File

@ -19,30 +19,30 @@ VERSION = 2
def LoadConfig():
if os.path.isfile(CONFIG):
with open(CONFIG, 'rb') as f:
config = json.load(f)
if config['version'] == VERSION:
config['countdown'] = max(0, config['countdown'] - 1)
return config
if os.path.isfile(CONFIG):
with open(CONFIG, 'rb') as f:
config = json.load(f)
if config['version'] == VERSION:
config['countdown'] = max(0, config['countdown'] - 1)
return config
return {
'is-googler': ninjalog_uploader.IsGoogler(
'chromium-build-stats.appspot.com'),
'countdown': 10,
'version': VERSION,
}
return {
'is-googler':
ninjalog_uploader.IsGoogler('chromium-build-stats.appspot.com'),
'countdown': 10,
'version': VERSION,
}
def SaveConfig(config):
with open(CONFIG, 'wb') as f:
json.dump(config, f)
with open(CONFIG, 'wb') as f:
json.dump(config, f)
def ShowMessage(countdown):
whitelisted = '\n'.join([' * %s' % config for config in
ninjalog_uploader.WHITELISTED_CONFIGS])
print("""
whitelisted = '\n'.join(
[' * %s' % config for config in ninjalog_uploader.WHITELISTED_CONFIGS])
print("""
Your ninjalog will be uploaded to build stats server. The uploaded log will be
used to analyze user side build performance.
@ -73,46 +73,47 @@ You can find a more detailed explanation in
def main():
config = LoadConfig()
if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
config['opt-in'] = True
config['countdown'] = 0
SaveConfig(config)
print('ninjalog upload is opted in.')
return 0
if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
config['opt-in'] = False
SaveConfig(config)
print('ninjalog upload is opted out.')
return 0
if 'opt-in' in config and not config['opt-in']:
# Upload is opted out.
return 0
if not config.get("is-googler", False):
# Not googler.
return 0
if config.get("countdown", 0) > 0:
# Need to show message.
ShowMessage(config["countdown"])
# Only save config if something has meaningfully changed.
SaveConfig(config)
return 0
if len(sys.argv) == 1:
# dry-run for debugging.
print("upload ninjalog dry-run")
return 0
# Run upload script without wait.
devnull = open(os.devnull, "w")
subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
stdout=devnull, stderr=devnull)
config = LoadConfig()
if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
config['opt-in'] = True
config['countdown'] = 0
SaveConfig(config)
print('ninjalog upload is opted in.')
return 0
if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
config['opt-in'] = False
SaveConfig(config)
print('ninjalog upload is opted out.')
return 0
if 'opt-in' in config and not config['opt-in']:
# Upload is opted out.
return 0
if not config.get("is-googler", False):
# Not googler.
return 0
if config.get("countdown", 0) > 0:
# Need to show message.
ShowMessage(config["countdown"])
# Only save config if something has meaningfully changed.
SaveConfig(config)
return 0
if len(sys.argv) == 1:
# dry-run for debugging.
print("upload ninjalog dry-run")
return 0
# Run upload script without wait.
devnull = open(os.devnull, "w")
subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
stdout=devnull,
stderr=devnull)
if __name__ == '__main__':
sys.exit(main())
sys.exit(main())

Write Preview
Loading…
Cancel
Save