From feb9e2a4ea4b22306652d5b3049da7ef4d741cca Mon Sep 17 00:00:00 2001 From: "hinoka@chromium.org" Date: Fri, 25 Sep 2015 19:11:09 +0000 Subject: [PATCH] git cl try --luci, a set of hacks to demonstrate and iterate LUCI BUG=532220 Review URL: https://codereview.chromium.org/1344183002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@296885 0039d316-1c4b-4281-b951-d872f2087c98 --- .gitignore | 4 + git_cl.py | 17 +++- luci_hacks/README.md | 35 ++++++++ luci_hacks/__init__.py | 0 luci_hacks/luci_recipe_run.isolate | 12 +++ luci_hacks/luci_recipe_run.py | 81 ++++++++++++++++++ luci_hacks/trigger_luci_job.py | 128 +++++++++++++++++++++++++++++ 7 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 luci_hacks/README.md create mode 100644 luci_hacks/__init__.py create mode 100644 luci_hacks/luci_recipe_run.isolate create mode 100755 luci_hacks/luci_recipe_run.py create mode 100755 luci_hacks/trigger_luci_job.py diff --git a/.gitignore b/.gitignore index a242c60359..3d9b0f44b9 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,7 @@ # Ignore virtualenv created during bootstrapping. /ENV + +# Ignore intermediate isolate files +*.isolated +*.isolated.state diff --git a/git_cl.py b/git_cl.py index 4964350b27..b981b8f76b 100755 --- a/git_cl.py +++ b/git_cl.py @@ -39,6 +39,7 @@ from third_party import colorama from third_party import httplib2 from third_party import upload import auth +from luci_hacks import trigger_luci_job as luci_trigger import breakpad # pylint: disable=W0611 import clang_format import dart_format @@ -238,6 +239,17 @@ def _prefix_master(master): return '%s%s' % (prefix, master) +def trigger_luci_job(changelist, masters, options): + """Send a job to run on LUCI.""" + issue_props = changelist.GetIssueProperties() + issue = changelist.GetIssue() + patchset = changelist.GetMostRecentPatchset() + for builders_and_tests in sorted(masters.itervalues()): + for builder in sorted(builders_and_tests.iterkeys()): + luci_trigger.trigger( + builder, 'HEAD', issue, patchset, issue_props['project']) + + def trigger_try_jobs(auth_config, changelist, options, masters, category): rietveld_url = settings.GetDefaultServerUrl() rietveld_host = urlparse.urlparse(rietveld_url).hostname @@ -3084,6 +3096,7 @@ def CMDtry(parser, args): group.add_option( "-m", "--master", default='', help=("Specify a try master where to run the tries.")) + group.add_option( "--luci", action='store_true') group.add_option( "-r", "--revision", help="Revision to use for the try job; default: the " @@ -3215,7 +3228,9 @@ def CMDtry(parser, args): '\nWARNING Mismatch between local config and server. Did a previous ' 'upload fail?\ngit-cl try always uses latest patchset from rietveld. ' 'Continuing using\npatchset %s.\n' % patchset) - if not options.use_rietveld: + if options.luci: + trigger_luci_job(cl, masters, options) + elif not options.use_rietveld: try: trigger_try_jobs(auth_config, cl, options, masters, 'git_cl_try') except BuildbucketResponseException as ex: diff --git a/luci_hacks/README.md b/luci_hacks/README.md new file mode 100644 index 0000000000..b2aa8c04a8 --- /dev/null +++ b/luci_hacks/README.md @@ -0,0 +1,35 @@ +LUCI Hacks - A set of shims used to provide an iterable end-to-end demo. + +The main goal of Luci Hakcs is to be able to use iterate on Milo as if it was +displaying real data. These are a couple of hacks used to get LUCI running from +"git cl try --luci" to displaying a page on Milo. These include: + +luci_recipe_run.py: +* Downloading a depot_tools tarball onto swarming from Google Storage to bootstrap gclient. +** LUCI shouldn't require depot_tools or gclient. +* Running gclient on a swarming slave to bootstrap a full build+infra checkout. +** M1: This should check out the recipes repo instead. +** M2: The recipes repo should have been already isolated. +* Seeding properties by emitting annotation in stdout so that Milo can pick it + up +* Running annotated_run.py from a fake build directory "build/slave/bot/build" + +trigger_luci_job.py: +* Master/Builder -> Recipe + Platform mapping is hardcoded into this file. This + is information that is otherwise encoded into master.cfg/slaves.cfg. +** Actually I lied, we just assume linux right now. +** M1: This information should be encoded into the recipe via luci.cfg +* Swarming client is checked out via "git clone " +* Swarming server is hard coded into the file. This info should also be pulled + out from luci.cfg +* Triggering is done directly to swarming. Once Swarming is able to pull from + DM we can send jobs to DM instead of swarming. + + +Misc: +* This just runs the full recipe on the bot. Yes, including bot_update. +** In the future this would be probably an isolated checkout? +** This also includes having git_cache either set up a local cache, or download + the bootstrap zip file on every invocation. In reality there isn't a huge + time penalty for doing this, but at scale it does incur a non-trival amount of + unnecessary bandwidth. diff --git a/luci_hacks/__init__.py b/luci_hacks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/luci_hacks/luci_recipe_run.isolate b/luci_hacks/luci_recipe_run.isolate new file mode 100644 index 0000000000..b86adcfe61 --- /dev/null +++ b/luci_hacks/luci_recipe_run.isolate @@ -0,0 +1,12 @@ +{ + 'variables': { + 'files': [ + 'luci_recipe_run.py', + ], + 'command': [ + 'python', + 'luci_recipe_run.py', + ], + }, +} + diff --git a/luci_hacks/luci_recipe_run.py b/luci_hacks/luci_recipe_run.py new file mode 100755 index 0000000000..9217b666c0 --- /dev/null +++ b/luci_hacks/luci_recipe_run.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Download recipe prerequisites and run a single recipe.""" + + +import base64 +import json +import os +import subprocess +import sys +import tarfile +import urllib2 +import zlib + + +def download(source, dest): + u = urllib2.urlopen(source) # TODO: Verify certificate? + with open(dest, 'wb') as f: + while True: + buf = u.read(8192) + if not buf: + break + f.write(buf) + + +def unzip(source, dest): + with tarfile.open(source, 'r') as z: + z.extractall(dest) + + +def get_infra(dt_dir, root_dir): + fetch = os.path.join(dt_dir, 'fetch.py') + subprocess.check_call([sys.executable, fetch, 'infra'], cwd=root_dir) + + +def seed_properties(args): + # Assumes args[0] is factory properties and args[1] is build properties. + fact_prop_str = args[0][len('--factory-properties-gz='):] + build_prop_str = args[1][len('--build-properties-gz='):] + fact_prop = json.loads(zlib.decompress(base64.b64decode(fact_prop_str))) + build_prop = json.loads(zlib.decompress(base64.b64decode(build_prop_str))) + for k, v in fact_prop.iteritems(): + print '@@@SET_BUILD_PROPERTY@%s@%s@@@' % (k, v) + for k, v in build_prop.iteritems(): + print '@@@SET_BUILD_PROPERTY@%s@%s@@@' % (k, v) + + +def main(args): + cwd = os.getcwd() + + # Bootstrap depot tools (required for fetching build/infra) + dt_url = 'https://storage.googleapis.com/dumbtest/depot_tools.tar.gz' + dt_dir = os.path.join(cwd, 'staging') + os.makedirs(dt_dir) + dt_zip = os.path.join(dt_dir, 'depot_tools.tar.gz') + download(dt_url, os.path.join(dt_zip)) + unzip(dt_zip, dt_dir) + dt_path = os.path.join(dt_dir, 'depot_tools') + os.environ['PATH'] = '%s:%s' % (dt_path, os.environ['PATH']) + + # Fetch infra (which comes with build, which comes with recipes) + root_dir = os.path.join(cwd, 'b') + os.makedirs(root_dir) + get_infra(dt_path, root_dir) + work_dir = os.path.join(root_dir, 'build', 'slave', 'bot', 'build') + os.makedirs(work_dir) + + # Emit annotations that encapsulates build properties. + seed_properties(args) + + # JUST DO IT. + cmd = [sys.executable, '-u', '../../../scripts/slave/annotated_run.py'] + cmd.extend(args) + subprocess.check_call(cmd, cwd=work_dir) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/luci_hacks/trigger_luci_job.py b/luci_hacks/trigger_luci_job.py new file mode 100755 index 0000000000..84cdec8da1 --- /dev/null +++ b/luci_hacks/trigger_luci_job.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Tool to send a recipe job to run on Swarming.""" + + +import argparse +import base64 +import json +import os +import re +import subprocess +import sys +import zlib + + +SWARMING_URL = 'https://chromium.googlesource.com/external/swarming.client.git' +CLIENT_LOCATION = os.path.expanduser('~/.swarming_client') +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +ISOLATE = os.path.join(THIS_DIR, 'luci_recipe_run.isolate') + +# This is put in place in order to not need to parse this information from +# master.cfg. In the LUCI future this would all be stored in a luci.cfg +# file alongside the repo. +RECIPE_MAPPING = { + 'Infra Linux Trusty 64 Tester': + ('tryserver.infra', 'infra/infra_repo_trybot', 'Ubuntu-14.04'), + 'Infra Linux Precise 32 Tester': + ('tryserver.infra', 'infra/infra_repo_trybot', 'Ubuntu-12.04'), + 'Infra Mac Tester': + ('tryserver.infra', 'infra/infra_repo_trybot', 'Mac'), + 'Infra Win Tester': + ('tryserver.infra', 'infra/infra_repo_trybot', 'Win'), + 'Infra Windows Tester': + ('tryserver.infra', 'infra/infra_repo_trybot', 'Win'), + 'Infra Presubmit': + ('tryserver.infra', 'run_presubmit', 'Linux') +} + + +def parse_args(args): + # Once Clank switches to bot_update, bot_update would no longer require + # master/builder detection, and we can remove the master/builder from the args + parser = argparse.ArgumentParser() + parser.add_argument('--builder', required=True) + parser.add_argument('--issue',required=True) + parser.add_argument('--patchset', required=True) + parser.add_argument('--revision', default='HEAD') + parser.add_argument('--patch_project') + + return parser.parse_args(args) + + +def ensure_swarming_client(): + if not os.path.exists(CLIENT_LOCATION): + parent, target = os.path.split(CLIENT_LOCATION) + subprocess.check_call(['git', 'clone', SWARMING_URL, target], cwd=parent) + else: + subprocess.check_call(['git', 'pull'], cwd=CLIENT_LOCATION) + + +def archive_isolate(isolate): + isolate_py = os.path.join(CLIENT_LOCATION, 'isolate.py') + cmd = [ + sys.executable, isolate_py, 'archive', + '--isolated=%sd' % isolate, + '--isolate-server', 'https://isolateserver.appspot.com', + '--isolate=%s' % isolate] + out = subprocess.check_output(cmd) + return out.split()[0].strip() + + +def trigger_swarm(isolated, platform, build_props, factory_props): + # TODO: Make this trigger DM instead. + swarm_py = os.path.join(CLIENT_LOCATION, 'swarming.py') + build_props_gz = base64.b64encode(zlib.compress(json.dumps(build_props))) + fac_props_gz = base64.b64encode(zlib.compress(json.dumps(factory_props))) + cmd = [ + sys.executable, swarm_py, 'trigger', isolated, + '--isolate-server', 'isolateserver.appspot.com', + '--swarming', 'chromium-swarm-dev.appspot.com', + '-d', 'os', platform, + '--', + '--factory-properties-gz=%s' % fac_props_gz, + '--build-properties-gz=%s' % build_props_gz + ] + out = subprocess.check_output(cmd) + m = re.search( + r'https://chromium-swarm-dev.appspot.com/user/task/(.*)', out) + return m.group(1) + + +def trigger(builder, revision, issue, patchset, project): + """Constructs/uploads an isolated file and send the job to swarming.""" + master, recipe, platform = RECIPE_MAPPING[builder] + build_props = { + 'buildnumber': 1, + 'buildername': builder, + 'recipe': recipe, + 'mastername': master, + 'slavename': 'fakeslave', + 'revision': revision, + 'patch_project': project, + } + if issue: + build_props['issue'] = issue + if patchset: + build_props['patchset'] = patchset + factory_props = { + 'recipe': recipe + } + ensure_swarming_client() + arun_isolated = archive_isolate(ISOLATE) + task = trigger_swarm(arun_isolated, platform, build_props, factory_props) + print 'https://luci-milo.appspot.com/swarming/%s' % task + + +def main(args): + args = parse_args(args) + trigger(args.builder, args.revision, args.issue, + args.patchset, args.patch_project) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))