Set up depot_tools as a recipe package, and add depot_tools recipe module.

This is a good prototypical example of how to do it :-).

TBR for OWNERS change for recipes.cfg.  

BUG=564920
R=iannucci@chromium.org,martiniss@chromium.org
TBR=phajdan.jr@chromium.org

Review URL: https://codereview.chromium.org/1494103004

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@297851 0039d316-1c4b-4281-b951-d872f2087c98

.gitignore

@@ -50,3 +50,6 @@
 # Ignore intermediate isolate files
 *.isolated
 *.isolated.state
+
+# Ignore recipe working directory.
+/.recipe_deps

PRESUBMIT.py

@@ -16,10 +16,12 @@ def CommonChecks(input_api, output_api, tests_to_black_list):
   results = []
   results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
   black_list = list(input_api.DEFAULT_BLACK_LIST) + [
+      r'^\.recipe_deps[\/\\].*',
       r'^cpplint\.py$',
       r'^cpplint_chromium\.py$',
       r'^external_bin[\/\\].+',
       r'^python[0-9]*_bin[\/\\].+',
+      r'^recipes\.py$',
       r'^site-packages-py[0-9]\.[0-9][\/\\].+',
       r'^svn_bin[\/\\].+',
       r'^testing_support[\/\\]_rietveld[\/\\].+']

infra/config/OWNERS

@@ -3,3 +3,5 @@ akuegel@chromium.org
 phajdan.jr@chromium.org
 sergiyb@chromium.org
 tandrii@chromium.org
+
+per-file recipes.cfg=luqui@chromium.org

infra/config/recipes.cfg

@@ -0,0 +1,9 @@
+api_version: 1
+project_id: "depot_tools"
+recipes_path: ""
+deps {
+  project_id: "recipe_engine"
+  url: "https://chromium.googlesource.com/external/github.com/luci/recipes-py.git"
+  branch: "master"
+  revision: "cbbcfad66fa7239561436e495963bf9a32cecdf2"
+}

recipe_modules/depot_tools/__init__.py

@@ -0,0 +1 @@
+DEPS = []

recipe_modules/depot_tools/api.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Entry point for interacting with depot_tools from recipes."""
+
+from recipe_engine import recipe_api
+
+class DepotToolsApi(recipe_api.RecipeApi):
+  @property
+  def gclient_py(self):
+    return self.package_resource('gclient.py')

recipe_modules/depot_tools/example.expected/basic.json

@@ -0,0 +1,17 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE[depot_tools]/gclient.py",
+      "--help"
+    ],
+    "cwd": "[SLAVE_BUILD]",
+    "name": "gclient help"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]

recipe_modules/depot_tools/example.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+  'recipe_engine/python',
+  'depot_tools',
+]
+
+def RunSteps(api):
+  api.python('gclient help', api.depot_tools.gclient_py, ['--help'])
+
+def GenTests(api):
+  yield api.test('basic')

recipes.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Bootstrap script to clone and forward to the recipe engine tool."""
+
+import ast
+import logging
+import os
+import random
+import re
+import subprocess
+import sys
+import time
+import traceback
+
+BOOTSTRAP_VERSION = 1
+# The root of the repository relative to the directory of this file.
+REPO_ROOT = ''
+# The path of the recipes.cfg file relative to the root of the repository.
+RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg')
+
+
+def parse_protobuf(fh):
+  """Parse the protobuf text format just well enough to understand recipes.cfg.
+
+  We don't use the protobuf library because we want to be as self-contained
+  as possible in this bootstrap, so it can be simply vendored into a client
+  repo.
+
+  We assume all fields are repeated since we don't have a proto spec to work
+  with.
+
+  Args:
+    fh: a filehandle containing the text format protobuf.
+  Returns:
+    A recursive dictionary of lists.
+  """
+  def parse_atom(text):
+    if text == 'true': return True
+    if text == 'false': return False
+    return ast.literal_eval(text)
+
+  ret = {}
+  for line in fh:
+    line = line.strip()
+    m = re.match(r'(\w+)\s*:\s*(.*)', line)
+    if m:
+      ret.setdefault(m.group(1), []).append(parse_atom(m.group(2)))
+      continue
+
+    m = re.match(r'(\w+)\s*{', line)
+    if m:
+      subparse = parse_protobuf(fh)
+      ret.setdefault(m.group(1), []).append(subparse)
+      continue
+
+    if line == '}': return ret
+    if line == '': continue
+
+    raise Exception('Could not understand line: <%s>' % line)
+
+  return ret
+
+
+def get_unique(things):
+  if len(things) == 1:
+    return things[0]
+  elif len(things) == 0:
+    raise ValueError("Expected to get one thing, but dinna get none.")
+  else:
+    logging.warn('Expected to get one thing, but got a bunch: %s\n%s' %
+                 (things, traceback.format_stack()))
+    return things[0]
+
+
+def main():
+  if sys.platform.startswith(('win', 'cygwin')):
+    git = 'git.bat'
+  else:
+    git = 'git'
+
+  # Find the repository and config file to operate on.
+  repo_root = os.path.abspath(
+      os.path.join(os.path.dirname(__file__), REPO_ROOT))
+  recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG)
+
+  with open(recipes_cfg_path, 'rU') as fh:
+    protobuf = parse_protobuf(fh)
+
+  engine_buf = get_unique([
+      b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ])
+  engine_url = get_unique(engine_buf['url'])
+  engine_revision = get_unique(engine_buf['revision'])
+  engine_subpath = (get_unique(engine_buf.get('path_override', ['']))
+                    .replace('/', os.path.sep))
+
+  recipes_path = os.path.join(repo_root,
+      get_unique(protobuf['recipes_path']).replace('/', os.path.sep))
+  deps_path = os.path.join(recipes_path, '.recipe_deps')
+  engine_path = os.path.join(deps_path, 'recipe_engine')
+
+  # Ensure that we have the recipe engine cloned.
+  def ensure_engine():
+    if not os.path.exists(deps_path):
+      os.makedirs(deps_path)
+    if not os.path.exists(engine_path):
+      subprocess.check_call([git, 'clone', engine_url, engine_path])
+
+    needs_fetch = subprocess.call(
+        [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision],
+        cwd=engine_path, stdout=open(os.devnull, 'w'))
+    if needs_fetch:
+      subprocess.check_call([git, 'fetch'], cwd=engine_path)
+    subprocess.check_call(
+        [git, 'checkout', '--quiet', engine_revision], cwd=engine_path)
+
+  try:
+    ensure_engine()
+  except subprocess.CalledProcessError as e:
+    if e.returncode == 128:  # Thrown when git gets a lock error.
+      time.sleep(random.uniform(2,5))
+      ensure_engine()
+    else:
+      raise
+
+  args = ['--package', recipes_cfg_path,
+          '--bootstrap-script', __file__] + sys.argv[1:]
+  return subprocess.call([
+      sys.executable, '-u',
+      os.path.join(engine_path, engine_subpath, 'recipes.py')] + args)
+
+if __name__ == '__main__':
+  sys.exit(main())

tests/recipes_test.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs simulation tests and lint on the recipes."""
+
+import os
+import subprocess
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+def recipes_py(*args):
+  subprocess.check_call([os.path.join(ROOT_DIR, 'recipes.py')] + list(args))
+
+recipes_py('simulation_test', '--threshold=92')
+recipes_py('lint')