[recipe_modules/cipd] Refactor creation methods.

This clarifies what the 'create' method actually expects,
and adds a nicer approach for recipes that want to synthesize
their own package definition files.

BUG=

Change-Id: Iee5d5c02260bf71d4775ddfc1d4d1ae43937b693
Reviewed-on: https://chromium-review.googlesource.com/440385
Commit-Queue: Robbie Iannucci <iannucci@chromium.org>
Reviewed-by: Vadim Shtayura <vadimsh@chromium.org>

recipe_modules/cipd/api.py

@@ -1,8 +1,145 @@
-# Copyright 2015 The Chromium Authors. All rights reserved.
+# Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import re
+
+from collections import namedtuple
+
 from recipe_engine import recipe_api
+from recipe_engine.config_types import Path
+
+
+def check_type(name, var, expect):
+  if not isinstance(var, expect):  # pragma: no cover
+    raise TypeError('%s is not %s: %r (%s)' % (
+      name, type(expect).__name__, var, type(var).__name__))
+
+
+def check_list_type(name, var, expect_inner):
+  check_type(name, var, list)
+  for i, itm in enumerate(var):
+    check_type('%s[%d]' % (name, i), itm, expect_inner)
+
+
+def check_dict_type(name, var, expect_key, expect_value):
+  check_type(name, var, dict)
+  for key, value in var.iteritems():
+    check_type('%s: key' % name, key, expect_key)
+    check_type('%s[%s]' % (name, key), value, expect_value)
+
+
+class PackageDefinition(object):
+  DIR = namedtuple('DIR', ['path', 'exclusions'])
+
+  def __init__(self, package_name, package_root, install_mode=None):
+    """Build a new PackageDefinition.
+
+    Args:
+      package_name (str) - the name of the CIPD package
+      package_root (Path) - the path on the current filesystem that all files
+        will be relative to. e.g. if your root is /.../foo, and you add the
+        file /.../foo/bar/baz.json, the final cipd package will contain
+        'bar/baz.json'.
+      install_mode (None|'copy'|'symlink') - the mechanism that the cipd client
+        should use when installing this package. If None, defaults to the
+        platform default ('copy' on windows, 'symlink' on everything else).
+    """
+    check_type('package_name', package_name, str)
+    check_type('package_root', package_root, Path)
+    check_type('install_mode', install_mode, (type(None), str))
+    if install_mode not in (None, 'copy', 'symlink'):
+      raise ValueError('invalid value for install_mode: %r' % install_mode)
+    self.package_name = package_name
+    self.package_root = package_root
+    self.install_mode = install_mode
+
+    self.dirs = []  # list(DIR)
+    self.files = []  # list(Path)
+    self.version_file = None  # str?
+
+  def _rel_path(self, path):
+    """Returns a forward-slash-delimited version of `path` which is relative to
+    the package root. Will raise ValueError if path is not inside the root."""
+    if not self.package_root.is_parent_of(path):
+      raise ValueError('path %r is not a child of the package root %r' %
+                       (path, self.package_root))
+    # we know that root has the same base and some prefix of path
+    return '/'.join(path.pieces[len(self.package_root.pieces):])
+
+  def add_dir(self, dir_path, exclusions=None):
+    """Recursively add a directory to the package.
+
+    Args:
+      dir_path (Path) - A path on the current filesystem under the
+        package_root to a directory which should be recursively included.
+      exclusions (list(str)) - A list of regexps to exclude when scanning the
+        given directory. These will be tested against the forward-slash path
+        to the file relative to `dir_path`.
+
+    Raises:
+      ValueError - dir_path is not a subdirectory of the package root.
+      re.error - one of the exclusions is not a valid regex.
+    """
+    check_type('dir_path', dir_path, Path)
+    exclusions = exclusions or []
+    check_list_type('exclusions', exclusions, str)
+    self.dirs.append(self.DIR(self._rel_path(dir_path), exclusions))
+
+  def add_file(self, file_path):
+    """Add a single file to the package.
+
+    Args:
+      file_path (Path) - A path on the current filesystem to the file you
+        wish to include.
+
+    Raises:
+      ValueError - file_path is not a subdirectory of the package root.
+    """
+    check_type('file_path', file_path, Path)
+    self.files.append(self._rel_path(file_path))
+
+  def add_version_file(self, ver_file_rel):
+    """Instruct the cipd client to place a version file in this location when
+    unpacking the package.
+
+    Version files are JSON which look like: {
+      "package_name": "infra/tools/cipd/android-amd64",
+      "instance_id": "433bfdf86c0bb82d1eee2d1a0473d3709c25d2c4"
+    }
+
+    The convention is to pick a location like '.versions/<name>.cipd_version'
+    so that a given cipd installation root might have a .versions folder full
+    of these files, one per package. This file allows executables contained
+    in the package to look for and report this file, allowing them to display
+    version information about themselves. <name> could be the name of the
+    binary tool, like 'cipd' in the example above.
+
+    A version file may be specifed exactly once per package.
+
+    Args:
+      ver_file_rel (str) - A path string relative to the installation root.
+        Should be specified in posix style (forward/slashes).
+    """
+    check_type('ver_file_rel', ver_file_rel, str)
+    if self.version_file is not None:
+      raise ValueError('add_version_file() may only be used once.')
+    self.version_file = ver_file_rel
+
+  def to_jsonish(self):
+    """Returns a JSON representation of this PackageDefinition."""
+    return {
+      'package': self.package_name,
+      'root': str(self.package_root),
+      'install_mode': self.install_mode or '',
+      'data': [
+        {'file': str(f)}
+        for f in self.files
+      ]+[
+        {'dir': str(d.path), 'exclude': d.exclusions}
+        for d in self.dirs
+      ]+([{'version_file': self.version_file}] if self.version_file else [])
+    }
 
 
 class CIPDApi(recipe_api.RecipeApi):
@@ -13,13 +150,19 @@ class CIPDApi(recipe_api.RecipeApi):
   your recipe is running inside of chrome-infrastructure's systems (buildbot,
   swarming).
   """
-  def __init__(self, *args, **kwargs):
-    super(CIPDApi, self).__init__(*args, **kwargs)
+  PackageDefinition = PackageDefinition
+
+  # pylint: disable=attribute-defined-outside-init
+  def initialize(self):
     self._cipd_credentials = None
 
   def set_service_account_credentials(self, path):
     self._cipd_credentials = path
 
+  @property
+  def executable(self):
+    return 'cipd' + ('.bat' if self.m.platform.is_win else '')
+
   @property
   def default_bot_service_account_credentials(self):
     # Path to a service account credentials to use to talk to CIPD backend.
@@ -45,11 +188,22 @@ class CIPDApi(recipe_api.RecipeApi):
     )
 
   def build(self, input_dir, output_package, package_name, install_mode=None):
+    """Builds, but does not upload, a cipd package from a directory.
+
+    Args:
+      input_dir (Path) - the directory to build the package from.
+      output_package (Path) - the file to write the package to.
+      package_name (str) - the name of the cipd package as it would appear when
+        uploaded to the cipd package server.
+      install_mode (None|'copy'|'symlink') - the mechanism that the cipd client
+        should use when installing this package. If None, defaults to the
+        platform default ('copy' on windows, 'symlink' on everything else).
+    """
     assert not install_mode or install_mode in ['copy', 'symlink']
     return self.m.step(
         'build %s' % self.m.path.basename(package_name),
         [
-          'cipd',
+          self.executable,
           'pkg-build',
           '-in', input_dir,
           '-name', package_name,
@@ -63,7 +217,7 @@ class CIPDApi(recipe_api.RecipeApi):
 
   def register(self, package_name, package_path, refs=None, tags=None):
     cmd = [
-      'cipd',
+      self.executable,
       'pkg-register', package_path,
       '-json-output', self.m.json.output(),
     ]
@@ -81,26 +235,75 @@ class CIPDApi(recipe_api.RecipeApi):
         step_test_data=lambda: self.test_api.example_register(package_name)
     )
 
-  def create(self, pkg_def, refs=None, tags=None):
-    """Creates a package based on YAML package definition file.
-
-    This builds and uploads the package in one step.
-    """
+  def _create(self, pkg_name, pkg_def_file_or_placeholder,
+              refs=None, tags=None):
+    refs = refs or []
+    tags = tags or {}
+    check_list_type('refs', refs, str)
+    check_dict_type('tags', tags, str, str)
     cmd = [
-      'cipd',
+      self.executable,
       'create',
-      '-pkg-def', pkg_def,
+      '-pkg-def', pkg_def_file_or_placeholder,
       '-json-output', self.m.json.output(),
     ]
     if self._cipd_credentials:
       cmd.extend(['-service-account-json', self._cipd_credentials])
-    if refs:
-      for ref in refs:
-        cmd.extend(['-ref', ref])
-    if tags:
-      for tag, value in sorted(tags.items()):
-        cmd.extend(['-tag', '%s:%s' % (tag, value)])
-    return self.m.step('create %s' % self.m.path.basename(pkg_def), cmd)
+    for ref in refs:
+      cmd.extend(['-ref', ref])
+    for tag, value in sorted(tags.items()):
+      cmd.extend(['-tag', '%s:%s' % (tag, value)])
+    result = self.m.step(
+      'create %s' % pkg_name, cmd,
+      step_test_data=lambda: self.test_api.m.json.output({
+        'result': self.test_api.make_pin(pkg_name),
+      }))
+    ret_data = result.json.output['result']
+    result.presentation.step_text = '</br>pkg: %(package)s' % ret_data
+    result.presentation.step_text += '</br>id: %(instance_id)s' % ret_data
+    return ret_data
+
+  def create_from_yaml(self, pkg_def, refs=None, tags=None):
+    """Builds and uploads a package based on on-disk YAML package definition
+    file.
+
+    This builds and uploads the package in one step.
+
+    Args:
+      pkg_def (Path) - The path to the yaml file.
+      refs (list(str)) - A list of ref names to set for the package instance.
+      tags (dict(str, str)) - A map of tag name -> value to set for the package
+                              instance.
+
+    Returns the JSON 'result' section, e.g.: {
+      "package": "infra/tools/cipd/android-amd64",
+      "instance_id": "433bfdf86c0bb82d1eee2d1a0473d3709c25d2c4"
+    }
+    """
+    check_type('pkg_def', pkg_def, Path)
+    return self._create(self.m.path.basename(pkg_def), pkg_def, refs, tags)
+
+  def create_from_pkg(self, pkg_def, refs=None, tags=None):
+    """Builds and uploads a package based on a PackageDefinition object.
+
+    This builds and uploads the package in one step.
+
+    Args:
+      pkg_def (PackageDefinition) - The description of the package we want to
+        create.
+      refs (list(str)) - A list of ref names to set for the package instance.
+      tags (dict(str, str)) - A map of tag name -> value to set for the package
+                              instance.
+
+    Returns the JSON 'result' section, e.g.: {
+      "package": "infra/tools/cipd/android-amd64",
+      "instance_id": "433bfdf86c0bb82d1eee2d1a0473d3709c25d2c4"
+    }
+    """
+    check_type('pkg_def', pkg_def, PackageDefinition)
+    return self._create(
+      pkg_def.package_name, self.m.json.input(pkg_def.to_jsonish()), refs, tags)
+
 
   def ensure(self, root, packages):
     """Ensures that packages are installed in a given root dir.
@@ -116,7 +319,7 @@ class CIPDApi(recipe_api.RecipeApi):
                     for name, version in sorted(packages.items())]
     ensure_file = self.m.raw_io.input('\n'.join(package_list))
     cmd = [
-      'cipd',
+      self.executable,
       'ensure',
       '-root', root,
       '-ensure-file', ensure_file,
@@ -131,7 +334,7 @@ class CIPDApi(recipe_api.RecipeApi):
 
   def set_tag(self, package_name, version, tags):
     cmd = [
-      'cipd',
+      self.executable,
       'set-tag', package_name,
       '-version', version,
       '-json-output', self.m.json.output(),
@@ -151,7 +354,7 @@ class CIPDApi(recipe_api.RecipeApi):
 
   def set_ref(self, package_name, version, refs):
     cmd = [
-      'cipd',
+      self.executable,
       'set-ref', package_name,
       '-version', version,
       '-json-output', self.m.json.output(),
@@ -173,7 +376,7 @@ class CIPDApi(recipe_api.RecipeApi):
     assert ':' in tag, 'tag must be in a form "k:v"'
 
     cmd = [
-      'cipd',
+      self.executable,
       'search', package_name,
       '-tag', tag,
       '-json-output', self.m.json.output(),
@@ -190,7 +393,7 @@ class CIPDApi(recipe_api.RecipeApi):
   def describe(self, package_name, version,
                test_data_refs=None, test_data_tags=None):
     cmd = [
-      'cipd',
+      self.executable,
       'describe', package_name,
       '-version', version,
       '-json-output', self.m.json.output(),

recipe_modules/cipd/example.expected/basic.json

@@ -302,8 +302,14 @@
     ],
     "name": "create fake-package.yaml",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
-      "@@@STEP_LOG_END@json.output (invalid)@@@"
+      "@@@STEP_TEXT@</br>pkg: fake-package.yaml</br>id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"fake-package.yaml\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {

recipe_modules/cipd/example.expected/basic_pkg.json

@@ -0,0 +1,405 @@
+[
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "public/package/linux-amd64",
+      "-tag",
+      "git_revision:40-chars-long-hash",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "public/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd describe public/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "private/package/linux-amd64 latest\npublic/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "ensure_installed (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "private/package/linux-amd64",
+      "-tag",
+      "key:value",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd search private/package/linux-amd64 key:value",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "private/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd describe private/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"custom:tagged\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"key:value\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "build fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-install-mode",
+      "copy"
+    ],
+    "name": "build fake-package (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-register",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-ref",
+      "fake-ref-1",
+      "-ref",
+      "fake-ref-2",
+      "-tag",
+      "fake_tag_1:fake_value_1",
+      "-tag",
+      "fake_tag_2:fake_value_2"
+    ],
+    "name": "register infra/fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "create",
+      "-pkg-def",
+      "{\"data\": [{\"file\": \"a/path/to/file.py\"}, {\"file\": \"some_config.cfg\"}, {\"dir\": \"directory\", \"exclude\": []}, {\"dir\": \"other_dir\", \"exclude\": [\".*\\\\.pyc\"]}], \"install_mode\": \"\", \"package\": \"infra/fake-package\", \"root\": \"[START_DIR]/some_subdir\"}",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-ref",
+      "fake-ref-1",
+      "-ref",
+      "fake-ref-2",
+      "-tag",
+      "fake_tag_1:fake_value_1",
+      "-tag",
+      "fake_tag_2:fake_value_2"
+    ],
+    "name": "create infra/fake-package",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@</br>pkg: infra/fake-package</br>id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "set-tag",
+      "fake-package",
+      "-version",
+      "long/weird/ref/which/doesn/not/fit/into/40chars",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-tag",
+      "dead:beaf",
+      "-tag",
+      "more:value"
+    ],
+    "name": "cipd set-tag fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"fake-package\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-long/weird/ref/w\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "set-ref",
+      "fake-package",
+      "-version",
+      "latest",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-ref",
+      "any",
+      "-ref",
+      "some"
+    ],
+    "name": "cipd set-ref fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"fake-package\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "fake-package/linux-amd64",
+      "-tag",
+      "dead:beaf",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd search fake-package/linux-amd64 dead:beaf",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"fake-package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]

recipe_modules/cipd/example.expected/describe-many-instances.json

@@ -302,8 +302,14 @@
     ],
     "name": "create fake-package.yaml",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
-      "@@@STEP_LOG_END@json.output (invalid)@@@"
+      "@@@STEP_TEXT@</br>pkg: fake-package.yaml</br>id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"fake-package.yaml\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {

recipe_modules/cipd/example.expected/mac64.json

@@ -302,8 +302,14 @@
     ],
     "name": "create fake-package.yaml",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
-      "@@@STEP_LOG_END@json.output (invalid)@@@"
+      "@@@STEP_TEXT@</br>pkg: fake-package.yaml</br>id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"fake-package.yaml\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {

recipe_modules/cipd/example.expected/pkg_bad_file.json

@@ -0,0 +1,289 @@
+[
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "public/package/linux-amd64",
+      "-tag",
+      "git_revision:40-chars-long-hash",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "public/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd describe public/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "private/package/linux-amd64 latest\npublic/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "ensure_installed (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "private/package/linux-amd64",
+      "-tag",
+      "key:value",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd search private/package/linux-amd64 key:value",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "private/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd describe private/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"custom:tagged\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"key:value\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "build fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-install-mode",
+      "copy"
+    ],
+    "name": "build fake-package (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-register",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-ref",
+      "fake-ref-1",
+      "-ref",
+      "fake-ref-2",
+      "-tag",
+      "fake_tag_1:fake_value_1",
+      "-tag",
+      "fake_tag_2:fake_value_2"
+    ],
+    "name": "register infra/fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Uncaught Exception: ValueError(\"path Path('start_dir', 'a','path','to','file.py') is not a child of the package root Path('start_dir', 'some_subdir')\",)",
+    "status_code": -1
+  }
+]

recipe_modules/cipd/example.expected/pkg_bad_mode.json

@@ -0,0 +1,289 @@
+[
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "public/package/linux-amd64",
+      "-tag",
+      "git_revision:40-chars-long-hash",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "public/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd describe public/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "private/package/linux-amd64 latest\npublic/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "ensure_installed (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "private/package/linux-amd64",
+      "-tag",
+      "key:value",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd search private/package/linux-amd64 key:value",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "private/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd describe private/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"custom:tagged\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"key:value\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "build fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-install-mode",
+      "copy"
+    ],
+    "name": "build fake-package (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-register",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-ref",
+      "fake-ref-1",
+      "-ref",
+      "fake-ref-2",
+      "-tag",
+      "fake_tag_1:fake_value_1",
+      "-tag",
+      "fake_tag_2:fake_value_2"
+    ],
+    "name": "register infra/fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Uncaught Exception: ValueError(\"invalid value for install_mode: ''\",)",
+    "status_code": -1
+  }
+]

recipe_modules/cipd/example.expected/pkg_bad_verfile.json

@@ -0,0 +1,289 @@
+[
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "public/package/linux-amd64",
+      "-tag",
+      "git_revision:40-chars-long-hash",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "public/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "/creds/service_accounts/service-account-cipd-builder.json"
+    ],
+    "name": "cipd describe public/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "ensure",
+      "-root",
+      "[START_DIR]/packages",
+      "-ensure-file",
+      "private/package/linux-amd64 latest\npublic/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "ensure_installed (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"public/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "search",
+      "private/package/linux-amd64",
+      "-tag",
+      "key:value",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd search private/package/linux-amd64 key:value",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "describe",
+      "private/package/linux-amd64",
+      "-version",
+      "40-chars-fake-of-the-package-instance_id",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json"
+    ],
+    "name": "cipd describe private/package/linux-amd64",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"pin\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"private/package/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"refs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"modified_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"ref\": \"latest\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ], @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"tags\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"custom:tagged\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"registered_ts\": 1446574210, @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"tag\": \"key:value\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }@@@",
+      "@@@STEP_LOG_LINE@json.output@    ]@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "build fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-build",
+      "-in",
+      "fake-input-dir",
+      "-name",
+      "infra/fake-package",
+      "-out",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-install-mode",
+      "copy"
+    ],
+    "name": "build fake-package (2)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "cipd",
+      "pkg-register",
+      "fake-package-path",
+      "-json-output",
+      "/path/to/tmp/json",
+      "-service-account-json",
+      "fake-credentials.json",
+      "-ref",
+      "fake-ref-1",
+      "-ref",
+      "fake-ref-2",
+      "-tag",
+      "fake_tag_1:fake_value_1",
+      "-tag",
+      "fake_tag_2:fake_value_2"
+    ],
+    "name": "register infra/fake-package",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"infra/fake-package\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "reason": "Uncaught Exception: ValueError('add_version_file() may only be used once.',)",
+    "status_code": -1
+  }
+]

recipe_modules/cipd/example.expected/win64.json

@@ -1,7 +1,7 @@
 [
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "ensure",
       "-root",
       "[START_DIR]\\packages",
@@ -27,7 +27,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "search",
       "public/package/windows-amd64",
       "-tag",
@@ -52,7 +52,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "describe",
       "public/package/windows-amd64",
       "-version",
@@ -103,7 +103,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "ensure",
       "-root",
       "[START_DIR]\\packages",
@@ -133,7 +133,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "search",
       "private/package/windows-amd64",
       "-tag",
@@ -158,7 +158,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "describe",
       "private/package/windows-amd64",
       "-version",
@@ -204,7 +204,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "pkg-build",
       "-in",
       "fake-input-dir",
@@ -228,7 +228,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "pkg-build",
       "-in",
       "fake-input-dir",
@@ -254,7 +254,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "pkg-register",
       "fake-package-path",
       "-json-output",
@@ -283,7 +283,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "create",
       "-pkg-def",
       "[START_DIR]\\fake-package.yaml",
@@ -302,13 +302,19 @@
     ],
     "name": "create fake-package.yaml",
     "~followup_annotations": [
-      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
-      "@@@STEP_LOG_END@json.output (invalid)@@@"
+      "@@@STEP_TEXT@</br>pkg: fake-package.yaml</br>id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+      "@@@STEP_LOG_LINE@json.output@    \"package\": \"fake-package.yaml\"@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
     ]
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "set-tag",
       "fake-package",
       "-version",
@@ -338,7 +344,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "set-ref",
       "fake-package",
       "-version",
@@ -368,7 +374,7 @@
   },
   {
     "cmd": [
-      "cipd",
+      "cipd.bat",
       "search",
       "fake-package/windows-amd64",
       "-tag",

recipe_modules/cipd/example.py

@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from recipe_engine.config import List, Single, ConfigList, ConfigGroup
+from recipe_engine.recipe_api import Property
+
 DEPS = [
   'recipe_engine/path',
   'recipe_engine/platform',
@@ -10,7 +13,18 @@ DEPS = [
   'cipd',
 ]
 
-def RunSteps(api):
+PROPERTIES = {
+  'use_pkg': Property(default=False, kind=bool),
+  'pkg_files': Property(default=(), kind=List(str)),
+  'pkg_dirs': Property(default=(), kind=ConfigList(lambda: ConfigGroup(
+    path=Single(str),
+    exclusions=List(str),
+  ))),
+  'ver_files': Property(default=(), kind=List(str)),
+  'install_mode': Property(default=None),
+}
+
+def RunSteps(api, use_pkg, pkg_files, pkg_dirs, ver_files, install_mode):
   # Need to set service account credentials.
   api.cipd.set_service_account_credentials(
       api.cipd.default_bot_service_account_credentials)
@@ -49,10 +63,27 @@ def RunSteps(api):
                           'fake_tag_2': 'fake_value_2'})
 
   # Create (build & register).
-  api.cipd.create(api.path['start_dir'].join('fake-package.yaml'),
-                  refs=['fake-ref-1', 'fake-ref-2'],
-                  tags={'fake_tag_1': 'fake_value_1',
-                        'fake_tag_2': 'fake_value_2'})
+  if use_pkg:
+    root = api.path['start_dir'].join('some_subdir')
+    pkg = api.cipd.PackageDefinition('infra/fake-package', root, install_mode)
+    for fullpath in pkg_files:
+      pkg.add_file(api.path.abs_to_path(fullpath))
+    for obj in pkg_dirs:
+      pkg.add_dir(api.path.abs_to_path(obj.get('path', '')),
+                  obj.get('exclusions'))
+    for pth in ver_files:
+      pkg.add_version_file(pth)
+
+    api.cipd.create_from_pkg(pkg,
+                             refs=['fake-ref-1', 'fake-ref-2'],
+                             tags={'fake_tag_1': 'fake_value_1',
+                                   'fake_tag_2': 'fake_value_2'})
+  else:
+    api.cipd.create_from_yaml(api.path['start_dir'].join('fake-package.yaml'),
+                              refs=['fake-ref-1', 'fake-ref-2'],
+                              tags={'fake_tag_1': 'fake_value_1',
+                                    'fake_tag_2': 'fake_value_2'})
+
 
   # Set tag or ref of an already existing package.
   api.cipd.set_tag('fake-package',
@@ -67,39 +98,89 @@ def RunSteps(api):
 def GenTests(api):
   yield (
     # This is very common dev workstation, but not all devs are on it.
-    api.test('basic') +
-    api.platform('linux', 64)
+    api.test('basic')
+    + api.platform('linux', 64)
   )
 
   yield (
-    api.test('mac64') +
-    api.platform('mac', 64)
+    api.test('mac64')
+    + api.platform('mac', 64)
   )
 
   yield (
-    api.test('win64') +
-    api.platform('win', 64)
+    api.test('win64')
+    + api.platform('win', 64)
   )
 
   yield (
-    api.test('describe-failed') +
-    api.platform('linux', 64) +
-    api.override_step_data(
+    api.test('describe-failed')
+    + api.platform('linux', 64)
+    + api.override_step_data(
       'cipd describe public/package/linux-amd64',
       api.cipd.example_error(
         'package "public/package/linux-amd64-ubuntu14_04" not registered',
-      )
-    )
+      ))
   )
 
   yield (
-    api.test('describe-many-instances') +
-    api.platform('linux', 64) +
-    api.override_step_data(
+    api.test('describe-many-instances')
+    + api.platform('linux', 64)
+    + api.override_step_data(
       'cipd search fake-package/linux-amd64 dead:beaf',
       api.cipd.example_search(
         'public/package/linux-amd64-ubuntu14_04',
         instances=3
-      )
+      ))
+  )
+
+  yield (
+    api.test('basic_pkg')
+    + api.properties(
+      use_pkg=True,
+      pkg_files=[
+        '[START_DIR]/some_subdir/a/path/to/file.py',
+        '[START_DIR]/some_subdir/some_config.cfg',
+      ],
+      pkg_dirs=[
+        {
+          'path': '[START_DIR]/some_subdir/directory',
+        },
+        {
+          'path': '[START_DIR]/some_subdir/other_dir',
+          'exclusions': [
+            r'.*\.pyc',
+          ]
+        },
+      ],
+      ver_file=['.versions/file.cipd_version'],
+    )
+  )
+
+  yield (
+    api.test('pkg_bad_verfile')
+    + api.properties(
+      use_pkg=True,
+      ver_files=['a', 'b'],
+    )
+    + api.expect_exception('ValueError')
+  )
+
+  yield (
+    api.test('pkg_bad_mode')
+    + api.properties(
+      use_pkg=True,
+      install_mode='',
+    )
+    + api.expect_exception('ValueError')
+  )
+
+  yield (
+    api.test('pkg_bad_file')
+    + api.properties(
+      use_pkg=True,
+      pkg_files=[
+        '[START_DIR]/a/path/to/file.py',
+      ],
     )
+    + api.expect_exception('ValueError')
   )