From 7929675b19082a057ceb9d1daa87565b6793add5 Mon Sep 17 00:00:00 2001 From: "phajdan.jr" Date: Fri, 12 Aug 2016 02:35:40 -0700 Subject: [PATCH] Add cipd recipe module to depot_tools Copied from https://chromium.googlesource.com/chromium/tools/build.git/+/dfd62794c4925e42fb06a493149683122f68a791/scripts/slave/recipe_modules/cipd/ It's being moved so that we can use the module in git recipe module in depot_tools. See https://goto.google.com/jmywn for more context. BUG=635421, 629679 Review-Url: https://codereview.chromium.org/2243773002 --- recipe_modules/cipd/__init__.py | 10 + recipe_modules/cipd/api.py | 244 ++++++++++ .../cipd/example.expected/basic.json | 445 +++++++++++++++++ .../example.expected/describe-failed.json | 128 +++++ .../describe-many-instances.json | 453 ++++++++++++++++++ .../cipd/example.expected/install-failed.json | 31 ++ .../cipd/example.expected/mac64.json | 445 +++++++++++++++++ .../cipd/example.expected/win64.json | 445 +++++++++++++++++ recipe_modules/cipd/example.py | 115 +++++ recipe_modules/cipd/resources/bootstrap.py | 218 +++++++++ recipe_modules/cipd/test_api.py | 102 ++++ 11 files changed, 2636 insertions(+) create mode 100644 recipe_modules/cipd/__init__.py create mode 100644 recipe_modules/cipd/api.py create mode 100644 recipe_modules/cipd/example.expected/basic.json create mode 100644 recipe_modules/cipd/example.expected/describe-failed.json create mode 100644 recipe_modules/cipd/example.expected/describe-many-instances.json create mode 100644 recipe_modules/cipd/example.expected/install-failed.json create mode 100644 recipe_modules/cipd/example.expected/mac64.json create mode 100644 recipe_modules/cipd/example.expected/win64.json create mode 100644 recipe_modules/cipd/example.py create mode 100644 recipe_modules/cipd/resources/bootstrap.py create mode 100644 recipe_modules/cipd/test_api.py diff --git a/recipe_modules/cipd/__init__.py b/recipe_modules/cipd/__init__.py new file mode 100644 index 0000000000..f05ee616bd --- /dev/null +++ b/recipe_modules/cipd/__init__.py @@ -0,0 +1,10 @@ +DEPS = [ + 'infra_paths', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', +] diff --git a/recipe_modules/cipd/api.py b/recipe_modules/cipd/api.py new file mode 100644 index 0000000000..94a777b931 --- /dev/null +++ b/recipe_modules/cipd/api.py @@ -0,0 +1,244 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine import recipe_api + + +class CIPDApi(recipe_api.RecipeApi): + """CIPDApi provides support for CIPD.""" + def __init__(self, *args, **kwargs): + super(CIPDApi, self).__init__(*args, **kwargs) + self._cipd_executable = None + self._cipd_version = None + self._cipd_credentials = None + + def set_service_account_credentials(self, path): + self._cipd_credentials = path + + @property + def default_bot_service_account_credentials(self): + # Path to a service account credentials to use to talk to CIPD backend. + # Deployed by Puppet. + if self.m.platform.is_win: + return 'C:\\creds\\service_accounts\\service-account-cipd-builder.json' + else: + return '/creds/service_accounts/service-account-cipd-builder.json' + + def platform_suffix(self): + """Use to get full package name that is platform indepdent. + + Example: + >>> 'my/package/%s' % api.cipd.platform_suffix() + 'my/package/linux-amd64' + """ + return '%s-%s' % ( + self.m.platform.name.replace('win', 'windows'), + { + 32: '386', + 64: 'amd64', + }[self.m.platform.bits], + ) + + def install_client(self, step_name='install cipd', version=None): + """Ensures the client is installed. + + If you specify version as a hash, make sure its correct platform. + """ + # TODO(seanmccullough): clean up older CIPD installations. + step = self.m.python( + name=step_name, + script=self.resource('bootstrap.py'), + args=[ + '--platform', self.platform_suffix(), + '--dest-directory', self.m.path['slave_build'].join('cipd'), + '--json-output', self.m.json.output(), + ] + + (['--version', version] if version else []), + step_test_data=lambda: self.test_api.example_install_client(version) + ) + self._cipd_executable = step.json.output['executable'] + + step.presentation.step_text = ( + 'cipd instance_id: %s' % step.json.output['instance_id']) + return step + + def get_executable(self): + return self._cipd_executable + + def build(self, input_dir, output_package, package_name, install_mode=None): + assert self._cipd_executable + assert not install_mode or install_mode in ['copy', 'symlink'] + return self.m.step( + 'build %s' % self.m.path.basename(package_name), + [ + self._cipd_executable, + 'pkg-build', + '--in', input_dir, + '--name', package_name, + '--out', output_package, + '--json-output', self.m.json.output(), + ] + ( + ['--install-mode', install_mode] if install_mode else [] + ), + step_test_data=lambda: self.test_api.example_build(package_name) + ) + + def register(self, package_name, package_path, refs=None, tags=None): + assert self._cipd_executable + + cmd = [ + self._cipd_executable, + 'pkg-register', package_path, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + if refs: + for ref in refs: + cmd.extend(['--ref', ref]) + if tags: + for tag, value in sorted(tags.items()): + cmd.extend(['--tag', '%s:%s' % (tag, value)]) + return self.m.step( + 'register %s' % package_name, + cmd, + step_test_data=lambda: self.test_api.example_register(package_name) + ) + + def create(self, pkg_def, refs=None, tags=None): + """Creates a package based on YAML package definition file. + + This builds and uploads the package in one step. + """ + assert self._cipd_executable + cmd = [ + self._cipd_executable, + 'create', + '--pkg-def', pkg_def, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + if refs: + for ref in refs: + cmd.extend(['--ref', ref]) + if tags: + for tag, value in sorted(tags.items()): + cmd.extend(['--tag', '%s:%s' % (tag, value)]) + return self.m.step('create %s' % self.m.path.basename(pkg_def), cmd) + + def ensure(self, root, packages): + """Ensures that packages are installed in a given root dir. + + packages must be a mapping from package name to its version, where + * name must be for right platform (see also ``platform_suffix``), + * version could be either instance_id, or ref, or unique tag. + + If installing a package requires credentials, call + ``set_service_account_credentials`` before calling this function. + """ + assert self._cipd_executable + + package_list = ['%s %s' % (name, version) + for name, version in sorted(packages.items())] + list_data = self.m.raw_io.input('\n'.join(package_list)) + cmd = [ + self._cipd_executable, + 'ensure', + '--root', root, + '--list', list_data, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + return self.m.step( + 'ensure_installed', cmd, + step_test_data=lambda: self.test_api.example_ensure(packages) + ) + + def set_tag(self, package_name, version, tags): + assert self._cipd_executable + + cmd = [ + self._cipd_executable, + 'set-tag', package_name, + '--version', version, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + for tag, value in sorted(tags.items()): + cmd.extend(['--tag', '%s:%s' % (tag, value)]) + + return self.m.step( + 'cipd set-tag %s' % package_name, + cmd, + step_test_data=lambda: self.test_api.example_set_tag( + package_name, version + ) + ) + + def set_ref(self, package_name, version, refs): + assert self._cipd_executable + + cmd = [ + self._cipd_executable, + 'set-ref', package_name, + '--version', version, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + for r in refs: + cmd.extend(['--ref', r]) + + return self.m.step( + 'cipd set-ref %s' % package_name, + cmd, + step_test_data=lambda: self.test_api.example_set_ref( + package_name, version + ) + ) + + def search(self, package_name, tag): + assert self._cipd_executable + assert ':' in tag, 'tag must be in a form "k:v"' + + cmd = [ + self._cipd_executable, + 'search', package_name, + '--tag', tag, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + + return self.m.step( + 'cipd search %s %s' % (package_name, tag), + cmd, + step_test_data=lambda: self.test_api.example_search(package_name) + ) + + def describe(self, package_name, version, + test_data_refs=None, test_data_tags=None): + assert self._cipd_executable + + cmd = [ + self._cipd_executable, + 'describe', package_name, + '--version', version, + '--json-output', self.m.json.output(), + ] + if self._cipd_credentials: + cmd.extend(['--service-account-json', self._cipd_credentials]) + + return self.m.step( + 'cipd describe %s' % package_name, + cmd, + step_test_data=lambda: self.test_api.example_describe( + package_name, version, + test_data_refs=test_data_refs, + test_data_tags=test_data_tags + ) + ) diff --git a/recipe_modules/cipd/example.expected/basic.json b/recipe_modules/cipd/example.expected/basic.json new file mode 100644 index 0000000000..49698b2622 --- /dev/null +++ b/recipe_modules/cipd/example.expected/basic.json @@ -0,0 +1,445 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json" + ], + "name": "install cipd", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json", + "--version", + "deadbeaf" + ], + "name": "install cipd (2)", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "public/package/linux-amd64", + "--tag", + "git_revision:40-chars-long-hash", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "public/package/linux-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd describe public/package/linux-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "private/package/linux-amd64 latest\npublic/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "ensure_installed (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "private/package/linux-amd64", + "--tag", + "key:value", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search private/package/linux-amd64 key:value", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "private/package/linux-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd describe private/package/linux-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"custom:tagged\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"key:value\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json" + ], + "name": "build fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--install-mode", + "copy" + ], + "name": "build fake-package (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-register", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "register infra/fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "create", + "--pkg-def", + "[SLAVE_BUILD]/fake-package.yaml", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "create fake-package.yaml", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output (invalid)@null@@@", + "@@@STEP_LOG_END@json.output (invalid)@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "set-tag", + "fake-package", + "--version", + "long/weird/ref/which/doesn/not/fit/into/40chars", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--tag", + "dead:beaf", + "--tag", + "more:value" + ], + "name": "cipd set-tag fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-long/weird/ref/w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "set-ref", + "fake-package", + "--version", + "latest", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "any", + "--ref", + "some" + ], + "name": "cipd set-ref fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "fake-package/linux-amd64", + "--tag", + "dead:beaf", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search fake-package/linux-amd64 dead:beaf", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/cipd/example.expected/describe-failed.json b/recipe_modules/cipd/example.expected/describe-failed.json new file mode 100644 index 0000000000..f4932176e9 --- /dev/null +++ b/recipe_modules/cipd/example.expected/describe-failed.json @@ -0,0 +1,128 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json" + ], + "name": "install cipd", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json", + "--version", + "deadbeaf" + ], + "name": "install cipd (2)", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "public/package/linux-amd64", + "--tag", + "git_revision:40-chars-long-hash", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "public/package/linux-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd describe public/package/linux-amd64", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"error\": \"package \\\"public/package/linux-amd64-ubuntu14_04\\\" not registered\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": null@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_FAILURE@@@" + ] + }, + { + "name": "$result", + "reason": "Step('cipd describe public/package/linux-amd64') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/recipe_modules/cipd/example.expected/describe-many-instances.json b/recipe_modules/cipd/example.expected/describe-many-instances.json new file mode 100644 index 0000000000..96e447ac8a --- /dev/null +++ b/recipe_modules/cipd/example.expected/describe-many-instances.json @@ -0,0 +1,453 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json" + ], + "name": "install cipd", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json", + "--version", + "deadbeaf" + ], + "name": "install cipd (2)", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "public/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "public/package/linux-amd64", + "--tag", + "git_revision:40-chars-long-hash", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd search public/package/linux-amd64 git_revision:40-chars-long-hash", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "public/package/linux-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd describe public/package/linux-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "private/package/linux-amd64 latest\npublic/package/linux-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "ensure_installed (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "private/package/linux-amd64", + "--tag", + "key:value", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search private/package/linux-amd64 key:value", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "private/package/linux-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd describe private/package/linux-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"custom:tagged\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"key:value\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json" + ], + "name": "build fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--install-mode", + "copy" + ], + "name": "build fake-package (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-register", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "register infra/fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "create", + "--pkg-def", + "[SLAVE_BUILD]/fake-package.yaml", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "create fake-package.yaml", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output (invalid)@null@@@", + "@@@STEP_LOG_END@json.output (invalid)@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "set-tag", + "fake-package", + "--version", + "long/weird/ref/which/doesn/not/fit/into/40chars", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--tag", + "dead:beaf", + "--tag", + "more:value" + ], + "name": "cipd set-tag fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-long/weird/ref/w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "set-ref", + "fake-package", + "--version", + "latest", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "any", + "--ref", + "some" + ], + "name": "cipd set-ref fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "fake-package/linux-amd64", + "--tag", + "dead:beaf", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search fake-package/linux-amd64 dead:beaf", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-instance_id_1---\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64-ubuntu14_04\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-instance_id_2---\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64-ubuntu14_04\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-instance_id_3---\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/linux-amd64-ubuntu14_04\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/cipd/example.expected/install-failed.json b/recipe_modules/cipd/example.expected/install-failed.json new file mode 100644 index 0000000000..bad2c73c3e --- /dev/null +++ b/recipe_modules/cipd/example.expected/install-failed.json @@ -0,0 +1,31 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "linux-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json" + ], + "name": "install cipd", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_FAILURE@@@" + ] + }, + { + "name": "$result", + "reason": "Step('install cipd') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/recipe_modules/cipd/example.expected/mac64.json b/recipe_modules/cipd/example.expected/mac64.json new file mode 100644 index 0000000000..eff6e36239 --- /dev/null +++ b/recipe_modules/cipd/example.expected/mac64.json @@ -0,0 +1,445 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "mac-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json" + ], + "name": "install cipd", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py", + "--platform", + "mac-amd64", + "--dest-directory", + "[SLAVE_BUILD]/cipd", + "--json-output", + "/path/to/tmp/json", + "--version", + "deadbeaf" + ], + "name": "install cipd (2)", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "public/package/mac-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "public/package/mac-amd64", + "--tag", + "git_revision:40-chars-long-hash", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd search public/package/mac-amd64 git_revision:40-chars-long-hash", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "public/package/mac-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "/creds/service_accounts/service-account-cipd-builder.json" + ], + "name": "cipd describe public/package/mac-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "ensure", + "--root", + "[SLAVE_BUILD]/packages", + "--list", + "private/package/mac-amd64 latest\npublic/package/mac-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "ensure_installed (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "private/package/mac-amd64", + "--tag", + "key:value", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search private/package/mac-amd64 key:value", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "describe", + "private/package/mac-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd describe private/package/mac-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"custom:tagged\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"key:value\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json" + ], + "name": "build fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--install-mode", + "copy" + ], + "name": "build fake-package (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "pkg-register", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "register infra/fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "create", + "--pkg-def", + "[SLAVE_BUILD]/fake-package.yaml", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "create fake-package.yaml", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output (invalid)@null@@@", + "@@@STEP_LOG_END@json.output (invalid)@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "set-tag", + "fake-package", + "--version", + "long/weird/ref/which/doesn/not/fit/into/40chars", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--tag", + "dead:beaf", + "--tag", + "more:value" + ], + "name": "cipd set-tag fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-long/weird/ref/w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "set-ref", + "fake-package", + "--version", + "latest", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "any", + "--ref", + "some" + ], + "name": "cipd set-ref fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/cipd/cipd", + "search", + "fake-package/mac-amd64", + "--tag", + "dead:beaf", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search fake-package/mac-amd64 dead:beaf", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package/mac-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/cipd/example.expected/win64.json b/recipe_modules/cipd/example.expected/win64.json new file mode 100644 index 0000000000..ebfdb9cda1 --- /dev/null +++ b/recipe_modules/cipd/example.expected/win64.json @@ -0,0 +1,445 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]\\resources\\bootstrap.py", + "--platform", + "windows-amd64", + "--dest-directory", + "[SLAVE_BUILD]\\cipd", + "--json-output", + "/path/to/tmp/json" + ], + "name": "install cipd", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]\\\\cipd\\\\cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::cipd]\\resources\\bootstrap.py", + "--platform", + "windows-amd64", + "--dest-directory", + "[SLAVE_BUILD]\\cipd", + "--json-output", + "/path/to/tmp/json", + "--version", + "deadbeaf" + ], + "name": "install cipd (2)", + "~followup_annotations": [ + "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"executable\": \"[SLAVE_BUILD]\\\\cipd\\\\cipd\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "ensure", + "--root", + "[SLAVE_BUILD]\\packages", + "--list", + "public/package/windows-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "C:\\creds\\service_accounts\\service-account-cipd-builder.json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "search", + "public/package/windows-amd64", + "--tag", + "git_revision:40-chars-long-hash", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "C:\\creds\\service_accounts\\service-account-cipd-builder.json" + ], + "name": "cipd search public/package/windows-amd64 git_revision:40-chars-long-hash", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "describe", + "public/package/windows-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "C:\\creds\\service_accounts\\service-account-cipd-builder.json" + ], + "name": "cipd describe public/package/windows-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"buildbot_build:some.waterfall/builder/1234\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_repository:https://chromium.googlesource.com/some/repo\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"git_revision:397a2597cdc237f3026e6143b683be4b9ab60540\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "ensure", + "--root", + "[SLAVE_BUILD]\\packages", + "--list", + "private/package/windows-amd64 latest\npublic/package/windows-amd64 7f751b2237df2fdf3c1405be00590fefffbaea2d", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "ensure_installed (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"7f751b2237df2fdf3c1405be00590fefffbaea2d\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"public/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "search", + "private/package/windows-amd64", + "--tag", + "key:value", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search private/package/windows-amd64 key:value", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "describe", + "private/package/windows-amd64", + "--version", + "40-chars-fake-of-the-package-instance_id", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd describe private/package/windows-amd64", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"private/package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"refs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ref\": \"latest\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"custom:tagged\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_by\": \"user:44-blablbla@developer.gserviceaccount.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"registered_ts\": 1446574210, @@@", + "@@@STEP_LOG_LINE@json.output@ \"tag\": \"key:value\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json" + ], + "name": "build fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "pkg-build", + "--in", + "fake-input-dir", + "--name", + "infra/fake-package", + "--out", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--install-mode", + "copy" + ], + "name": "build fake-package (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "pkg-register", + "fake-package-path", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "register infra/fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "create", + "--pkg-def", + "[SLAVE_BUILD]\\fake-package.yaml", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "fake-ref-1", + "--ref", + "fake-ref-2", + "--tag", + "fake_tag_1:fake_value_1", + "--tag", + "fake_tag_2:fake_value_2" + ], + "name": "create fake-package.yaml", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output (invalid)@null@@@", + "@@@STEP_LOG_END@json.output (invalid)@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "set-tag", + "fake-package", + "--version", + "long/weird/ref/which/doesn/not/fit/into/40chars", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--tag", + "dead:beaf", + "--tag", + "more:value" + ], + "name": "cipd set-tag fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-long/weird/ref/w\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "set-ref", + "fake-package", + "--version", + "latest", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json", + "--ref", + "any", + "--ref", + "some" + ], + "name": "cipd set-ref fake-package", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"pin\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]\\cipd\\cipd", + "search", + "fake-package/windows-amd64", + "--tag", + "dead:beaf", + "--json-output", + "/path/to/tmp/json", + "--service-account-json", + "fake-credentials.json" + ], + "name": "cipd search fake-package/windows-amd64 dead:beaf", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fake-package/windows-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/cipd/example.py b/recipe_modules/cipd/example.py new file mode 100644 index 0000000000..2629bd7006 --- /dev/null +++ b/recipe_modules/cipd/example.py @@ -0,0 +1,115 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/properties', + 'recipe_engine/step', + 'cipd', +] + +def RunSteps(api): + # First, you need a cipd client. + api.cipd.install_client('install cipd') + api.cipd.install_client('install cipd', version='deadbeaf') + assert api.cipd.get_executable() + + # Need to set service account credentials. + api.cipd.set_service_account_credentials( + api.cipd.default_bot_service_account_credentials) + + package_name = 'public/package/%s' % api.cipd.platform_suffix() + package_instance_id = '7f751b2237df2fdf3c1405be00590fefffbaea2d' + packages = {package_name: package_instance_id} + + cipd_root = api.path['slave_build'].join('packages') + # Some packages don't require credentials to be installed or queried. + api.cipd.ensure(cipd_root, packages) + step = api.cipd.search(package_name, tag='git_revision:40-chars-long-hash') + api.cipd.describe(package_name, + version=step.json.output['result'][0]['instance_id']) + + # Others do, so provide creds first. + api.cipd.set_service_account_credentials('fake-credentials.json') + private_package_name = 'private/package/%s' % api.cipd.platform_suffix() + packages[private_package_name] = 'latest' + api.cipd.ensure(cipd_root, packages) + step = api.cipd.search(private_package_name, tag='key:value') + api.cipd.describe(private_package_name, + version=step.json.output['result'][0]['instance_id'], + test_data_tags=['custom:tagged', 'key:value'], + test_data_refs=['latest']) + + # The rest of commands expect credentials to be set. + + # Build & register new package version. + api.cipd.build('fake-input-dir', 'fake-package-path', 'infra/fake-package') + api.cipd.build('fake-input-dir', 'fake-package-path', 'infra/fake-package', + install_mode='copy') + api.cipd.register('infra/fake-package', 'fake-package-path', + refs=['fake-ref-1', 'fake-ref-2'], + tags={'fake_tag_1': 'fake_value_1', + 'fake_tag_2': 'fake_value_2'}) + + # Create (build & register). + api.cipd.create(api.path['slave_build'].join('fake-package.yaml'), + refs=['fake-ref-1', 'fake-ref-2'], + tags={'fake_tag_1': 'fake_value_1', + 'fake_tag_2': 'fake_value_2'}) + + # Set tag or ref of an already existing package. + api.cipd.set_tag('fake-package', + version='long/weird/ref/which/doesn/not/fit/into/40chars', + tags={'dead': 'beaf', 'more': 'value'}) + api.cipd.set_ref('fake-package', version='latest', refs=['any', 'some']) + # Search by the new tag. + api.cipd.search('fake-package/%s' % api.cipd.platform_suffix(), + tag='dead:beaf') + + +def GenTests(api): + yield ( + # This is very common dev workstation, but not all devs are on it. + api.test('basic') + + api.platform('linux', 64) + ) + + yield ( + api.test('mac64') + + api.platform('mac', 64) + ) + + yield ( + api.test('win64') + + api.platform('win', 64) + ) + + yield ( + api.test('install-failed') + + api.step_data('install cipd', retcode=1) + ) + + yield ( + api.test('describe-failed') + + api.platform('linux', 64) + + api.override_step_data( + 'cipd describe public/package/linux-amd64', + api.cipd.example_error( + 'package "public/package/linux-amd64-ubuntu14_04" not registered', + ) + ) + ) + + yield ( + api.test('describe-many-instances') + + api.platform('linux', 64) + + api.override_step_data( + 'cipd search fake-package/linux-amd64 dead:beaf', + api.cipd.example_search( + 'public/package/linux-amd64-ubuntu14_04', + instances=3 + ) + ) + ) diff --git a/recipe_modules/cipd/resources/bootstrap.py b/recipe_modules/cipd/resources/bootstrap.py new file mode 100644 index 0000000000..75df3eb7ea --- /dev/null +++ b/recipe_modules/cipd/resources/bootstrap.py @@ -0,0 +1,218 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import errno +import hashlib +import json +import os +import sys +import tempfile +import time +import traceback +import urllib +import urllib2 + + +# Default package repository URL. +CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com' + +# ./cipd resolve \ +# infra/tools/cipd/ \ +# -version=git_revision:768586a9aa72fe6a41a8a0205ed56ceb1495625c +CLIENT_VERSIONS = { + 'linux-386': '3f5d6d3906c3b2c6a057d1ab1634ac06aa418708', + 'linux-amd64': '1ea0b6b254ad3f546c826dd3e437798ace2c2480', + 'mac-amd64': 'c111be343c692e5285113a6b1c999887adbb268e', + 'windows-386': 'dc3d1bd5b4b93945640bac4bb047c333a8fa12fd', + 'windows-amd64': 'bccdb9a605037e3dd2a8a64e79e08f691a6f159d', +} + + +class CipdBootstrapError(Exception): + """Raised by install_cipd_client on fatal error.""" + +def install_cipd_client(path, package, version): + """Installs CIPD client to /cipd. + + Args: + path: root directory to install CIPD client into. + package: cipd client package name, e.g. infra/tools/cipd/linux-amd64. + version: version of the package to install. + + Returns: + Absolute path to CIPD executable. + """ + print 'Ensuring CIPD client is up-to-date' + version_file = os.path.join(path, 'VERSION') + bin_file = os.path.join(path, 'cipd') + + # Resolve version to concrete instance ID, e.g "live" -> "abcdef0123....". + instance_id = call_cipd_api( + 'repo/v1/instance/resolve', + {'package_name': package, 'version': version})['instance_id'] + print 'CIPD client %s => %s' % (version, instance_id) + + # Already installed? + installed_instance_id = (read_file(version_file) or '').strip() + if installed_instance_id == instance_id and os.path.exists(bin_file): + return bin_file, instance_id + + # Resolve instance ID to an URL to fetch client binary from. + client_info = call_cipd_api( + 'repo/v1/client', + {'package_name': package, 'instance_id': instance_id}) + print 'CIPD client binary info:\n%s' % dump_json(client_info) + + # Fetch the client. It is ~10 MB, so don't bother and fetch it into memory. + status, raw_client_bin = fetch_url(client_info['client_binary']['fetch_url']) + if status != 200: + print 'Failed to fetch client binary, HTTP %d' % status + raise CipdBootstrapError('Failed to fetch client binary, HTTP %d' % status) + digest = hashlib.sha1(raw_client_bin).hexdigest() + if digest != client_info['client_binary']['sha1']: + raise CipdBootstrapError('Client SHA1 mismatch') + + # Success. + print 'Fetched CIPD client %s:%s at %s' % (package, instance_id, bin_file) + write_file(bin_file, raw_client_bin) + os.chmod(bin_file, 0755) + write_file(version_file, instance_id + '\n') + return bin_file, instance_id + + +def call_cipd_api(endpoint, query): + """Sends GET request to CIPD backend, parses JSON response.""" + url = '%s/_ah/api/%s' % (CIPD_BACKEND_URL, endpoint) + if query: + url += '?' + urllib.urlencode(query) + status, body = fetch_url(url) + if status != 200: + raise CipdBootstrapError('Server replied with HTTP %d' % status) + try: + body = json.loads(body) + except ValueError: + raise CipdBootstrapError('Server returned invalid JSON') + status = body.get('status') + if status != 'SUCCESS': + m = body.get('error_message') or '' + raise CipdBootstrapError('Server replied with error %s: %s' % (status, m)) + return body + + +def fetch_url(url, headers=None): + """Sends GET request (with retries). + + Args: + url: URL to fetch. + headers: dict with request headers. + + Returns: + (200, reply body) on success. + (HTTP code, None) on HTTP 401, 403, or 404 reply. + + Raises: + Whatever urllib2 raises. + """ + req = urllib2.Request(url) + req.add_header('User-Agent', 'cipd recipe bootstrap.py') + for k, v in (headers or {}).iteritems(): + req.add_header(str(k), str(v)) + i = 0 + while True: + i += 1 + try: + print 'GET %s' % url + return 200, urllib2.urlopen(req, timeout=60).read() + except Exception as e: + if isinstance(e, urllib2.HTTPError): + print 'Failed to fetch %s, server returned HTTP %d' % (url, e.code) + if e.code in (401, 403, 404): + return e.code, None + else: + print 'Failed to fetch %s' % url + if i == 20: + raise + print 'Retrying in %d sec.' % i + time.sleep(i) + + +def ensure_directory(path): + """Creates a directory.""" + # Handle a case where a file is being converted into a directory. + chunks = path.split(os.sep) + for i in xrange(len(chunks)): + p = os.sep.join(chunks[:i+1]) + if os.path.exists(p) and not os.path.isdir(p): + os.remove(p) + break + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def read_file(path): + """Returns contents of a file or None if missing.""" + try: + with open(path, 'r') as f: + return f.read() + except IOError as e: + if e.errno == errno.ENOENT: + return None + raise + + +def write_file(path, data): + """Puts a file on disk, atomically.""" + ensure_directory(os.path.dirname(path)) + fd, temp_file = tempfile.mkstemp(dir=os.path.dirname(path)) + with os.fdopen(fd, 'w') as f: + f.write(data) + if not sys.platform in ('linux2', 'darwin'): + # On windows we should remove destination file if it exists. + if os.path.exists(path): + os.remove(path) + # At this point we may crash, and it's OK, as next time we'll just + # re-install CIPD from scratch. + os.rename(temp_file, path) + + +def dump_json(obj): + """Pretty-formats object to JSON.""" + return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':')) + + +def main(args): + parser = argparse.ArgumentParser('bootstrap cipd') + parser.add_argument('--json-output', default=None) + parser.add_argument('--version', default=None) + parser.add_argument('--platform', required=True) + parser.add_argument('--dest-directory', required=True) + opts = parser.parse_args(args) + + package = "infra/tools/cipd/%s" % opts.platform + version = opts.version or CLIENT_VERSIONS[opts.platform] + + try: + exe_path, instance_id = install_cipd_client(opts.dest_directory, + package, version) + result = { + 'executable': exe_path, + 'instance_id': instance_id + } + if opts.json_output: + with open(opts.json_output, 'w') as f: + json.dump(result, f) + except Exception as e: + print 'Exception installing cipd: %s' % e + _exc_type, _exc_value, exc_traceback = sys.exc_info() + traceback.print_tb(exc_traceback) + return 1 + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/recipe_modules/cipd/test_api.py b/recipe_modules/cipd/test_api.py new file mode 100644 index 0000000000..6316535926 --- /dev/null +++ b/recipe_modules/cipd/test_api.py @@ -0,0 +1,102 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine import recipe_test_api + + +class CIPDTestApi(recipe_test_api.RecipeTestApi): + def make_resolved_version(self, v): + if not v: + return '40-chars-fake-of-the-package-instance_id' + if len(v) == 40: + return v + # Truncate or pad to 40 chars. + prefix = 'resolved-instance_id-of-' + if len(v) + len(prefix) >= 40: + return '%s%s' % (prefix, v[:40-len(prefix)]) + return '%s%s%s' % (prefix, v, '-' * (40 - len(prefix) - len(v))) + + def make_pin(self, package_name, version=None): + return { + 'package': package_name, + 'instance_id': self.make_resolved_version(version), + } + + def _resultify(self, result, error=None, retcode=None): + dic = {'result': result} + if error: + dic['error'] = error + return self.m.json.output(dic, retcode=retcode) + + def make_test_executable(self): + return str(self.m.path['slave_build'].join('cipd', 'cipd')) + + def example_error(self, error, retcode=None): + return self._resultify( + result=None, + error=error, + retcode=1 if retcode is None else retcode) + + def example_install_client(self, package_name, version=None, retcode=None): + return self.m.json.output({ + 'executable': self.make_test_executable(), + 'instance_id': self.make_resolved_version(version), + }, retcode=retcode) + + def example_build(self, package_name, version=None): + return self._resultify(self.make_pin(package_name, version)) + + example_register = example_build + + def example_ensure(self, packages): + return self._resultify([self.make_pin(name, version) + for name, version in sorted(packages.items())]) + + def example_set_tag(self, package_name, version): + return self._resultify({ + 'package': package_name, + 'pin': self.make_pin(package_name, version) + }) + + example_set_ref = example_set_tag + + def example_search(self, package_name, instances=None): + if instances is None: + # Return one instance by default. + return self._resultify([self.make_pin(package_name)]) + if isinstance(instances, int): + instances = ['instance_id_%i' % (i+1) for i in xrange(instances)] + return self._resultify([self.make_pin(package_name, instance) + for instance in instances]) + + def example_describe(self, package_name, version=None, + test_data_refs=None, test_data_tags=None, + user='user:44-blablbla@developer.gserviceaccount.com', + tstamp=1446574210): + assert not test_data_tags or all(':' in tag for tag in test_data_tags) + return self._resultify({ + 'pin': self.make_pin(package_name, version), + 'registered_by': user, + 'registered_ts': tstamp, + 'refs': [ + { + 'ref': ref, + 'modified_by': user, + 'modified_ts': tstamp, + } + for ref in (['latest'] if test_data_refs is None else test_data_refs) + ], + 'tags': [ + { + 'tag': tag, + 'registered_by': user, + 'registered_ts': tstamp, + } + for tag in ([ + 'buildbot_build:some.waterfall/builder/1234', + 'git_repository:https://chromium.googlesource.com/some/repo', + 'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540', + ] if test_data_tags is None else test_data_tags) + ], + })