diff --git a/recipe_modules/bot_update/api.py b/recipe_modules/bot_update/api.py index 4b870a4b7..610872e5f 100644 --- a/recipe_modules/bot_update/api.py +++ b/recipe_modules/bot_update/api.py @@ -8,12 +8,6 @@ from recipe_engine import recipe_api -# This is just for testing, to indicate if a master is using a Git scheduler -# or not. -SVN_MASTERS = ( - 'experimental.svn', -) - def jsonish_to_python(spec, is_top=False): """Turn a json spec into a python parsable object. @@ -67,9 +61,10 @@ class BotUpdateApi(recipe_api.RecipeApi): def properties(self): return self._properties + # Note: force is ignored. def ensure_checkout(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, - force=False, patch_root=None, no_shallow=False, + force=True, patch_root=None, no_shallow=False, with_branch_heads=False, refs=None, patch_project_roots=None, patch_oauth2=False, output_manifest=True, clobber=False, @@ -80,11 +75,6 @@ class BotUpdateApi(recipe_api.RecipeApi): cfg = gclient_config or self.m.gclient.c spec_string = jsonish_to_python(cfg.as_jsonish(), True) - # Used by bot_update to determine if we want to run or not. - master = self.m.properties['mastername'] - builder = self.m.properties['buildername'] - slave = self.m.properties['slavename'] - # Construct our bot_update command. This basically be inclusive of # everything required for bot_update to know: root = patch_root @@ -136,18 +126,13 @@ class BotUpdateApi(recipe_api.RecipeApi): rev_map = self.m.gclient.c.got_revision_mapping.as_jsonish() flags = [ - # 1. Do we want to run? (master/builder/slave). - ['--master', master], - ['--builder', builder], - ['--slave', slave], - - # 2. What do we want to check out (spec/root/rev/rev_map). + # 1. What do we want to check out (spec/root/rev/rev_map). ['--spec', spec_string], ['--root', root], ['--revision_mapping_file', self.m.json.input(rev_map)], ['--git-cache-dir', self.m.path['git_cache']], - # 3. How to find the patch, if any (issue/patchset/patch_url). + # 2. How to find the patch, if any (issue/patchset/patch_url). ['--issue', issue], ['--patchset', patchset], ['--patch_url', patch_url], @@ -157,7 +142,7 @@ class BotUpdateApi(recipe_api.RecipeApi): ['--apply_issue_email_file', email_file], ['--apply_issue_key_file', key_file], - # 4. Hookups to JSON output back into recipes. + # 3. Hookups to JSON output back into recipes. ['--output_json', self.m.json.output()],] @@ -197,8 +182,6 @@ class BotUpdateApi(recipe_api.RecipeApi): if clobber: cmd.append('--clobber') - if force: - cmd.append('--force') if no_shallow: cmd.append('--no_shallow') if output_manifest: @@ -207,11 +190,9 @@ class BotUpdateApi(recipe_api.RecipeApi): cmd.append('--with_branch_heads') # Inject Json output for testing. - git_mode = self.m.properties.get('mastername') not in SVN_MASTERS first_sln = cfg.solutions[0].name step_test_data = lambda: self.test_api.output_json( - master, builder, slave, root, first_sln, rev_map, git_mode, force, - self.m.properties.get('fail_patch', False), + root, first_sln, rev_map, self.m.properties.get('fail_patch', False), output_manifest=output_manifest, fixed_revisions=fixed_revisions) # Add suffixes to the step name, if specified. diff --git a/recipe_modules/bot_update/example.expected/basic.json b/recipe_modules/bot_update/example.expected/basic.json index 0cff6c0fe..0ad0c943f 100644 --- a/recipe_modules/bot_update/example.expected/basic.json +++ b/recipe_modules/bot_update/example.expected/basic.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "chromium.linux", - "--builder", - "Linux Builder", - "--slave", - "totallyaslave-m1", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/basic_output_manifest.json b/recipe_modules/bot_update/example.expected/basic_output_manifest.json index 170b4ab6d..6dbca9a27 100644 --- a/recipe_modules/bot_update/example.expected/basic_output_manifest.json +++ b/recipe_modules/bot_update/example.expected/basic_output_manifest.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "chromium.linux", - "--builder", - "Linux Builder", - "--slave", - "totallyaslave-m1", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json b/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json index 214604319..67d212db9 100644 --- a/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json +++ b/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "chromium.linux", - "--builder", - "Linux Builder", - "--slave", - "totallyaslave-m1", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/clobber.json b/recipe_modules/bot_update/example.expected/clobber.json index ee6f935c9..2adc26fdd 100644 --- a/recipe_modules/bot_update/example.expected/clobber.json +++ b/recipe_modules/bot_update/example.expected/clobber.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "experimental", - "--builder", - "Experimental Builder", - "--slave", - "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -30,11 +24,24 @@ }, "name": "bot_update", "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@" + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" ] }, { diff --git a/recipe_modules/bot_update/example.expected/forced.json b/recipe_modules/bot_update/example.expected/forced.json index 07b1af682..a043d5a12 100644 --- a/recipe_modules/bot_update/example.expected/forced.json +++ b/recipe_modules/bot_update/example.expected/forced.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "experimental", - "--builder", - "Experimental Builder", - "--slave", - "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -21,8 +15,7 @@ "--output_json", "/path/to/tmp/json", "--revision", - "src@HEAD", - "--force" + "src@HEAD" ], "cwd": "[SLAVE_BUILD]", "env": { diff --git a/recipe_modules/bot_update/example.expected/no_shallow.json b/recipe_modules/bot_update/example.expected/no_shallow.json index 2008a4fa3..18abe3181 100644 --- a/recipe_modules/bot_update/example.expected/no_shallow.json +++ b/recipe_modules/bot_update/example.expected/no_shallow.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "experimental", - "--builder", - "Experimental Builder", - "--slave", - "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -30,11 +24,24 @@ }, "name": "bot_update", "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@" + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" ] }, { diff --git a/recipe_modules/bot_update/example.expected/off.json b/recipe_modules/bot_update/example.expected/off.json deleted file mode 100644 index 48109b0ba..000000000 --- a/recipe_modules/bot_update/example.expected/off.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "experimental", - "--builder", - "Experimental Builder", - "--slave", - "somehost", - "--spec", - "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", - "--root", - "src", - "--revision_mapping_file", - "{\"src\": \"got_cr_revision\"}", - "--git-cache-dir", - "[GIT_CACHE]", - "--output_json", - "/path/to/tmp/json", - "--revision", - "src@HEAD" - ], - "cwd": "[SLAVE_BUILD]", - "env": { - "PATH": "%(PATH)s:RECIPE_PACKAGE[depot_tools]" - }, - "name": "bot_update", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", - "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -] \ No newline at end of file diff --git a/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json b/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json index e50485c41..69370788f 100644 --- a/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json +++ b/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "experimental", - "--builder", - "Experimental Builder", - "--slave", - "somehost", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", @@ -29,11 +23,24 @@ }, "name": "bot_update", "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src\": \"revision\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@" + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", + "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" ] }, { diff --git a/recipe_modules/bot_update/example.expected/svn_mode.json b/recipe_modules/bot_update/example.expected/svn_mode.json deleted file mode 100644 index b12f9e40c..000000000 --- a/recipe_modules/bot_update/example.expected/svn_mode.json +++ /dev/null @@ -1,60 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "experimental.svn", - "--builder", - "Experimental SVN Builder", - "--slave", - "somehost", - "--spec", - "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", - "--root", - "src", - "--revision_mapping_file", - "{\"src\": \"got_cr_revision\"}", - "--git-cache-dir", - "[GIT_CACHE]", - "--output_json", - "/path/to/tmp/json", - "--revision", - "src@HEAD", - "--force" - ], - "cwd": "[SLAVE_BUILD]", - "env": { - "PATH": "%(PATH)s:RECIPE_PACKAGE[depot_tools]" - }, - "name": "bot_update", - "~followup_annotations": [ - "@@@STEP_TEXT@Some step text@@@", - "@@@STEP_LOG_LINE@json.output@{@@@", - "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", - "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"src\": \"HEAD\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", - "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision\": 170242, @@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_cp\": \"refs/heads/master@{#170242}\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"got_cr_revision_git\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", - "@@@STEP_LOG_LINE@json.output@ }, @@@", - "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", - "@@@STEP_LOG_LINE@json.output@}@@@", - "@@@STEP_LOG_END@json.output@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision@170242@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision_git@\"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", - "@@@SET_BUILD_PROPERTY@got_cr_revision_cp@\"refs/heads/master@{#170242}\"@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -] \ No newline at end of file diff --git a/recipe_modules/bot_update/example.expected/trychange.json b/recipe_modules/bot_update/example.expected/trychange.json index a7aa47328..a4c0f646c 100644 --- a/recipe_modules/bot_update/example.expected/trychange.json +++ b/recipe_modules/bot_update/example.expected/trychange.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/trychange_oauth2.json b/recipe_modules/bot_update/example.expected/trychange_oauth2.json index c81ede5ff..f364c3bc8 100644 --- a/recipe_modules/bot_update/example.expected/trychange_oauth2.json +++ b/recipe_modules/bot_update/example.expected/trychange_oauth2.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob.json b/recipe_modules/bot_update/example.expected/tryjob.json index f34e0fe7e..7d2f32a58 100644 --- a/recipe_modules/bot_update/example.expected/tryjob.json +++ b/recipe_modules/bot_update/example.expected/tryjob.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_fail.json b/recipe_modules/bot_update/example.expected/tryjob_fail.json index 66b377c29..b8979be0b 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_fail.json +++ b/recipe_modules/bot_update/example.expected/tryjob_fail.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json b/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json index 9e2cd6f8d..b1163786b 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json +++ b/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json b/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json index e7a93bdf1..6c38ce865 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json +++ b/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.expected/tryjob_v8.json b/recipe_modules/bot_update/example.expected/tryjob_v8.json index f426d8b95..6d4d75afb 100644 --- a/recipe_modules/bot_update/example.expected/tryjob_v8.json +++ b/recipe_modules/bot_update/example.expected/tryjob_v8.json @@ -4,12 +4,6 @@ "python", "-u", "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", - "--master", - "tryserver.chromium.linux", - "--builder", - "linux_rel", - "--slave", - "totallyaslave-c4", "--spec", "cache_dir = None\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]", "--root", diff --git a/recipe_modules/bot_update/example.py b/recipe_modules/bot_update/example.py index d866c0ceb..3e512e774 100644 --- a/recipe_modules/bot_update/example.py +++ b/recipe_modules/bot_update/example.py @@ -43,110 +43,57 @@ def RunSteps(api): def GenTests(api): yield api.test('basic') + api.properties( - mastername='chromium.linux', - buildername='Linux Builder', - slavename='totallyaslave-m1', patch=False, revision='abc' ) yield api.test('basic_with_branch_heads') + api.properties( - mastername='chromium.linux', - buildername='Linux Builder', - slavename='totallyaslave-m1', with_branch_heads=True, suffix='with branch heads' ) yield api.test('basic_output_manifest') + api.properties( - mastername='chromium.linux', - buildername='Linux Builder', - slavename='totallyaslave-m1', output_manifest=True, ) yield api.test('tryjob') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar' ) yield api.test('trychange') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', refs=['+refs/change/1/2/333'], ) yield api.test('trychange_oauth2') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', oauth2=True, ) yield api.test('tryjob_fail') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', ) + api.step_data('bot_update', retcode=1) yield api.test('tryjob_fail_patch') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', fail_patch='apply', ) + api.step_data('bot_update', retcode=88) yield api.test('tryjob_fail_patch_download') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', fail_patch='download' ) + api.step_data('bot_update', retcode=87) yield api.test('forced') + api.properties( - mastername='experimental', - buildername='Experimental Builder', - slavename='somehost', force=1 ) yield api.test('no_shallow') + api.properties( - mastername='experimental', - buildername='Experimental Builder', - slavename='somehost', no_shallow=1 ) - yield api.test('off') + api.properties( - mastername='experimental', - buildername='Experimental Builder', - slavename='somehost', - ) - yield api.test('svn_mode') + api.properties( - mastername='experimental.svn', - buildername='Experimental SVN Builder', - slavename='somehost', - force=1 - ) yield api.test('clobber') + api.properties( - mastername='experimental', - buildername='Experimental Builder', - slavename='somehost', clobber=1 ) yield api.test('reset_root_solution_revision') + api.properties( - mastername='experimental', - buildername='Experimental Builder', - slavename='somehost', root_solution_revision='revision', ) yield api.test('tryjob_v8') + api.properties( - mastername='tryserver.chromium.linux', - buildername='linux_rel', - slavename='totallyaslave-c4', issue=12345, patchset=654321, patch_url='http://src.chromium.org/foo/bar', diff --git a/recipe_modules/bot_update/resources/bot_update.py b/recipe_modules/bot_update/resources/bot_update.py index 39c4bf61d..4ecbf6597 100755 --- a/recipe_modules/bot_update/resources/bot_update.py +++ b/recipe_modules/bot_update/resources/bot_update.py @@ -73,19 +73,18 @@ ROOT_DIR = path.dirname(BUILD_DIR) DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) -BUILD_INTERNAL_DIR = check_dir( - 'build_internal', [ - path.join(ROOT_DIR, 'build_internal'), - path.join(ROOT_DIR, # .recipe_deps - path.pardir, # slave - path.pardir, # scripts - path.pardir), # build_internal - ]) - - CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' +RECOGNIZED_PATHS = { + '/chrome/trunk/src': + CHROMIUM_SRC_URL, + '/chrome/trunk/src/tools/cros.DEPS': + CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git', + '/chrome-internal/trunk/src-internal': + 'https://chrome-internal.googlesource.com/chrome/src-internal.git', +} + # Official builds use buildspecs, so this is a special case. BUILDSPEC_TYPE = collections.namedtuple('buildspec', ('container', 'version')) @@ -168,39 +167,8 @@ GOT_REVISION_MAPPINGS = { BOT_UPDATE_MESSAGE = """ -What is the "Bot Update" step? -============================== - -This step ensures that the source checkout on the bot (e.g. Chromium's src/ and -its dependencies) is checked out in a consistent state. This means that all of -the necessary repositories are checked out, no extra repositories are checked -out, and no locally modified files are present. - -These actions used to be taken care of by the "gclient revert" and "update" -steps. However, those steps are known to be buggy and occasionally flaky. This -step has two main advantages over them: - * it only operates in Git, so the logic can be clearer and cleaner; and - * it is a slave-side script, so its behavior can be modified without - restarting the master. - -Why Git, you ask? Because that is the direction that the Chromium project is -heading. This step is an integral part of the transition from using the SVN repo -at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while -we fully convert everything to Git. This message will get out of your way -eventually, and the waterfall will be a happier place because of it. - -This step can be activated or deactivated independently on every builder on -every master. When it is active, the "gclient revert" and "update" steps become -no-ops. When it is inactive, it prints this message, cleans up after itself, and -lets everything else continue as though nothing has changed. Eventually, when -everything is stable enough, this step will replace them entirely. - -Debugging information: +Bot Update Debugging information: (master/builder/slave may be unspecified on recipes) -master: %(master)s -builder: %(builder)s -slave: %(slave)s -forced by recipes: %(recipe)s CURRENT_DIR: %(CURRENT_DIR)s BUILDER_DIR: %(BUILDER_DIR)s SLAVE_DIR: %(SLAVE_DIR)s @@ -208,20 +176,7 @@ THIS_DIR: %(THIS_DIR)s SCRIPTS_DIR: %(SCRIPTS_DIR)s BUILD_DIR: %(BUILD_DIR)s ROOT_DIR: %(ROOT_DIR)s -DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s -bot_update.py is:""" - -ACTIVATED_MESSAGE = """ACTIVE. -The bot will perform a Git checkout in this step. -The "gclient revert" and "update" steps are no-ops. - -""" - -NOT_ACTIVATED_MESSAGE = """INACTIVE. -This step does nothing. You actually want to look at the "update" step. - -""" - +DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s""" GCLIENT_TEMPLATE = """solutions = %(solutions)s @@ -231,137 +186,14 @@ cache_dir = r%(cache_dir)s """ -internal_data = {} -if BUILD_INTERNAL_DIR: - local_vars = {} - try: - execfile(os.path.join( - BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'), - local_vars) - except Exception: - # Same as if BUILD_INTERNAL_DIR didn't exist in the first place. - print 'Warning: unable to read internal configuration file.' - print 'If this is an internal bot, this step may be erroneously inactive.' - internal_data = local_vars - -RECOGNIZED_PATHS = { - # If SVN path matches key, the entire URL is rewritten to the Git url. - '/chrome/trunk/src': - CHROMIUM_SRC_URL, - '/chrome/trunk/src/tools/cros.DEPS': - CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git', -} -RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {})) - -ENABLED_MASTERS = [ - 'bot_update.always_on', - 'chromium.android', - 'chromium.angle', - 'chromium.chrome', - 'chromium.chromedriver', - 'chromium.chromiumos', - 'chromium', - 'chromium.fyi', - 'chromium.goma', - 'chromium.gpu', - 'chromium.gpu.fyi', - 'chromium.infra', - 'chromium.infra.cron', - 'chromium.linux', - 'chromium.lkgr', - 'chromium.mac', - 'chromium.memory', - 'chromium.memory.fyi', - 'chromium.perf', - 'chromium.perf.fyi', - 'chromium.swarm', - 'chromium.webkit', - 'chromium.webrtc', - 'chromium.webrtc.fyi', - 'chromium.win', - 'client.catapult', - 'client.drmemory', - 'client.mojo', - 'client.nacl', - 'client.nacl.ports', - 'client.nacl.sdk', - 'client.nacl.toolchain', - 'client.pdfium', - 'client.skia', - 'client.skia.fyi', - 'client.v8', - 'client.v8.branches', - 'client.v8.fyi', - 'client.webrtc', - 'client.webrtc.fyi', - 'tryserver.blink', - 'tryserver.client.catapult', - 'tryserver.client.mojo', - 'tryserver.chromium.android', - 'tryserver.chromium.angle', - 'tryserver.chromium.linux', - 'tryserver.chromium.mac', - 'tryserver.chromium.perf', - 'tryserver.chromium.win', - 'tryserver.infra', - 'tryserver.nacl', - 'tryserver.v8', - 'tryserver.webrtc', -] -ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', []) - -ENABLED_BUILDERS = { - 'client.dart.fyi': [ - 'v8-linux-release', - 'v8-mac-release', - 'v8-win-release', - ], - 'client.dynamorio': [ - 'linux-v8-dr', - ], -} -ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {})) - -ENABLED_SLAVES = {} -ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {})) - -# Disabled filters get run AFTER enabled filters, so for example if a builder -# config is enabled, but a bot on that builder is disabled, that bot will -# be disabled. -DISABLED_BUILDERS = {} -DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) - -DISABLED_SLAVES = {} -DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) - -# These masters work only in Git, meaning for got_revision, always output -# a git hash rather than a SVN rev. -GIT_MASTERS = [ - 'client.v8', - 'client.v8.branches', - 'tryserver.v8', -] -GIT_MASTERS += internal_data.get('GIT_MASTERS', []) - - # How many times to try before giving up. ATTEMPTS = 5 -# Find deps2git -DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') -DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') -S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', - 'svn_to_git_internal.py') GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') # Find the patch tool. if sys.platform.startswith('win'): - if not BUILD_INTERNAL_DIR: - print 'Warning: could not find patch tool because there is no ' - print 'build_internal present.' - PATCH_TOOL = None - else: - PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE') + PATCH_TOOL = path.join(THIS_DIR, 'patch.EXE') else: PATCH_TOOL = '/usr/bin/patch' @@ -393,11 +225,6 @@ class InvalidDiff(Exception): pass -class Inactive(Exception): - """Not really an exception, just used to exit early cleanly.""" - pass - - RETRY = object() OK = object() FAIL = object() @@ -526,34 +353,6 @@ def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir): } -def check_enabled(master, builder, slave): - if master in ENABLED_MASTERS: - return True - builder_list = ENABLED_BUILDERS.get(master) - if builder_list and builder in builder_list: - return True - slave_list = ENABLED_SLAVES.get(master) - if slave_list and slave in slave_list: - return True - return False - - -def check_disabled(master, builder, slave): - """Returns True if disabled, False if not disabled.""" - builder_list = DISABLED_BUILDERS.get(master) - if builder_list and builder in builder_list: - return True - slave_list = DISABLED_SLAVES.get(master) - if slave_list and slave in slave_list: - return True - return False - - -def check_valid_host(master, builder, slave): - return (check_enabled(master, builder, slave) - and not check_disabled(master, builder, slave)) - - def maybe_ignore_revision(revision, buildspec): """Handle builders that don't care what buildbot tells them to build. @@ -781,17 +580,6 @@ def get_commit_message_footer(message, key): return get_commit_message_footer_map(message).get(key) -def get_svn_rev(git_hash, dir_name): - log = git('log', '-1', git_hash, cwd=dir_name) - git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY) - if not git_svn_id: - return None - m = GIT_SVN_ID_RE.match(git_svn_id) - if not m: - return None - return int(m.group(2)) - - def get_git_hash(revision, branch, sln_dir): """We want to search for the SVN revision on the git-svn branch. @@ -807,59 +595,6 @@ def get_git_hash(revision, branch, sln_dir): (revision, sln_dir)) -def _last_commit_for_file(filename, repo_base): - cmd = ['log', '--format=%H', '--max-count=1', '--', filename] - return git(*cmd, cwd=repo_base).strip() - - -def need_to_run_deps2git(repo_base, deps_file, deps_git_file): - """Checks to see if we need to run deps2git. - - Returns True if there was a DEPS change after the last .DEPS.git update - or if DEPS has local modifications. - """ - # See if DEPS is dirty - deps_file_status = git( - 'status', '--porcelain', deps_file, cwd=repo_base).strip() - if deps_file_status and deps_file_status.startswith('M '): - return True - - last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) - last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) - merge_base_ref = git('merge-base', last_known_deps_ref, - last_known_deps_git_ref, cwd=repo_base).strip() - - # If the merge base of the last DEPS and last .DEPS.git file is not - # equivilent to the hash of the last DEPS file, that means the DEPS file - # was committed after the last .DEPS.git file. - return last_known_deps_ref != merge_base_ref - - -def ensure_deps2git(solution, shallow, git_cache_dir): - repo_base = path.join(os.getcwd(), solution['name']) - deps_file = path.join(repo_base, 'DEPS') - deps_git_file = path.join(repo_base, '.DEPS.git') - if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or - not git('ls-files', '.DEPS.git', cwd=repo_base).strip()): - return - - print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) - if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): - return - - print '===DEPS file modified, need to run deps2git===' - cmd = [sys.executable, DEPS2GIT_PATH, - '--workspace', os.getcwd(), - '--cache_dir', git_cache_dir, - '--deps', deps_file, - '--out', deps_git_file] - if 'chrome-internal.googlesource' in solution['url']: - cmd.extend(['--extra-rules', S2G_INTERNAL_PATH]) - if shallow: - cmd.append('--shallow') - call(*cmd) - - def emit_log_lines(name, lines): for line in lines.splitlines(): print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line) @@ -925,6 +660,7 @@ def force_revision(folder_name, revision): ref = branch if branch.startswith('refs/') else 'origin/%s' % branch git('checkout', '--force', ref, cwd=folder_name) + def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): build_dir = os.getcwd() # Before we do anything, break all git_cache locks. @@ -985,16 +721,6 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): else: raise remove(sln_dir) - except SVNRevisionNotFound: - tries_left -= 1 - if tries_left > 0: - # If we don't have the correct revision, wait and try again. - print 'We can\'t find revision %s.' % revision - print 'The svn to git replicator is probably falling behind.' - print 'waiting 5 seconds and trying again...' - time.sleep(5) - else: - raise git('clean', '-dff', cwd=sln_dir) @@ -1005,16 +731,6 @@ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): return git_ref -def _download(url): - """Fetch url and return content, with retries for flake.""" - for attempt in xrange(ATTEMPTS): - try: - return urllib2.urlopen(url).read() - except Exception: - if attempt == ATTEMPTS - 1: - raise - - def parse_diff(diff): """Takes a unified diff and returns a list of diffed files and their diffs. @@ -1224,12 +940,8 @@ def get_commit_position(git_path, revision='HEAD'): return None -def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs): - """Translate git gclient revision mapping to build properties. - - If use_svn_revs is True, then translate git hashes in the revision mapping - to svn revision numbers. - """ +def parse_got_revision(gclient_output, got_revision_mapping): + """Translate git gclient revision mapping to build properties.""" properties = {} solutions_output = { # Make sure path always ends with a single slash. @@ -1249,12 +961,7 @@ def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs): # Since we are using .DEPS.git, everything had better be git. assert solution_output.get('scm') == 'git' git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip() - if use_svn_revs: - revision = get_svn_rev(git_revision, dir_name) - if not revision: - revision = git_revision - else: - revision = git_revision + revision = git_revision commit_position = get_commit_position(dir_name) properties[property_name] = revision @@ -1286,7 +993,6 @@ def ensure_deps_revisions(deps_url_mapping, solutions, revisions): revisions) if not revision: continue - # TODO(hinoka): Catch SVNRevisionNotFound error maybe? git('fetch', 'origin', cwd=deps_name) force_revision(deps_name, revision) @@ -1323,11 +1029,6 @@ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, apply_issue_key_file, whitelist=[target]) already_patched.append(target) - if not buildspec: - # Run deps2git if there is a DEPS change after the last .DEPS.git commit. - for solution in solutions: - ensure_deps2git(solution, shallow, git_cache_dir) - # Ensure our build/ directory is set up with the correct .gclient file. gclient_configure(solutions, target_os, target_os_only, git_cache_dir) @@ -1431,8 +1132,6 @@ def parse_args(): help='--private-key-file option passthrough for ' 'apply_patch.py.') parse.add_option('--patch_url', help='Optional URL to SVN patch.') - parse.add_option('--root', dest='patch_root', - help='DEPRECATED: Use --patch_root.') parse.add_option('--patch_root', help='Directory to patch on top of.') parse.add_option('--rietveld_server', default='codereview.chromium.org', @@ -1441,10 +1140,6 @@ def parse_args(): help='Gerrit repository to pull the ref from.') parse.add_option('--gerrit_ref', help='Gerrit ref to apply.') parse.add_option('--specs', help='Gcilent spec.') - parse.add_option('--master', help='Master name.') - parse.add_option('-f', '--force', action='store_true', - help='Bypass check to see if we want to be run. ' - 'Should ONLY be used locally or by smart recipes.') parse.add_option('--revision_mapping', help='{"path/to/repo/": "property_name"}') parse.add_option('--revision_mapping_file', @@ -1460,11 +1155,6 @@ def parse_args(): 'set to :.') parse.add_option('--output_manifest', action='store_true', help=('Add manifest json to the json output.')) - parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], - help='Hostname of the current machine, ' - 'used for determining whether or not to activate.') - parse.add_option('--builder_name', help='Name of the builder, ' - 'used for determining whether or not to activate.') parse.add_option('--build_dir', default=os.getcwd()) parse.add_option('--flag_file', default=path.join(os.getcwd(), 'update.flag')) @@ -1526,25 +1216,17 @@ def parse_args(): return options, args -def prepare(options, git_slns, active): +def prepare(options, git_slns): """Prepares the target folder before we checkout.""" dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] - # If we're active now, but the flag file doesn't exist (we weren't active - # last run) or vice versa, blow away all checkouts. - if bool(active) != bool(check_flag(options.flag_file)): - ensure_no_checkout(dir_names, '*') if options.output_json: # Make sure we tell recipes that we didn't run if the script exits here. - emit_json(options.output_json, did_run=active) - if active: - if options.clobber: - ensure_no_checkout(dir_names, '*') - else: - ensure_no_checkout(dir_names, '.svn') - emit_flag(options.flag_file) + emit_json(options.output_json, did_run=True) + if options.clobber: + ensure_no_checkout(dir_names, '*') else: - delete_flag(options.flag_file) - raise Inactive # This is caught in main() and we exit cleanly. + ensure_no_checkout(dir_names, '.svn') + emit_flag(options.flag_file) # Do a shallow checkout if the disk is less than 100GB. total_disk_space, free_disk_space = get_total_disk_space() @@ -1571,7 +1253,7 @@ def prepare(options, git_slns, active): return revisions, step_text -def checkout(options, git_slns, specs, buildspec, master, +def checkout(options, git_slns, specs, buildspec, svn_root, revisions, step_text): first_sln = git_slns[0]['name'] dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] @@ -1633,9 +1315,6 @@ def checkout(options, git_slns, specs, buildspec, master, print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text raise - # Revision is an svn revision, unless it's a git master. - use_svn_rev = master not in GIT_MASTERS - # Take care of got_revisions outputs. revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) if options.revision_mapping: @@ -1647,8 +1326,7 @@ def checkout(options, git_slns, specs, buildspec, master, if not revision_mapping: revision_mapping[first_sln] = 'got_revision' - got_revisions = parse_got_revision(gclient_output, revision_mapping, - use_svn_rev) + got_revisions = parse_got_revision(gclient_output, revision_mapping) if not got_revisions: # TODO(hinoka): We should probably bail out here, but in the interest @@ -1673,22 +1351,9 @@ def checkout(options, git_slns, specs, buildspec, master, emit_properties(got_revisions) -def print_help_text(force, output_json, active, master, builder, slave): +def print_help_text(master, builder, slave): """Print helpful messages to tell devs whats going on.""" - if force and output_json: - recipe_force = 'Forced on by recipes' - elif active and output_json: - recipe_force = 'Off by recipes, but forced on by bot update' - elif not active and output_json: - recipe_force = 'Forced off by recipes' - else: - recipe_force = 'N/A. Was not called by recipes' - print BOT_UPDATE_MESSAGE % { - 'master': master or 'Not specified', - 'builder': builder or 'Not specified', - 'slave': slave or 'Not specified', - 'recipe': recipe_force, 'CURRENT_DIR': CURRENT_DIR, 'BUILDER_DIR': BUILDER_DIR, 'SLAVE_DIR': SLAVE_DIR, @@ -1698,7 +1363,6 @@ def print_help_text(force, output_json, active, master, builder, slave): 'ROOT_DIR': ROOT_DIR, 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, }, - print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE def main(): @@ -1708,12 +1372,8 @@ def main(): slave = options.slave_name master = options.master - # Check if this script should activate or not. - active = check_valid_host(master, builder, slave) or options.force or False - - # Print a helpful message to tell developers whats going on with this step. - print_help_text( - options.force, options.output_json, active, master, builder, slave) + # Prints some debugging information. + print_help_text(master, builder, slave) # Parse, munipulate, and print the gclient solutions. specs = {} @@ -1726,13 +1386,10 @@ def main(): try: # Dun dun dun, the main part of bot_update. - revisions, step_text = prepare(options, git_slns, active) - checkout(options, git_slns, specs, buildspec, master, svn_root, revisions, + revisions, step_text = prepare(options, git_slns) + checkout(options, git_slns, specs, buildspec, svn_root, revisions, step_text) - except Inactive: - # Not active, should count as passing. - pass except PatchFailed as e: emit_flag(options.flag_file) # Return a specific non-zero exit code for patch failure (because it is diff --git a/recipe_modules/bot_update/resources/patch.exe b/recipe_modules/bot_update/resources/patch.exe new file mode 100755 index 000000000..b8eca1167 Binary files /dev/null and b/recipe_modules/bot_update/resources/patch.exe differ diff --git a/recipe_modules/bot_update/test_api.py b/recipe_modules/bot_update/test_api.py index 64eab8ab1..97fac0852 100644 --- a/recipe_modules/bot_update/test_api.py +++ b/recipe_modules/bot_update/test_api.py @@ -14,73 +14,61 @@ import bot_update class BotUpdateTestApi(recipe_test_api.RecipeTestApi): - def output_json(self, master, builder, slave, root, first_sln, - revision_mapping, git_mode, force=False, fail_patch=False, + def output_json(self, root, first_sln, revision_mapping, fail_patch=False, output_manifest=False, fixed_revisions=None): """Deterministically synthesize json.output test data for gclient's --output-json option. """ - active = bot_update.check_valid_host(master, builder, slave) or force output = { - 'did_run': active, + 'did_run': True, 'patch_failure': False } - # Add in extra json output if active. - if active: - properties = { - property_name: self.gen_revision(project_name, git_mode) - for project_name, property_name in revision_mapping.iteritems() - } - properties.update({ - '%s_cp' % property_name: ('refs/heads/master@{#%s}' % - self.gen_revision(project_name, False)) - for project_name, property_name in revision_mapping.iteritems() - }) + properties = { + property_name: self.gen_revision(project_name, True) + for project_name, property_name in revision_mapping.iteritems() + } + properties.update({ + '%s_cp' % property_name: ('refs/heads/master@{#%s}' % + self.gen_revision(project_name, False)) + for project_name, property_name in revision_mapping.iteritems() + }) - # We also want to simulate outputting "got_revision_git": ... - # when git mode is off to match what bot_update.py does. - if not git_mode: - properties.update({ - '%s_git' % property_name: self.gen_revision(project_name, True) - for project_name, property_name in revision_mapping.iteritems() - }) + output.update({ + 'patch_root': root or first_sln, + 'root': first_sln, + 'properties': properties, + 'step_text': 'Some step text' + }) + if output_manifest: output.update({ - 'patch_root': root or first_sln, - 'root': first_sln, - 'properties': properties, - 'step_text': 'Some step text' - }) - - if output_manifest: - output.update({ - 'manifest': { - project_name: { - 'repository': 'https://fake.org/%s.git' % project_name, - 'revision': self.gen_revision(project_name, git_mode), - } - for project_name in revision_mapping + 'manifest': { + project_name: { + 'repository': 'https://fake.org/%s.git' % project_name, + 'revision': self.gen_revision(project_name, True), } - }) + for project_name in revision_mapping + } + }) - if fixed_revisions: - output['fixed_revisions'] = fixed_revisions + if fixed_revisions: + output['fixed_revisions'] = fixed_revisions - if fail_patch: - output['log_lines'] = [('patch error', 'Patch failed to apply'),] - output['patch_failure'] = True - output['patch_apply_return_code'] = 1 - if fail_patch == 'download': - output['patch_apply_return_code'] = 3 + if fail_patch: + output['log_lines'] = [('patch error', 'Patch failed to apply'),] + output['patch_failure'] = True + output['patch_apply_return_code'] = 1 + if fail_patch == 'download': + output['patch_apply_return_code'] = 3 return self.m.json.output(output) @staticmethod - def gen_revision(project, GIT_MODE): + def gen_revision(project, git_mode): """Hash project to bogus deterministic revision values.""" h = hashlib.sha1(project) - if GIT_MODE: + if git_mode: return h.hexdigest() else: return struct.unpack('!I', h.digest()[:4])[0] % 300000 diff --git a/tests/bot_update_coverage_test.py b/tests/bot_update_coverage_test.py new file mode 100755 index 000000000..19adfd2f8 --- /dev/null +++ b/tests/bot_update_coverage_test.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import codecs +import copy +import json +import os +import sys +import unittest + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RESOURCE_DIR = os.path.join( + ROOT_DIR, 'recipe_modules', 'bot_update', 'resources') + +sys.path.insert(0, RESOURCE_DIR) +sys.platform = 'linux2' # For consistency, ya know? +import bot_update + +DEFAULT_PARAMS = { + 'solutions': [{ + 'name': 'somename', + 'url': 'https://fake.com' + }], + 'revisions': [], + 'first_sln': 'somename', + 'target_os': None, + 'target_os_only': None, + 'patch_root': None, + 'issue': None, + 'patchset': None, + 'patch_url': None, + 'rietveld_server': None, + 'gerrit_ref': None, + 'gerrit_repo': None, + 'revision_mapping': {}, + 'apply_issue_email_file': None, + 'apply_issue_key_file': None, + 'buildspec': False, + 'gyp_env': None, + 'shallow': False, + 'runhooks': False, + 'refs': [], + 'git_cache_dir': 'fake_cache_dir' +} + + +class MockedPopen(object): + """A fake instance of a called subprocess. + + This is meant to be used in conjunction with MockedCall. + """ + def __init__(self, args=None, kwargs=None): + self.args = args or [] + self.kwargs = kwargs or {} + self.return_value = None + self.fails = False + + def returns(self, rv): + """Set the return value when this popen is called. + + rv can be a string, or a callable (eg function). + """ + self.return_value = rv + return self + + def check(self, args, kwargs): + """Check to see if the given args/kwargs call match this instance. + + This does a partial match, so that a call to "git clone foo" will match + this instance if this instance was recorded as "git clone" + """ + if any(input_arg != expected_arg + for (input_arg, expected_arg) in zip(args, self.args)): + return False + return self.return_value + + def __call__(self, args, kwargs): + """Actually call this popen instance.""" + if hasattr(self.return_value, '__call__'): + return self.return_value(*args, **kwargs) + return self.return_value + + +class MockedCall(object): + """A fake instance of bot_update.call(). + + This object is pre-seeded with "answers" in self.expectations. The type + is a MockedPopen object, or any object with a __call__() and check() method. + The check() method is used to check to see if the correct popen object is + chosen (can be a partial match, eg a "git clone" popen module would match + a "git clone foo" call). + By default, if no answers have been pre-seeded, the call() returns successful + with an empty string. + """ + def __init__(self, fake_filesystem): + self.expectations = [] + self.records = [] + + def expect(self, args=None, kwargs=None): + args = args or [] + kwargs = kwargs or {} + popen = MockedPopen(args, kwargs) + self.expectations.append(popen) + return popen + + def __call__(self, *args, **kwargs): + self.records.append((args, kwargs)) + for popen in self.expectations: + if popen.check(args, kwargs): + self.expectations.remove(popen) + return popen(args, kwargs) + return '' + + +class MockedGclientSync(): + """A class producing a callable instance of gclient sync. + + Because for bot_update, gclient sync also emits an output json file, we need + a callable object that can understand where the output json file is going, and + emit a (albite) fake file for bot_update to consume. + """ + def __init__(self, fake_filesystem): + self.output = {} + self.fake_filesystem = fake_filesystem + + def __call__(self, *args, **_): + output_json_index = args.index('--output-json') + 1 + with self.fake_filesystem.open(args[output_json_index], 'w') as f: + json.dump(self.output, f) + + +class FakeFile(): + def __init__(self): + self.contents = '' + + def write(self, buf): + self.contents += buf + + def read(self): + return self.contents + + def __enter__(self): + return self + + def __exit__(self, _, __, ___): + pass + + +class FakeFilesystem(): + def __init__(self): + self.files = {} + + def open(self, target, mode='r', encoding=None): + if 'w' in mode: + self.files[target] = FakeFile() + return self.files[target] + return self.files[target] + + +def fake_git(*args, **kwargs): + return bot_update.call('git', *args, **kwargs) + + +class EnsureCheckoutUnittests(unittest.TestCase): + def setUp(self): + self.filesystem = FakeFilesystem() + self.call = MockedCall(self.filesystem) + self.gclient = MockedGclientSync(self.filesystem) + self.call.expect(('gclient', 'sync')).returns(self.gclient) + self.old_call = getattr(bot_update, 'call') + self.params = copy.deepcopy(DEFAULT_PARAMS) + setattr(bot_update, 'call', self.call) + setattr(bot_update, 'git', fake_git) + + self.old_os_cwd = os.getcwd + setattr(os, 'getcwd', lambda: '/b/build/slave/foo/build') + + setattr(bot_update, 'open', self.filesystem.open) + self.old_codecs_open = codecs.open + setattr(codecs, 'open', self.filesystem.open) + + def tearDown(self): + setattr(bot_update, 'call', self.old_call) + setattr(os, 'getcwd', self.old_os_cwd) + delattr(bot_update, 'open') + setattr(codecs, 'open', self.old_codecs_open) + + def testBasic(self): + bot_update.ensure_checkout(**self.params) + return self.call.records + + def testBasicBuildspec(self): + self.params['buildspec'] = bot_update.BUILDSPEC_TYPE( + container='branches', + version='1.1.1.1' + ) + bot_update.ensure_checkout(**self.params) + return self.call.records + + def testBasicShallow(self): + self.params['shallow'] = True + bot_update.ensure_checkout(**self.params) + return self.call.records + + def testBasicSVNPatch(self): + self.params['patch_url'] = 'svn://server.com/patch.diff' + self.params['patch_root'] = 'somename' + bot_update.ensure_checkout(**self.params) + return self.call.records + + def testBasicRietveldPatch(self): + self.params['issue'] = '12345' + self.params['patchset'] = '1' + self.params['rietveld_server'] = 'https://rietveld.com' + self.params['patch_root'] = 'somename' + bot_update.ensure_checkout(**self.params) + return self.call.records + + +class SolutionsUnittests(unittest.TestCase): + def testBasicSVN(self): + sln = [{ + 'name': 'src', + 'url': 'svn://svn-mirror.golo.chromium.org/chrome/trunk/src' + }, { + 'name': 'git', + 'url': 'https://abc.googlesource.com/foo.git' + }] + git_slns, root, buildspec = bot_update.solutions_to_git(sln) + self.assertEquals(root, '/chrome/trunk/src') + self.assertEquals(git_slns, [{ + 'deps_file': '.DEPS.git', + 'managed': False, + 'name': 'src', + 'url': 'https://chromium.googlesource.com/chromium/src.git' + }, { + 'url': 'https://abc.googlesource.com/foo.git', + 'managed': False, + 'name': 'git' + }]) + self.assertFalse(buildspec) + bot_update.solutions_printer(git_slns) + + def testBasicBuildspec(self): + sln = [{ + 'name': 'buildspec', + 'url': ('svn://svn.chromium.org/chrome-internal/' + 'trunk/tools/buildspec/releases/1234'), + }] + git_slns, root, buildspec = bot_update.solutions_to_git(sln) + self.assertEquals( + buildspec, + bot_update.BUILDSPEC_TYPE(container='releases', version='1234')) + self.assertEquals( + root, '/chrome-internal/trunk/tools/buildspec/releases/1234') + self.assertEquals(git_slns[0]['deps_file'], 'releases/1234/DEPS') + +if __name__ == '__main__': + unittest.main()