From 124365b8ea553111c78af62c0d7455423a2238e5 Mon Sep 17 00:00:00 2001 From: Andrii Shyshkalov Date: Thu, 21 Jun 2018 20:01:20 +0000 Subject: [PATCH] bot_update: default to non-shallow checkouts. shallow mode is a special git mode that only checks out a shallow copy of git so the full history is not downloaded. It is not well supported by googlesource.com servers because git has to do a giant negotiation with the server and transmit every single commit that the local checkout has or doesn't have (instead of just the latest commit of each ref), which ends up taking more than 10x longer than a non-shallow checkout. R=ehmaldonado, hinoka, iannucci Recipe-Nontrivial-Roll: infra Recipe-Nontrivial-Roll: build Recipe-Nontrivial-Roll: build_limited_scripts_slave Bug: 855137 Change-Id: I5f9e31f8b2730cf80b79bfd0e08201c33eb942f5 Reviewed-on: https://chromium-review.googlesource.com/1110450 Commit-Queue: Andrii Shyshkalov Reviewed-by: Ryan Tseng Reviewed-by: Robbie Iannucci --- recipes/README.recipes.md | 2 +- recipes/recipe_modules/bot_update/api.py | 6 +++--- .../full.expected/{no_shallow.json => shallow.json} | 2 +- recipes/recipe_modules/bot_update/examples/full.py | 6 +++--- recipes/recipe_modules/bot_update/resources/bot_update.py | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) rename recipes/recipe_modules/bot_update/examples/full.expected/{no_shallow.json => shallow.json} (99%) diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index b091dc418..8bde607ad 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -53,7 +53,7 @@ Wrapper for easy calling of bot_update. Deapplies a patch, taking care of DEPS and solution revisions properly. -— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, no_shallow=False, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** +— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, no_shallow=True, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** Args: gclient_config: The gclient configuration to use when running bot_update. diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py index 2e4e13e3b..ff55263a0 100644 --- a/recipes/recipe_modules/bot_update/api.py +++ b/recipes/recipe_modules/bot_update/api.py @@ -67,7 +67,7 @@ class BotUpdateApi(recipe_api.RecipeApi): def ensure_checkout(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, - patch_root=None, no_shallow=False, + patch_root=None, no_shallow=True, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, @@ -208,8 +208,8 @@ class BotUpdateApi(recipe_api.RecipeApi): if clobber: cmd.append('--clobber') - if no_shallow: - cmd.append('--no_shallow') + if no_shallow is False: + cmd.append('--maybe_shallow') if with_branch_heads or cfg.with_branch_heads: cmd.append('--with_branch_heads') if with_tags or cfg.with_tags: diff --git a/recipes/recipe_modules/bot_update/examples/full.expected/no_shallow.json b/recipes/recipe_modules/bot_update/examples/full.expected/shallow.json similarity index 99% rename from recipes/recipe_modules/bot_update/examples/full.expected/no_shallow.json rename to recipes/recipe_modules/bot_update/examples/full.expected/shallow.json index 039848f94..9eba242af 100644 --- a/recipes/recipe_modules/bot_update/examples/full.expected/no_shallow.json +++ b/recipes/recipe_modules/bot_update/examples/full.expected/shallow.json @@ -18,7 +18,7 @@ "/path/to/tmp/json", "--revision", "src@HEAD", - "--no_shallow", + "--maybe_shallow", "--disable-syntax-validation" ], "env_prefixes": { diff --git a/recipes/recipe_modules/bot_update/examples/full.py b/recipes/recipe_modules/bot_update/examples/full.py index 7413c5054..45a9ca255 100644 --- a/recipes/recipe_modules/bot_update/examples/full.py +++ b/recipes/recipe_modules/bot_update/examples/full.py @@ -40,7 +40,7 @@ def RunSteps(api): patch = api.properties.get('patch', True) clobber = True if api.properties.get('clobber') else False - no_shallow = True if api.properties.get('no_shallow') else False + no_shallow = True if api.properties.get('no_shallow', 1) else False with_branch_heads = api.properties.get('with_branch_heads', False) with_tags = api.properties.get('with_tags', False) refs = api.properties.get('refs', []) @@ -148,8 +148,8 @@ def GenTests(api): rietveld='https://rietveld.example.com/', fail_patch='download' ) + api.step_data('bot_update', retcode=87) - yield api.test('no_shallow') + api.properties( - no_shallow=1 + yield api.test('shallow') + api.properties( + no_shallow=0 ) yield api.test('clobber') + api.properties( clobber=1 diff --git a/recipes/recipe_modules/bot_update/resources/bot_update.py b/recipes/recipe_modules/bot_update/resources/bot_update.py index a4fd22198..a51494ce3 100755 --- a/recipes/recipe_modules/bot_update/resources/bot_update.py +++ b/recipes/recipe_modules/bot_update/resources/bot_update.py @@ -1038,9 +1038,9 @@ def parse_args(): help='Delete checkout first, always') parse.add_option('--output_json', help='Output JSON information into a specified file') - parse.add_option('--no_shallow', action='store_true', - help='Bypass disk detection and never shallow clone. ' - 'Does not override the --shallow flag') + parse.add_option('--maybe_shallow', action='store_true', + help='Enables turning on shallow mode if total disk ' + 'space is low.') parse.add_option('--refs', action='append', help='Also fetch this refspec for the main solution(s). ' 'Eg. +refs/branch-heads/*') @@ -1126,7 +1126,7 @@ def prepare(options, git_slns, active): total_disk_space_gb, percent_used) shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD - and not options.no_shallow) + and options.maybe_shallow) # The first solution is where the primary DEPS file resides. first_sln = dir_names[0]