From 0cba8dd68ec726fa01170ab898604515aaa88135 Mon Sep 17 00:00:00 2001 From: Josip Sokcevic Date: Wed, 10 Jan 2024 20:01:33 +0000 Subject: [PATCH] [owners] Limit number of concurrent Gerrit connections R=gavinmak@google.com Fixed: 319469729 Change-Id: I955fa17c80cee25bc2163a8bca0234cdbb659f62 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5186760 Auto-Submit: Josip Sokcevic Commit-Queue: Gavin Mak Reviewed-by: Gavin Mak Commit-Queue: Josip Sokcevic --- gerrit_util.py | 3 +++ git_cl.py | 5 ++--- owners_client.py | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/gerrit_util.py b/gerrit_util.py index 7fe1bf0303..be4ee81adb 100644 --- a/gerrit_util.py +++ b/gerrit_util.py @@ -49,6 +49,9 @@ MIN_BACKOFF = 1.8 # This is parameterized primarily to enable GerritTestCase. GERRIT_PROTOCOL = 'https' +# Controls how many concurrent Gerrit connections there can be. +MAX_CONCURRENT_CONNECTION = 20 + def time_sleep(seconds): # Use this so that it can be mocked in tests without interfering with python diff --git a/git_cl.py b/git_cl.py index 496e8d50b7..9b8d91b8bc 100755 --- a/git_cl.py +++ b/git_cl.py @@ -81,7 +81,6 @@ GIT_HASH_RE = re.compile(r'\b([a-f0-9]{6})[a-f0-9]{34}\b', flags=re.I) GITCOOKIES_REDACT_RE = re.compile(r'1/.*') MAX_ATTEMPTS = 3 -MAX_CONCURRENT_CONNECTION = 20 # The maximum number of traces we will keep. Multiplied by 3 since we store # 3 files per trace. @@ -3928,7 +3927,7 @@ def get_cl_statuses(changes, fine_grained, max_processes=None): If max_processes is specified, it is used as the maximum number of processes to spawn to fetch CL status from the server. Otherwise 1 process per branch - is spawned, up to max of MAX_CONCURRENT_CONNECTION. + is spawned, up to max of gerrit_util.MAX_CONCURRENT_CONNECTION. See GetStatus() for a list of possible statuses. """ @@ -3957,7 +3956,7 @@ def get_cl_statuses(changes, fine_grained, max_processes=None): cl.GetIssue()) raise - threads_count = min(MAX_CONCURRENT_CONNECTION, len(changes)) + threads_count = min(gerrit_util.MAX_CONCURRENT_CONNECTION, len(changes)) if max_processes: threads_count = max(1, min(threads_count, max_processes)) logging.debug('querying %d CLs using %d threads', len(changes), diff --git a/owners_client.py b/owners_client.py index 7a523db308..4a0fdba679 100644 --- a/owners_client.py +++ b/owners_client.py @@ -48,7 +48,10 @@ class OwnersClient(object): Returns a dictionary {path: [owners]}. """ - with git_common.ScopedPool(kind='threads') as pool: + if not paths: + return dict() + nproc = min(gerrit_util.MAX_CONCURRENT_CONNECTION, len(paths)) + with git_common.ScopedPool(nproc, kind='threads') as pool: return dict( pool.imap_unordered(lambda p: (p, self.ListOwners(p)), paths))