From 0af3bb37eb2d7e194063d0614a769d85a6a315c3 Mon Sep 17 00:00:00 2001 From: "iannucci@chromium.org" Date: Fri, 12 Jun 2015 20:44:35 +0000 Subject: [PATCH] Parallelize pylint PRESUBMIT checks. R=maruel@chromium.org BUG=479837,499650 Review URL: https://codereview.chromium.org/1181103002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295664 0039d316-1c4b-4281-b951-d872f2087c98 --- presubmit_canned_checks.py | 25 +++++++++++++++++++------ presubmit_support.py | 2 ++ tests/presubmit_unittest.py | 28 ++++++++++++++++------------ 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/presubmit_canned_checks.py b/presubmit_canned_checks.py index 9179fb6ee..edd633219 100644 --- a/presubmit_canned_checks.py +++ b/presubmit_canned_checks.py @@ -771,20 +771,28 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None, env['PYTHONPATH'] = input_api.os_path.pathsep.join( extra_paths_list + sys.path).encode('utf8') - def GetPylintCmd(files): + def GetPylintCmd(files, extra, parallel): # Windows needs help running python files so we explicitly specify # the interpreter to use. It also has limitations on the size of # the command-line, so we pass arguments via a pipe. + cmd = [input_api.python_executable, + input_api.os_path.join(_HERE, 'third_party', 'pylint.py'), + '--args-on-stdin'] if len(files) == 1: description = files[0] else: description = '%s files' % len(files) + if extra: + cmd.extend(extra) + description += ' using %s' % (extra,) + if parallel: + cmd.append('--jobs=%s' % input_api.cpu_count) + description += ' on %d cores' % input_api.cpu_count + return input_api.Command( name='Pylint (%s)' % description, - cmd=[input_api.python_executable, - input_api.os_path.join(_HERE, 'third_party', 'pylint.py'), - '--args-on-stdin'], + cmd=cmd, kwargs={'env': env, 'stdin': '\n'.join(files + extra_args)}, message=error_type) @@ -797,9 +805,14 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None, # a quick local edit to diagnose pylint issues more # easily. if True: - return [GetPylintCmd(files)] + # pylint's cycle detection doesn't work in parallel, so spawn a second, + # single-threaded job for just that check. + return [ + GetPylintCmd(files, ["--disable=cyclic-import"], True), + GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], False) + ] else: - return map(lambda x: GetPylintCmd([x]), files) + return map(lambda x: GetPylintCmd([x], extra_args, 1), files) def RunPylint(input_api, *args, **kwargs): diff --git a/presubmit_support.py b/presubmit_support.py index 0abd42191..4baee7788 100755 --- a/presubmit_support.py +++ b/presubmit_support.py @@ -311,6 +311,8 @@ class InputApi(object): # InputApi.platform is the platform you're currently running on. self.platform = sys.platform + self.cpu_count = multiprocessing.cpu_count() + # The local path of the currently-being-processed presubmit script. self._current_presubmit_path = os.path.dirname(presubmit_path) diff --git a/tests/presubmit_unittest.py b/tests/presubmit_unittest.py index 0d2b4d68f..f37bf6ca6 100755 --- a/tests/presubmit_unittest.py +++ b/tests/presubmit_unittest.py @@ -1183,17 +1183,15 @@ class InputApiUnittest(PresubmitTestsBase): self.mox.ReplayAll() members = [ 'AbsoluteLocalPaths', 'AffectedFiles', 'AffectedSourceFiles', - 'AffectedTextFiles', - 'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST', - 'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths', - 'LocalToDepotPath', 'Command', 'RunTests', - 'PresubmitLocalPath', 'ReadFile', 'RightHandSideLines', 'ServerPaths', - 'basename', 'cPickle', 'cpplint', 'cStringIO', 'canned_checks', 'change', - 'environ', 'glob', 'host_url', 'is_committing', 'json', 'logging', - 'marshal', 'os_listdir', 'os_walk', 'os_path', 'os_stat', 'owners_db', - 'pickle', 'platform', 'python_executable', 're', 'rietveld', 'subprocess', - 'tbr', 'tempfile', 'time', 'traceback', 'unittest', 'urllib2', 'version', - 'verbose', + 'AffectedTextFiles', 'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST', + 'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths', 'LocalToDepotPath', + 'Command', 'RunTests', 'PresubmitLocalPath', 'ReadFile', + 'RightHandSideLines', 'ServerPaths', 'basename', 'cPickle', 'cpplint', + 'cStringIO', 'canned_checks', 'change', 'cpu_count', 'environ', 'glob', + 'host_url', 'is_committing', 'json', 'logging', 'marshal', 'os_listdir', + 'os_walk', 'os_path', 'os_stat', 'owners_db', 'pickle', 'platform', + 'python_executable', 're', 'rietveld', 'subprocess', 'tbr', 'tempfile', + 'time', 'traceback', 'unittest', 'urllib2', 'version', 'verbose', ] # If this test fails, you should add the relevant test. self.compareMembers( @@ -1852,6 +1850,7 @@ class CannedChecksUnittest(PresubmitTestsBase): input_api.tbr = False input_api.python_executable = 'pyyyyython' input_api.platform = sys.platform + input_api.cpu_count = 2 input_api.time = time input_api.canned_checks = presubmit_canned_checks input_api.Command = presubmit.CommandData @@ -2525,7 +2524,12 @@ class CannedChecksUnittest(PresubmitTestsBase): pylintrc = os.path.join(_ROOT, 'pylintrc') CommHelper(input_api, - ['pyyyyython', pylint, '--args-on-stdin'], + ['pyyyyython', pylint, '--args-on-stdin', '--disable=cyclic-import', + '--jobs=2'], + env=mox.IgnoreArg(), stdin='file1.py\n--rcfile=%s' % pylintrc) + CommHelper(input_api, + ['pyyyyython', pylint, '--args-on-stdin', '--disable=all', + '--enable=cyclic-import'], env=mox.IgnoreArg(), stdin='file1.py\n--rcfile=%s' % pylintrc) self.mox.ReplayAll()