Parallelize pylint PRESUBMIT checks.

R=maruel@chromium.org
BUG=479837,499650

Review URL: https://codereview.chromium.org/1181103002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295664 0039d316-1c4b-4281-b951-d872f2087c98

presubmit_canned_checks.py

@@ -771,20 +771,28 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
   env['PYTHONPATH'] = input_api.os_path.pathsep.join(
       extra_paths_list + sys.path).encode('utf8')
 
-  def GetPylintCmd(files):
+  def GetPylintCmd(files, extra, parallel):
     # Windows needs help running python files so we explicitly specify
     # the interpreter to use. It also has limitations on the size of
     # the command-line, so we pass arguments via a pipe.
+    cmd = [input_api.python_executable,
+           input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
+           '--args-on-stdin']
     if len(files) == 1:
       description = files[0]
     else:
       description = '%s files' % len(files)
 
+    if extra:
+      cmd.extend(extra)
+      description += ' using %s' % (extra,)
+    if parallel:
+      cmd.append('--jobs=%s' % input_api.cpu_count)
+      description += ' on %d cores' % input_api.cpu_count
+
     return input_api.Command(
         name='Pylint (%s)' % description,
-        cmd=[input_api.python_executable,
-             input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
-             '--args-on-stdin'],
+        cmd=cmd,
         kwargs={'env': env, 'stdin': '\n'.join(files + extra_args)},
         message=error_type)
 
@@ -797,9 +805,14 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
   # a quick local edit to diagnose pylint issues more
   # easily.
   if True:
-    return [GetPylintCmd(files)]
+    # pylint's cycle detection doesn't work in parallel, so spawn a second,
+    # single-threaded job for just that check.
+    return [
+      GetPylintCmd(files, ["--disable=cyclic-import"], True),
+      GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], False)
+    ]
   else:
-    return map(lambda x: GetPylintCmd([x]), files)
+    return map(lambda x: GetPylintCmd([x], extra_args, 1), files)
 
 
 def RunPylint(input_api, *args, **kwargs):

presubmit_support.py

@@ -311,6 +311,8 @@ class InputApi(object):
     # InputApi.platform is the platform you're currently running on.
     self.platform = sys.platform
 
+    self.cpu_count = multiprocessing.cpu_count()
+
     # The local path of the currently-being-processed presubmit script.
     self._current_presubmit_path = os.path.dirname(presubmit_path)
 

tests/presubmit_unittest.py

@@ -1183,17 +1183,15 @@ class InputApiUnittest(PresubmitTestsBase):
     self.mox.ReplayAll()
     members = [
       'AbsoluteLocalPaths', 'AffectedFiles', 'AffectedSourceFiles',
-      'AffectedTextFiles',
-      'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST',
-      'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths',
-      'LocalToDepotPath', 'Command', 'RunTests',
-      'PresubmitLocalPath', 'ReadFile', 'RightHandSideLines', 'ServerPaths',
-      'basename', 'cPickle', 'cpplint', 'cStringIO', 'canned_checks', 'change',
-      'environ', 'glob', 'host_url', 'is_committing', 'json', 'logging',
-      'marshal', 'os_listdir', 'os_walk', 'os_path', 'os_stat', 'owners_db',
-      'pickle', 'platform', 'python_executable', 're', 'rietveld', 'subprocess',
-      'tbr', 'tempfile', 'time', 'traceback', 'unittest', 'urllib2', 'version',
-      'verbose',
+      'AffectedTextFiles', 'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST',
+      'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths', 'LocalToDepotPath',
+      'Command', 'RunTests', 'PresubmitLocalPath', 'ReadFile',
+      'RightHandSideLines', 'ServerPaths', 'basename', 'cPickle', 'cpplint',
+      'cStringIO', 'canned_checks', 'change', 'cpu_count', 'environ', 'glob',
+      'host_url', 'is_committing', 'json', 'logging', 'marshal', 'os_listdir',
+      'os_walk', 'os_path', 'os_stat', 'owners_db', 'pickle', 'platform',
+      'python_executable', 're', 'rietveld', 'subprocess', 'tbr', 'tempfile',
+      'time', 'traceback', 'unittest', 'urllib2', 'version', 'verbose',
     ]
     # If this test fails, you should add the relevant test.
     self.compareMembers(
@@ -1852,6 +1850,7 @@ class CannedChecksUnittest(PresubmitTestsBase):
     input_api.tbr = False
     input_api.python_executable = 'pyyyyython'
     input_api.platform = sys.platform
+    input_api.cpu_count = 2
     input_api.time = time
     input_api.canned_checks = presubmit_canned_checks
     input_api.Command = presubmit.CommandData
@@ -2525,7 +2524,12 @@ class CannedChecksUnittest(PresubmitTestsBase):
     pylintrc = os.path.join(_ROOT, 'pylintrc')
 
     CommHelper(input_api,
-        ['pyyyyython', pylint, '--args-on-stdin'],
+        ['pyyyyython', pylint, '--args-on-stdin', '--disable=cyclic-import',
+         '--jobs=2'],
+        env=mox.IgnoreArg(), stdin='file1.py\n--rcfile=%s' % pylintrc)
+    CommHelper(input_api,
+        ['pyyyyython', pylint, '--args-on-stdin', '--disable=all',
+         '--enable=cyclic-import'],
         env=mox.IgnoreArg(), stdin='file1.py\n--rcfile=%s' % pylintrc)
     self.mox.ReplayAll()