Revert "presubmit: remove presubmit step running python2"

This reverts commit f155639336.

Reason for revert:
https://ci.chromium.org/ui/p/infra/builders/try/infra-try-presubmit/b8781520593544576993/overview

Original change's description:
> presubmit: remove presubmit step running python2
>
> This is to deactivate `luci.buildbucket.omit_python2` experiment.
>
> I'll submit this after https://crrev.com/c/4515085 is submitted.
>
> Bug: 1401307, 1441784
> Recipe-Nontrivial-Roll: build
> Change-Id: I0aec58e9b96c961da21f0de0850c4078bb679c33
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4513661
> Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
> Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
> Commit-Queue: Takuto Ikuta <tikuta@chromium.org>

Bug: 1401307, 1441784
Change-Id: I47a9921b74917a4b0caa6f577c9fc6d3542f8b54
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4519043
Reviewed-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>

recipes/README.recipes.md

@@ -758,7 +758,7 @@ Raises:
 
 Returns a presubmit step.
 
-— **def [execute](/recipes/recipe_modules/presubmit/api.py#103)(self, bot_update_step, skip_owners=False, run_all=False):**
+— **def [execute](/recipes/recipe_modules/presubmit/api.py#130)(self, bot_update_step, skip_owners=False, run_all=False):**
 
 Runs presubmit and sets summary markdown if applicable.
 
@@ -769,7 +769,7 @@ Args:
 Returns:
   a RawResult object, suitable for being returned from RunSteps.
 
-— **def [prepare](/recipes/recipe_modules/presubmit/api.py#51)(self, root_solution_revision=None):**
+— **def [prepare](/recipes/recipe_modules/presubmit/api.py#78)(self, root_solution_revision=None):**
 
 Sets up a presubmit run.
 

recipes/recipe_modules/presubmit/api.py

@@ -30,16 +30,43 @@ class PresubmitApi(recipe_api.RecipeApi):
 
     name = kwargs.pop('name', 'presubmit')
     with self.m.depot_tools.on_path():
+      # Use only vpython3 on bots that don't have vpython2 on the path any
+      # longer.
+      # TODO(https://crbug.com/1401307): Switch this to vpython3 premanently
+      # and remove py3 part below.
+      experiments = self.m.buildbucket.build.input.experiments
+      if 'luci.buildbucket.omit_python2' in experiments:
+        cmd = ['vpython3', self.presubmit_support_path, '--use-python3']
+      else:
+        cmd = ['vpython', self.presubmit_support_path]
+
+      cmd.extend(args)
+      cmd.extend(['--json_output', self.m.json.output()])
       if self.m.resultdb.enabled:
         kwargs['wrapper'] = ('rdb', 'stream', '--')
+      step_data = self.m.step(name, cmd, **kwargs)
+      output = step_data.json.output or {}
+      if self.m.step.active_result.retcode != 0 or \
+          'luci.buildbucket.omit_python2' in experiments:
+        return output
+
       presubmit_args = list(args) + [
           '--json_output',
           self.m.json.output(),
       ]
-      step_data = self.m.step(name,
+      step_data = self.m.step(name + " py3",
                               ['vpython3', self.presubmit_support_path] +
                               presubmit_args, **kwargs)
-      return step_data.json.output or {}
+      output2 = step_data.json.output or {}
+
+      # combine outputs
+      for key in output:
+        if key in output2:
+          output[key] += output2[key]
+          del (output2[key])
+      for key in output2:
+        output[key] = output2[key]
+      return output
 
   @property
   def _relative_root(self):

recipes/recipe_modules/presubmit/examples/full.expected/basic.json

@@ -1,7 +1,7 @@
 [
   {
     "cmd": [
-      "vpython3",
+      "vpython",
       "RECIPE_REPO[depot_tools]/presubmit_support.py",
       "--json_output",
       "/path/to/tmp/json"
@@ -20,6 +20,28 @@
       "@@@STEP_LOG_END@json.output@@@"
     ]
   },
+  {
+    "cmd": [
+      "vpython3",
+      "RECIPE_REPO[depot_tools]/presubmit_support.py",
+      "--json_output",
+      "/path/to/tmp/json"
+    ],
+    "env_suffixes": {
+      "DEPOT_TOOLS_UPDATE": [
+        "0"
+      ],
+      "PATH": [
+        "RECIPE_REPO[depot_tools]"
+      ]
+    },
+    "name": "presubmit py3",
+    "~followup_annotations": [
+      "@@@STEP_LOG_END@json.output (invalid)@@@",
+      "@@@STEP_LOG_LINE@json.output (exception)@No JSON object could be decoded@@@",
+      "@@@STEP_LOG_END@json.output (exception)@@@"
+    ]
+  },
   {
     "name": "$result"
   }

recipes/recipe_modules/presubmit/tests/execute.py

@@ -38,6 +38,7 @@ def GenTests(api):
           git_repo='https://chromium.googlesource.com/infra/infra'),
       api.properties(run_all=True),
       api.step_data('presubmit', api.json.output({})),
+      api.step_data('presubmit py3', api.json.output({})),
       api.post_process(post_process.StatusSuccess),
       api.post_process(post_process.DropExpectation),
   )
@@ -70,7 +71,7 @@ def GenTests(api):
 
   yield (api.test('vpython3') + api.runtime(is_experimental=False) +
          api.buildbucket.try_build(
-            project='infra') +
+            project='infra', experiments=['luci.buildbucket.omit_python2']) +
          api.post_process(post_process.StepCommandContains, 'presubmit',
                           ['vpython3']) +
          api.post_process(post_process.StatusSuccess) +
@@ -144,6 +145,57 @@ def GenTests(api):
           Expected "," after item in list
           ```
 
+          #### To see notifications and warnings, look at the stdout of the presubmit step.
+        ''').strip()) + api.post_process(post_process.DropExpectation))
+  yield (
+      api.test('failure py3', status="FAILURE") +
+      api.runtime(is_experimental=False) +
+      api.buildbucket.try_build(project='infra') + api.step_data(
+          'presubmit py3',
+          api.json.output(
+              {
+                  'errors': [{
+                      'message': 'Missing LGTM',
+                      'long_text': 'Here are some suggested OWNERS: fake@',
+                      'items': [],
+                      'fatal': True
+                  }, {
+                      'message': 'Syntax error in fake.py',
+                      'long_text': 'Expected "," after item in list',
+                      'items': [],
+                      'fatal': True
+                  }],
+                  'notifications': [{
+                      'message': 'If there is a bug associated please add it.',
+                      'long_text': '',
+                      'items': [],
+                      'fatal': False
+                  }],
+                  'warnings': [{
+                      'message': 'Line 100 has more than 80 characters',
+                      'long_text': '',
+                      'items': [],
+                      'fatal': False
+                  }]
+              },
+              retcode=1)) + api.post_process(post_process.StatusFailure) +
+      api.post_process(
+          post_process.ResultReason,
+          textwrap.dedent(u'''
+          #### There are 2 error(s), 1 warning(s), and 1 notifications(s). Here are the errors:
+
+          **ERROR**
+          ```
+          Missing LGTM
+          Here are some suggested OWNERS: fake@
+          ```
+
+          **ERROR**
+          ```
+          Syntax error in fake.py
+          Expected "," after item in list
+          ```
+
           #### To see notifications and warnings, look at the stdout of the presubmit step.
         ''').strip()) + api.post_process(post_process.DropExpectation))
 
@@ -239,3 +291,25 @@ def GenTests(api):
          api.post_process(post_process.StatusException) +
          api.post_process(post_process.ResultReason, bug_msg) +
          api.post_process(post_process.DropExpectation))
+
+  yield (api.test('warnings-merged') + api.runtime(is_experimental=False) +
+         api.buildbucket.try_build(project='infra') + api.step_data(
+             'presubmit',
+             api.json.output({
+                 'errors': [],
+                 'notifications': [],
+                 'warnings': [{
+                     'message': 'warning py2'
+                 }]
+             }),
+         ) + api.step_data(
+             'presubmit py3',
+             api.json.output({
+                 'errors': [],
+                 'extra': [],
+                 'warnings': [{
+                     'message': 'warning py3'
+                 }]
+             }),
+         ) + api.post_process(post_process.StatusSuccess) +
+         api.post_process(post_process.DropExpectation))