#!/usr/bin/env python3
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Enables directory-specific presubmit checks to run at upload and/or commit.
"""
__version__ = '2.0.0'
# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
# caching (between all different invocations of presubmit scripts for a given
# change). We should add it as our presubmit scripts start feeling slow.
import argparse
import ast # Exposed through the API.
import contextlib
import cpplint
import fnmatch # Exposed through the API.
import glob
import inspect
import json # Exposed through the API.
import logging
import mimetypes
import multiprocessing
import os # Somewhat exposed through the API.
import random
import re # Exposed through the API.
import shutil
import signal
import sys # Parts exposed through API.
import tempfile # Exposed through the API.
import threading
import time
import traceback
import unittest # Exposed through the API.
import urllib.parse as urlparse
import urllib.request as urllib_request
import urllib.error as urllib_error
from typing import Mapping
from warnings import warn
# Local imports.
import gclient_paths # Exposed through the API
import gclient_utils
import git_footers
import gerrit_util
import owners_client
import owners_finder
import presubmit_canned_checks
import presubmit_diff
import rdb_wrapper
import scm
import subprocess2 as subprocess # Exposed through the API.
# TODO: Should fix these warnings.
# pylint: disable=line-too-long
# Ask for feedback only once in program lifetime.
_ASKED_FOR_FEEDBACK = False
# Set if super-verbose mode is requested, for tracking where presubmit messages
# are coming from.
_SHOW_CALLSTACKS = False
def time_time():
# Use this so that it can be mocked in tests without interfering with python
# system machinery.
return time.time()
class PresubmitFailure(Exception):
pass
class CommandData(object):
def __init__(self, name, cmd, kwargs, message, python3=True):
# The python3 argument is ignored but has to be retained because of the
# many callers in other repos that pass it in.
del python3
self.name = name
self.cmd = cmd
self.stdin = kwargs.get('stdin', None)
self.kwargs = kwargs.copy()
self.kwargs['stdout'] = subprocess.PIPE
self.kwargs['stderr'] = subprocess.STDOUT
self.kwargs['stdin'] = subprocess.PIPE
self.message = message
self.info = None
# Adapted from
# https://github.com/google/gtest-parallel/blob/master/gtest_parallel.py#L37
#
# An object that catches SIGINT sent to the Python process and notices
# if processes passed to wait() die by SIGINT (we need to look for
# both of those cases, because pressing Ctrl+C can result in either
# the main process or one of the subprocesses getting the signal).
#
# Before a SIGINT is seen, wait(p) will simply call p.wait() and
# return the result. Once a SIGINT has been seen (in the main process
# or a subprocess, including the one the current call is waiting for),
# wait(p) will call p.terminate().
class SigintHandler(object):
sigint_returncodes = {
-signal.SIGINT, # Unix
-1073741510, # Windows
}
def __init__(self):
self.__lock = threading.Lock()
self.__processes = set()
self.__got_sigint = False
self.__previous_signal = signal.signal(signal.SIGINT, self.interrupt)
def __on_sigint(self):
self.__got_sigint = True
while self.__processes:
try:
self.__processes.pop().terminate()
except OSError:
pass
def interrupt(self, signal_num, frame):
with self.__lock:
self.__on_sigint()
self.__previous_signal(signal_num, frame)
def got_sigint(self):
with self.__lock:
return self.__got_sigint
def wait(self, p, stdin):
with self.__lock:
if self.__got_sigint:
p.terminate()
self.__processes.add(p)
stdout, stderr = p.communicate(stdin)
code = p.returncode
with self.__lock:
self.__processes.discard(p)
if code in self.sigint_returncodes:
self.__on_sigint()
return stdout, stderr
sigint_handler = SigintHandler()
class Timer(object):
def __init__(self, timeout, fn):
self.completed = False
self._fn = fn
self._timer = threading.Timer(timeout,
self._onTimer) if timeout else None
def __enter__(self):
if self._timer:
self._timer.start()
return self
def __exit__(self, _type, _value, _traceback):
if self._timer:
self._timer.cancel()
def _onTimer(self):
self._fn()
self.completed = True
class ThreadPool(object):
def __init__(self, pool_size=None, timeout=None):
self.timeout = timeout
self._pool_size = pool_size or multiprocessing.cpu_count()
if sys.platform == 'win32':
# TODO(crbug.com/1190269) - we can't use more than 56 child
# processes on Windows or Python3 may hang.
self._pool_size = min(self._pool_size, 56)
self._messages = []
self._messages_lock = threading.Lock()
self._tests = []
self._tests_lock = threading.Lock()
self._nonparallel_tests = []
def _GetCommand(self, test):
vpython = 'vpython3'
if sys.platform == 'win32':
vpython += '.bat'
cmd = test.cmd
if cmd[0] == 'python':
cmd = list(cmd)
cmd[0] = vpython
elif cmd[0].endswith('.py'):
cmd = [vpython] + cmd
# On Windows, scripts on the current directory take precedence over
# PATH, so that when testing depot_tools on Windows, calling
# `vpython3.bat` will execute the copy of vpython of the depot_tools
# under test instead of the one in the bot. As a workaround, we run the
# tests from the parent directory instead.
if (cmd[0] == vpython and 'cwd' in test.kwargs
and os.path.basename(test.kwargs['cwd']) == 'depot_tools'):
test.kwargs['cwd'] = os.path.dirname(test.kwargs['cwd'])
cmd[1] = os.path.join('depot_tools', cmd[1])
return cmd
def _RunWithTimeout(self, cmd, stdin, kwargs):
p = subprocess.Popen(cmd, **kwargs)
with Timer(self.timeout, p.terminate) as timer:
stdout, _ = sigint_handler.wait(p, stdin)
stdout = stdout.decode('utf-8', 'ignore')
if timer.completed:
stdout = 'Process timed out after %ss\n%s' % (self.timeout,
stdout)
return p.returncode, stdout
def CallCommand(self, test, show_callstack=None):
"""Runs an external program.
This function converts invocation of .py files and invocations of 'python'
to vpython invocations.
"""
cmd = self._GetCommand(test)
try:
start = time_time()
returncode, stdout = self._RunWithTimeout(cmd, test.stdin,
test.kwargs)
duration = time_time() - start
except Exception:
duration = time_time() - start
return test.message(
'%s\n%s exec failure (%4.2fs)\n%s' %
(test.name, ' '.join(cmd), duration, traceback.format_exc()),
show_callstack=show_callstack)
if returncode != 0:
return test.message('%s\n%s (%4.2fs) failed\n%s' %
(test.name, ' '.join(cmd), duration, stdout),
show_callstack=show_callstack)
if test.info:
return test.info('%s\n%s (%4.2fs)' %
(test.name, ' '.join(cmd), duration),
show_callstack=show_callstack)
def AddTests(self, tests, parallel=True):
if parallel:
self._tests.extend(tests)
else:
self._nonparallel_tests.extend(tests)
def RunAsync(self):
self._messages = []
def _WorkerFn():
while True:
test = None
with self._tests_lock:
if not self._tests:
break
test = self._tests.pop()
result = self.CallCommand(test, show_callstack=False)
if result:
with self._messages_lock:
self._messages.append(result)
def _StartDaemon():
t = threading.Thread(target=_WorkerFn)
t.daemon = True
t.start()
return t
while self._nonparallel_tests:
test = self._nonparallel_tests.pop()
result = self.CallCommand(test)
if result:
self._messages.append(result)
if self._tests:
threads = [_StartDaemon() for _ in range(self._pool_size)]
for worker in threads:
worker.join()
return self._messages
def normpath(path):
"""Version of os.path.normpath that also changes backward slashes to
forward slashes when not running on Windows.
"""
# This is safe to always do because the Windows version of os.path.normpath
# will replace forward slashes with backward slashes.
path = path.replace(os.sep, '/')
return os.path.normpath(path)
def _RightHandSideLinesImpl(affected_files):
"""Implements RightHandSideLines for InputApi and GclChange."""
for af in affected_files:
lines = af.ChangedContents()
for line in lines:
yield (af, line[0], line[1])
def prompt_should_continue(prompt_string):
sys.stdout.write(prompt_string)
sys.stdout.flush()
response = sys.stdin.readline().strip().lower()
return response in ('y', 'yes')
# Top level object so multiprocessing can pickle
# Public access through OutputApi object.
class _PresubmitResult(object):
"""Base class for result objects."""
fatal = False
should_prompt = False
def __init__(self, message, items=None, long_text='', show_callstack=None):
"""
message: A short one-line message to indicate errors.
items: A list of short strings to indicate where errors occurred.
long_text: multi-line text output, e.g. from another tool
"""
self._message = _PresubmitResult._ensure_str(message)
self._items = items or []
self._long_text = _PresubmitResult._ensure_str(long_text.rstrip())
if show_callstack is None:
show_callstack = _SHOW_CALLSTACKS
if show_callstack:
self._long_text += 'Presubmit result call stack is:\n'
self._long_text += ''.join(traceback.format_stack(None, 8))
@staticmethod
def _ensure_str(val):
"""
val: A "stringish" value. Can be any of str or bytes.
returns: A str after applying encoding/decoding as needed.
Assumes/uses UTF-8 for relevant inputs/outputs.
"""
if isinstance(val, str):
return val
if isinstance(val, bytes):
return val.decode()
raise ValueError("Unknown string type %s" % type(val))
def handle(self):
sys.stdout.write(self._message)
sys.stdout.write('\n')
for item in self._items:
sys.stdout.write(' ')
# Write separately in case it's unicode.
sys.stdout.write(str(item))
sys.stdout.write('\n')
if self._long_text:
sys.stdout.write('\n***************\n')
# Write separately in case it's unicode.
sys.stdout.write(self._long_text)
sys.stdout.write('\n***************\n')
def json_format(self):
return {
'message': self._message,
'items': [str(item) for item in self._items],
'long_text': self._long_text,
'fatal': self.fatal
}
# Top level object so multiprocessing can pickle
# Public access through OutputApi object.
class _PresubmitError(_PresubmitResult):
"""A hard presubmit error."""
fatal = True
# Top level object so multiprocessing can pickle
# Public access through OutputApi object.
class _PresubmitPromptWarning(_PresubmitResult):
"""An warning that prompts the user if they want to continue."""
should_prompt = True
# Top level object so multiprocessing can pickle
# Public access through OutputApi object.
class _PresubmitNotifyResult(_PresubmitResult):
"""Just print something to the screen -- but it's not even a warning."""
# Top level object so multiprocessing can pickle
# Public access through OutputApi object.
class _MailTextResult(_PresubmitResult):
"""A warning that should be included in the review request email."""
def __init__(self, *args, **kwargs):
super(_MailTextResult, self).__init__()
raise NotImplementedError()
class GerritAccessor(object):
"""Limited Gerrit functionality for canned presubmit checks to work.
To avoid excessive Gerrit calls, caches the results.
"""
def __init__(self, url=None, project=None, branch=None):
self.host = urlparse.urlparse(url).netloc if url else None
self.project = project
self.branch = branch
self.cache = {}
self.code_owners_enabled = None
def _FetchChangeDetail(self, issue):
# Separate function to be easily mocked in tests.
try:
return gerrit_util.GetChangeDetail(
self.host, str(issue),
['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS'])
except gerrit_util.GerritError as e:
if e.http_status == 404:
raise Exception('Either Gerrit issue %s doesn\'t exist, or '
'no credentials to fetch issue details' % issue)
raise
def GetChangeInfo(self, issue):
"""Returns labels and all revisions (patchsets) for this issue.
The result is a dictionary according to Gerrit REST Api.
https://gerrit-review.googlesource.com/Documentation/rest-api.html
However, API isn't very clear what's inside, so see tests for example.
"""
assert issue
cache_key = int(issue)
if cache_key not in self.cache:
self.cache[cache_key] = self._FetchChangeDetail(issue)
return self.cache[cache_key]
def GetChangeDescription(self, issue, patchset=None):
"""If patchset is none, fetches current patchset."""
info = self.GetChangeInfo(issue)
# info is a reference to cache. We'll modify it here adding description
# to it to the right patchset, if it is not yet there.
# Find revision info for the patchset we want.
if patchset is not None:
for rev, rev_info in info['revisions'].items():
if str(rev_info['_number']) == str(patchset):
break
else:
raise Exception('patchset %s doesn\'t exist in issue %s' %
(patchset, issue))
else:
rev = info['current_revision']
rev_info = info['revisions'][rev]
return rev_info['commit']['message']
def GetDestRef(self, issue):
ref = self.GetChangeInfo(issue)['branch']
if not ref.startswith('refs/'):
# NOTE: it is possible to create 'refs/x' branch,
# aka 'refs/heads/refs/x'. However, this is ill-advised.
ref = 'refs/heads/%s' % ref
return ref
def _GetApproversForLabel(self, issue, label):
change_info = self.GetChangeInfo(issue)
label_info = change_info.get('labels', {}).get(label, {})
values = label_info.get('values', {}).keys()
if not values:
return []
max_value = max(int(v) for v in values)
return [
v for v in label_info.get('all', [])
if v.get('value', 0) == max_value
]
def IsBotCommitApproved(self, issue):
return bool(self._GetApproversForLabel(issue, 'Bot-Commit'))
def IsOwnersOverrideApproved(self, issue):
return bool(self._GetApproversForLabel(issue, 'Owners-Override'))
def GetChangeOwner(self, issue):
return self.GetChangeInfo(issue)['owner']['email']
def GetChangeReviewers(self, issue, approving_only=True):
changeinfo = self.GetChangeInfo(issue)
if approving_only:
reviewers = self._GetApproversForLabel(issue, 'Code-Review')
else:
reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', [])
return [r.get('email') for r in reviewers]
def UpdateDescription(self, description, issue):
gerrit_util.SetCommitMessage(self.host,
issue,
description,
notify='NONE')
def IsCodeOwnersEnabledOnRepo(self):
if self.code_owners_enabled is None:
self.code_owners_enabled = gerrit_util.IsCodeOwnersEnabledOnRepo(
self.host, self.project)
return self.code_owners_enabled
class OutputApi(object):
"""An instance of OutputApi gets passed to presubmit scripts so that they
can output various types of results.
"""
PresubmitResult = _PresubmitResult
PresubmitError = _PresubmitError
PresubmitPromptWarning = _PresubmitPromptWarning
PresubmitNotifyResult = _PresubmitNotifyResult
MailTextResult = _MailTextResult
def __init__(self, is_committing):
self.is_committing = is_committing
self.more_cc = []
def AppendCC(self, cc):
"""Appends a user to cc for this change."""
self.more_cc.append(cc)
def PresubmitPromptOrNotify(self, *args, **kwargs):
"""Warn the user when uploading, but only notify if committing."""
if self.is_committing:
return self.PresubmitNotifyResult(*args, **kwargs)
return self.PresubmitPromptWarning(*args, **kwargs)
class InputApi(object):
"""An instance of this object is passed to presubmit scripts so they can
know stuff about the change they're looking at.
"""
# Method could be a function
# pylint: disable=no-self-use
# File extensions that are considered source files from a style guide
# perspective. Don't modify this list from a presubmit script!
#
# Files without an extension aren't included in the list. If you want to
# filter them as source files, add r'(^|.*?[\\\/])[^.]+$' to the allow list.
# Note that ALL CAPS files are skipped in DEFAULT_FILES_TO_SKIP below.
DEFAULT_FILES_TO_CHECK = (
# C++ and friends
r'.+\.c$',
r'.+\.cc$',
r'.+\.cpp$',
r'.+\.h$',
r'.+\.m$',
r'.+\.mm$',
r'.+\.inl$',
r'.+\.asm$',
r'.+\.hxx$',
r'.+\.hpp$',
r'.+\.s$',
r'.+\.S$',
# Scripts
r'.+\.js$',
r'.+\.ts$',
r'.+\.py$',
r'.+\.sh$',
r'.+\.rb$',
r'.+\.pl$',
r'.+\.pm$',
# Other
r'.+\.java$',
r'.+\.mk$',
r'.+\.am$',
r'.+\.css$',
r'.+\.mojom$',
r'.+\.fidl$',
r'.+\.rs$',
)
# Path regexp that should be excluded from being considered containing
# source files. Don't modify this list from a presubmit script!
DEFAULT_FILES_TO_SKIP = (
r'testing_support[\\\/]google_appengine[\\\/].*',
r'.*\bexperimental[\\\/].*',
# Exclude third_party/.* but NOT third_party/{WebKit,blink}
# (crbug.com/539768 and crbug.com/836555).
r'.*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*',
# Output directories (just in case)
r'.*\bDebug[\\\/].*',
r'.*\bRelease[\\\/].*',
r'.*\bxcodebuild[\\\/].*',
r'.*\bout[\\\/].*',
# All caps files like README and LICENCE.
r'.*\b[A-Z0-9_]{2,}$',
# SCM (can happen in dual SCM configuration). (Slightly over aggressive)
r'(|.*[\\\/])\.git[\\\/].*',
r'(|.*[\\\/])\.svn[\\\/].*',
# There is no point in processing a patch file.
r'.+\.diff$',
r'.+\.patch$',
)
def __init__(self,
change,
presubmit_path,
is_committing,
verbose,
gerrit_obj,
dry_run=None,
thread_pool=None,
parallel=False,
no_diffs=False):
"""Builds an InputApi object.
Args:
change: A presubmit.Change object.
presubmit_path: The path to the presubmit script being processed.
is_committing: True if the change is about to be committed.
gerrit_obj: provides basic Gerrit codereview functionality.
dry_run: if true, some Checks will be skipped.
parallel: if true, all tests reported via input_api.RunTests for all
PRESUBMIT files will be run in parallel.
no_diffs: if true, implies that --files or --all was specified so some
checks can be skipped, and some errors will be messages.
"""
# Version number of the presubmit_support script.
self.version = [int(x) for x in __version__.split('.')]
self.change = change
self.is_committing = is_committing
self.gerrit = gerrit_obj
self.dry_run = dry_run
self.no_diffs = no_diffs
self.parallel = parallel
self.thread_pool = thread_pool or ThreadPool()
# We expose various modules and functions as attributes of the input_api
# so that presubmit scripts don't have to import them.
self.ast = ast
self.basename = os.path.basename
self.cpplint = cpplint
self.fnmatch = fnmatch
self.gclient_paths = gclient_paths
self.glob = glob.glob
self.json = json
self.logging = logging.getLogger('PRESUBMIT')
self.os_listdir = os.listdir
self.os_path = os.path
self.os_stat = os.stat
self.os_walk = os.walk
self.re = re
self.subprocess = subprocess
self.sys = sys
self.tempfile = tempfile
self.time = time
self.unittest = unittest
self.urllib_request = urllib_request
self.urllib_error = urllib_error
self.is_windows = sys.platform == 'win32'
# Set python_executable to 'vpython3' in order to allow scripts in other
# repos (e.g. src.git) to automatically pick up that repo's .vpython
# file, instead of inheriting the one in depot_tools.
self.python_executable = 'vpython3'
# Offer a python 3 executable for use during the migration off of python
# 2.
self.python3_executable = 'vpython3'
self.environ = os.environ
# InputApi.platform is the platform you're currently running on.
self.platform = sys.platform
self.cpu_count = multiprocessing.cpu_count()
if self.is_windows:
# TODO(crbug.com/1190269) - we can't use more than 56 child
# processes on Windows or Python3 may hang.
self.cpu_count = min(self.cpu_count, 56)
# The local path of the currently-being-processed presubmit script.
self._current_presubmit_path = os.path.dirname(presubmit_path)
# We carry the canned checks so presubmit scripts can easily use them.
self.canned_checks = presubmit_canned_checks
# Temporary files we must manually remove at the end of a run.
self._named_temporary_files = []
self.owners_client = None
if self.gerrit and not 'PRESUBMIT_SKIP_NETWORK' in self.environ:
try:
self.owners_client = owners_client.GetCodeOwnersClient(
host=self.gerrit.host,
project=self.gerrit.project,
branch=self.gerrit.branch)
except Exception as e:
print('Failed to set owners_client - %s' % str(e))
self.owners_finder = owners_finder.OwnersFinder
self.verbose = verbose
self.Command = CommandData
# Replace <hash_map> and <hash_set> as headers that need to be included
# with 'base/containers/hash_tables.h' instead.
# Access to a protected member _XX of a client class
# pylint: disable=protected-access
self.cpplint._re_pattern_templates = [
(a, b,
'base/containers/hash_tables.h') if header in ('<hash_map>',
'<hash_set>') else
(a, b, header) for (a, b, header) in cpplint._re_pattern_templates
]
def SetTimeout(self, timeout):
self.thread_pool.timeout = timeout
def PresubmitLocalPath(self):
"""Returns the local path of the presubmit script currently being run.
This is useful if you don't want to hard-code absolute paths in the
presubmit script. For example, It can be used to find another file
relative to the PRESUBMIT.py script, so the whole tree can be branched and
the presubmit script still works, without editing its content.
"""
return self._current_presubmit_path
def AffectedFiles(self, include_deletes=True, file_filter=None):
"""Same as input_api.change.AffectedFiles() except only lists files
(and optionally directories) in the same directory as the current presubmit
script, or subdirectories thereof. Note that files are listed using the OS
path separator, so backslashes are used as separators on Windows.
"""
dir_with_slash = normpath(self.PresubmitLocalPath())
# normpath strips trailing path separators, so the trailing separator
# has to be added after the normpath call.
if len(dir_with_slash) > 0:
dir_with_slash += os.path.sep
return list(
filter(
lambda x: normpath(x.AbsoluteLocalPath()).startswith(
dir_with_slash),
self.change.AffectedFiles(include_deletes, file_filter)))
def LocalPaths(self):
"""Returns local paths of input_api.AffectedFiles()."""
paths = [af.LocalPath() for af in self.AffectedFiles()]
logging.debug('LocalPaths: %s', paths)
return paths
def AbsoluteLocalPaths(self):
"""Returns absolute local paths of input_api.AffectedFiles()."""
return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
def AffectedTestableFiles(self, include_deletes=None, **kwargs):
"""Same as input_api.change.AffectedTestableFiles() except only lists files
in the same directory as the current presubmit script, or subdirectories
thereof.
"""
if include_deletes is not None:
warn('AffectedTestableFiles(include_deletes=%s)'
' is deprecated and ignored' % str(include_deletes),
category=DeprecationWarning,
stacklevel=2)
# pylint: disable=consider-using-generator
return [
x for x in self.AffectedFiles(include_deletes=False, **kwargs)
if x.IsTestableFile()
]
def AffectedTextFiles(self, include_deletes=None):
"""An alias to AffectedTestableFiles for backwards compatibility."""
return self.AffectedTestableFiles(include_deletes=include_deletes)
def FilterSourceFile(self,
affected_file,
files_to_check=None,
files_to_skip=None):
"""Filters out files that aren't considered 'source file'.
If files_to_check or files_to_skip is None, InputApi.DEFAULT_FILES_TO_CHECK
and InputApi.DEFAULT_FILES_TO_SKIP is used respectively.
affected_file.LocalPath() needs to re.match an entry in the files_to_check
list and not re.match any entries in the files_to_skip list.
'/' path separators should be used in the regular expressions and will work
on Windows as well as other platforms.
Note: Copy-paste this function to suit your needs or use a lambda function.
"""
if files_to_check is None:
files_to_check = self.DEFAULT_FILES_TO_CHECK
if files_to_skip is None:
files_to_skip = self.DEFAULT_FILES_TO_SKIP
def Find(affected_file, items):
local_path = affected_file.LocalPath()
for item in items:
if self.re.match(item, local_path):
return True
# Handle the cases where the files regex only handles /, but the
# local path uses \.
if self.is_windows and self.re.match(
item, local_path.replace('\\', '/')):
return True
return False
return (Find(affected_file, files_to_check)
and not Find(affected_file, files_to_skip))
def AffectedSourceFiles(self, source_file):
"""Filter the list of AffectedTestableFiles by the function source_file.
If source_file is None, InputApi.FilterSourceFile() is used.
"""
if not source_file:
source_file = self.FilterSourceFile
return list(filter(source_file, self.AffectedTestableFiles()))
def RightHandSideLines(self, source_file_filter=None):
"""An iterator over all text lines in 'new' version of changed files.
Only lists lines from new or modified text files in the change that are
contained by the directory of the currently executing presubmit script.
This is useful for doing line-by-line regex checks, like checking for
trailing whitespace.
Yields:
a 3 tuple:
the AffectedFile instance of the current file;
integer line number (1-based); and
the contents of the line as a string.
Note: The carriage return (LF or CR) is stripped off.
"""
files = self.AffectedSourceFiles(source_file_filter)
return _RightHandSideLinesImpl(files)
def ReadFile(self, file_item, mode='r'):
"""Reads an arbitrary file.
Deny reading anything outside the repository.
"""
if isinstance(file_item, AffectedFile):
file_item = file_item.AbsoluteLocalPath()
if not file_item.startswith(self.change.RepositoryRoot()):
raise IOError('Access outside the repository root is denied.')
return gclient_utils.FileRead(file_item, mode)
def CreateTemporaryFile(self, **kwargs):
"""Returns a named temporary file that must be removed with a call to
RemoveTemporaryFiles().
All keyword arguments are forwarded to tempfile.NamedTemporaryFile(),
except for |delete|, which is always set to False.
Presubmit checks that need to create a temporary file and pass it for
reading should use this function instead of NamedTemporaryFile(), as
Windows fails to open a file that is already open for writing.
with input_api.CreateTemporaryFile() as f:
f.write('xyz')
input_api.subprocess.check_output(['script-that', '--reads-from',
f.name])
Note that callers of CreateTemporaryFile() should not worry about removing
any temporary file; this is done transparently by the presubmit handling
code.
"""
if 'delete' in kwargs:
# Prevent users from passing |delete|; we take care of file deletion
# ourselves and this prevents unintuitive error messages when we
# pass delete=False and 'delete' is also in kwargs.
raise TypeError(
'CreateTemporaryFile() does not take a "delete" '
'argument, file deletion is handled automatically by '
'the same presubmit_support code that creates InputApi '
'objects.')
temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs)
self._named_temporary_files.append(temp_file.name)
return temp_file
def ListSubmodules(self):
"""Returns submodule paths for current change's repo."""
return self.change._repo_submodules()
@property
def tbr(self):
"""Returns if a change is TBR'ed."""
return 'TBR' in self.change.tags or self.change.TBRsFromDescription()
def RunTests(self, tests_mix, parallel=True):
tests = []
msgs = []
for t in tests_mix:
if isinstance(t, OutputApi.PresubmitResult) and t:
msgs.append(t)
else:
assert issubclass(t.message, _PresubmitResult)
tests.append(t)
if self.verbose:
t.info = _PresubmitNotifyResult
if not t.kwargs.get('cwd'):
t.kwargs['cwd'] = self.PresubmitLocalPath()
self.thread_pool.AddTests(tests, parallel)
# When self.parallel is True (i.e. --parallel is passed as an option)
# RunTests doesn't actually run tests. It adds them to a ThreadPool that
# will run all tests once all PRESUBMIT files are processed.
# Otherwise, it will run them and return the results.
if not self.parallel:
msgs.extend(self.thread_pool.RunAsync())
return msgs
class _DiffCache(object):
"""Caches diffs retrieved from a particular SCM."""
def GetDiff(self, path, local_root):
"""Get the diff for a particular path."""
raise NotImplementedError()
def GetOldContents(self, path, local_root):
"""Get the old version for a particular path."""
raise NotImplementedError()
class _GitDiffCache(_DiffCache):
"""DiffCache implementation for git; gets all file diffs at once."""
def __init__(self, upstream):
"""Stores the upstream revision against which all diffs are computed."""
super(_GitDiffCache, self).__init__()
self._upstream = upstream
self._diffs_by_file = None
def GetDiff(self, path, local_root):
# Compare against None to distinguish between None and an initialized
# but empty dictionary.
if self._diffs_by_file == None:
# Don't specify any filenames below, because there are command line
# length limits on some platforms and GenerateDiff would fail.
unified_diff = scm.GIT.GenerateDiff(local_root,
files=[],
full_move=True,
branch=self._upstream)
# Compute a single diff for all files and parse the output; with git
# this is much faster than computing one diff for each file.
self._diffs_by_file = _parse_unified_diff(unified_diff)
if path not in self._diffs_by_file:
# SCM didn't have any diff on this file. It could be that the file
# was not modified at all (e.g. user used --all flag in git cl
# presubmit). Intead of failing, return empty string. See:
# https://crbug.com/808346.
return ''
return self._diffs_by_file[path]
def GetOldContents(self, path, local_root):
return scm.GIT.GetOldContents(local_root, path, branch=self._upstream)
class _ProvidedDiffCache(_DiffCache):
"""Caches diffs from the provided diff file."""
def __init__(self, diff):
"""Stores all diffs and diffs per file."""
super(_ProvidedDiffCache, self).__init__()
self._diffs_by_file = None
self._diff = diff
def GetDiff(self, path, local_root):
"""Get the diff for a particular path."""
if self._diffs_by_file == None:
self._diffs_by_file = _parse_unified_diff(self._diff)
return self._diffs_by_file.get(path, '')
def GetOldContents(self, path, local_root):
"""Get the old version for a particular path."""
full_path = os.path.join(local_root, path)
diff = self.GetDiff(path, local_root)
is_file = os.path.isfile(full_path)
if not diff:
if is_file:
return gclient_utils.FileRead(full_path)
return ''
with gclient_utils.temporary_file() as diff_file:
gclient_utils.FileWrite(diff_file, diff)
try:
scm.GIT.Capture(['apply', '--reverse', '--check', diff_file],
cwd=local_root)
except subprocess.CalledProcessError:
raise RuntimeError('Provided diff does not apply cleanly.')
# Apply the reverse diff to a temporary file and read its contents.
with gclient_utils.temporary_directory() as tmp_dir:
copy_dst = os.path.join(tmp_dir, path)
os.makedirs(os.path.dirname(copy_dst), exist_ok=True)
if is_file:
shutil.copyfile(full_path, copy_dst)
scm.GIT.Capture([
'apply', '--reverse', '--directory', tmp_dir,
'--unsafe-paths', diff_file
],
cwd=tmp_dir)
# Applying the patch can create a new file if the file at
# full_path was deleted, so check if the new file at copy_dst
# exists.
if os.path.isfile(copy_dst):
return gclient_utils.FileRead(copy_dst)
return ''
class AffectedFile(object):
"""Representation of a file in a change."""
DIFF_CACHE = _DiffCache
# Method could be a function
# pylint: disable=no-self-use
def __init__(self, path, action, repository_root, diff_cache):
self._path = path
self._action = action
self._local_root = repository_root
self._is_directory = None
self._cached_changed_contents = None
self._cached_new_contents = None
self._diff_cache = diff_cache
self._is_testable_file = None
logging.debug('%s(%s)', self.__class__.__name__, self._path)
def LocalPath(self):
"""Returns the path of this file on the local disk relative to client root.
This should be used for error messages but not for accessing files,
because presubmit checks are run with CWD=PresubmitLocalPath() (which is
often != client root).
"""
return normpath(self._path)
def AbsoluteLocalPath(self):
"""Returns the absolute path of this file on the local disk."""
return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
def Action(self):
"""Returns the action on this opened file, e.g. A, M, D, etc."""
return self._action
def IsTestableFile(self):
"""Returns True if the file is a text file and not a binary file.
Deleted files are not text file."""
if self._is_testable_file is None:
if self.Action() == 'D':
# A deleted file is not testable.
self._is_testable_file = False
else:
t, _ = mimetypes.guess_type(self.AbsoluteLocalPath())
self._is_testable_file = bool(t and t.startswith('text/'))
return self._is_testable_file
def IsTextFile(self):
"""An alias to IsTestableFile for backwards compatibility."""
return self.IsTestableFile()
def OldContents(self):
"""Returns an iterator over the lines in the old version of file.
The old version is the file before any modifications in the user's
workspace, i.e. the 'left hand side'.
Contents will be empty if the file is a directory or does not exist.
Note: The carriage returns (LF or CR) are stripped off.
"""
return self._diff_cache.GetOldContents(self.LocalPath(),
self._local_root).splitlines()
def NewContents(self, flush_cache=False):
"""Returns an iterator over the lines in the new version of file.
The new version is the file in the user's workspace, i.e. the 'right hand
side'.
If flush_cache is True, read from disk and replace any cached contents.
Contents will be empty if the file is a directory or does not exist.
Note: The carriage returns (LF or CR) are stripped off.
"""
if self._cached_new_contents is None or flush_cache:
self._cached_new_contents = []
try:
self._cached_new_contents = gclient_utils.FileRead(
self.AbsoluteLocalPath(), 'rU').splitlines()
except IOError:
pass # File not found? That's fine; maybe it was deleted.
except UnicodeDecodeError as e:
# log the filename since we're probably trying to read a binary
# file, and shouldn't be.
print('Error reading %s: %s' % (self.AbsoluteLocalPath(), e))
raise
return self._cached_new_contents[:]
def ChangedContents(self, keeplinebreaks=False):
"""Returns a list of tuples (line number, line text) of all new lines.
This relies on the scm diff output describing each changed code section
with a line of the form
^@@ <old line num>,<old size> <new line num>,<new size> @@$
"""
# Don't return cached results when line breaks are requested.
if not keeplinebreaks and self._cached_changed_contents is not None:
return self._cached_changed_contents[:]
result = []
line_num = 0
# The keeplinebreaks parameter to splitlines must be True or else the
# CheckForWindowsLineEndings presubmit will be a NOP.
for line in self.GenerateScmDiff().splitlines(keeplinebreaks):
m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
if m:
line_num = int(m.groups(1)[0])
continue
if line.startswith('+') and not line.startswith('++'):
result.append((line_num, line[1:]))
if not line.startswith('-'):
line_num += 1
# Don't cache results with line breaks.
if keeplinebreaks:
return result
self._cached_changed_contents = result
return self._cached_changed_contents[:]
def __str__(self):
return self.LocalPath()
def GenerateScmDiff(self):
return self._diff_cache.GetDiff(self.LocalPath(), self._local_root)
class GitAffectedFile(AffectedFile):
"""Representation of a file in a change out of a git checkout."""
# Method 'NNN' is abstract in class 'NNN' but is not overridden
# pylint: disable=abstract-method
DIFF_CACHE = _GitDiffCache
def __init__(self, *args, **kwargs):
AffectedFile.__init__(self, *args, **kwargs)
def IsTestableFile(self):
if self._is_testable_file is None:
if self.Action() == 'D':
# A deleted file is not testable.
self._is_testable_file = False
else:
self._is_testable_file = os.path.isfile(
self.AbsoluteLocalPath())
return self._is_testable_file
class ProvidedDiffAffectedFile(AffectedFile):
"""Representation of a file in a change described by a diff."""
DIFF_CACHE = _ProvidedDiffCache
class Change(object):
"""Describe a change.
Used directly by the presubmit scripts to query the current change being
tested.
Instance members:
tags: Dictionary of KEY=VALUE pairs found in the change description.
self.KEY: equivalent to tags['KEY']
"""
_AFFECTED_FILES = AffectedFile
# Matches key/value (or 'tag') lines in changelist descriptions.
TAG_LINE_RE = re.compile(
'^[ \t]*(?P<key>[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P<value>.*?)[ \t]*$')
scm = ''
def __init__(self, name, description, local_root, files, issue, patchset,
author):
if files is None:
files = []
self._name = name
# Convert root into an absolute path.
self._local_root = os.path.abspath(local_root)
self.issue = issue
self.patchset = patchset
self.author_email = author
self._full_description = ''
self.tags = {}
self._description_without_tags = ''
self.SetDescriptionText(description)
# List of submodule paths in the repo.
self._submodules = None
assert all((isinstance(f, (list, tuple)) and len(f) == 2)
for f in files), files
diff_cache = self._diff_cache()
self._affected_files = [
self._AFFECTED_FILES(path, action.strip(), self._local_root,
diff_cache) for action, path in files
]
def _diff_cache(self):
return self._AFFECTED_FILES.DIFF_CACHE()
def Name(self):
"""Returns the change name."""
return self._name
def DescriptionText(self):
"""Returns the user-entered changelist description, minus tags.
Any line in the user-provided description starting with e.g. 'FOO='
(whitespace permitted before and around) is considered a tag line. Such
lines are stripped out of the description this function returns.
"""
return self._description_without_tags
def FullDescriptionText(self):
"""Returns the complete changelist description including tags."""
return self._full_description
def SetDescriptionText(self, description):
"""Sets the full description text (including tags) to |description|.
Also updates the list of tags."""
self._full_description = description
# From the description text, build up a dictionary of key/value pairs
# plus the description minus all key/value or 'tag' lines.
description_without_tags = []
self.tags = {}
for line in self._full_description.splitlines():
m = self.TAG_LINE_RE.match(line)
if m:
self.tags[m.group('key')] = m.group('value')
else:
description_without_tags.append(line)
# Change back to text and remove whitespace at end.
self._description_without_tags = (
'\n'.join(description_without_tags).rstrip())
def AddDescriptionFooter(self, key, value):
"""Adds the given footer to the change description.
Args:
key: A string with the key for the git footer. It must conform to
the git footers format (i.e. 'List-Of-Tokens') and will be case
normalized so that each token is title-cased.
value: A string with the value for the git footer.
"""
description = git_footers.add_footer(self.FullDescriptionText(),
git_footers.normalize_name(key),
value)
self.SetDescriptionText(description)
def RepositoryRoot(self):
"""Returns the repository (checkout) root directory for this change,
as an absolute path.
"""
return self._local_root
def __getattr__(self, attr):
"""Return tags directly as attributes on the object."""
if not re.match(r'^[A-Z_]*$', attr):
raise AttributeError(self, attr)
return self.tags.get(attr)
def GitFootersFromDescription(self):
"""Return the git footers present in the description.
Returns:
footers: A dict of {footer: [values]} containing a multimap of the footers
in the change description.
"""
return git_footers.parse_footers(self.FullDescriptionText())
def BugsFromDescription(self):
"""Returns all bugs referenced in the commit description."""
bug_tags = ['BUG', 'FIXED']
tags = []
for tag in bug_tags:
values = self.tags.get(tag)
if values:
tags += [value.strip() for value in values.split(',')]
footers = []
parsed = self.GitFootersFromDescription()
unsplit_footers = parsed.get('Bug', []) + parsed.get('Fixed', [])
for unsplit_footer in unsplit_footers:
footers += [b.strip() for b in unsplit_footer.split(',')]
return sorted(set(tags + footers))
def ReviewersFromDescription(self):
"""Returns all reviewers listed in the commit description."""
# We don't support a 'R:' git-footer for reviewers; that is in metadata.
tags = [
r.strip() for r in self.tags.get('R', '').split(',') if r.strip()
]
return sorted(set(tags))
def TBRsFromDescription(self):
"""Returns all TBR reviewers listed in the commit description."""
tags = [
r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip()
]
# TODO(crbug.com/839208): Remove support for 'Tbr:' when TBRs are
# programmatically determined by self-CR+1s.
footers = self.GitFootersFromDescription().get('Tbr', [])
return sorted(set(tags + footers))
# TODO(crbug.com/753425): Delete these once we're sure they're unused.
@property
def BUG(self):
return ','.join(self.BugsFromDescription())
@property
def R(self):
return ','.join(self.ReviewersFromDescription())
@property
def TBR(self):
return ','.join(self.TBRsFromDescription())
def UpstreamBranch(self):
"""Returns the upstream branch for the change.
This is only applicable to Git changes.
"""
return None
def AllFiles(self, root=None):
"""List all files under source control in the repo."""
raise NotImplementedError()
def AffectedFiles(self, include_deletes=True, file_filter=None):
"""Returns a list of AffectedFile instances for all files in the change.
Args:
include_deletes: If false, deleted files will be filtered out.
file_filter: An additional filter to apply.
Returns:
[AffectedFile(path, action), AffectedFile(path, action)]
"""
affected = list(filter(file_filter, self._affected_files))
if include_deletes:
return affected
return list(filter(lambda x: x.Action() != 'D', affected))
def AffectedSubmodules(self):
"""Returns a list of AffectedFile instances for submodules in the change."""
return [
af for af in self._affected_files
if af.LocalPath() in self._repo_submodules()
]
def AffectedTestableFiles(self, include_deletes=None, **kwargs):
"""Return a list of the existing text files in a change."""
if include_deletes is not None:
warn('AffectedTestableFiles(include_deletes=%s)'
' is deprecated and ignored' % str(include_deletes),
category=DeprecationWarning,
stacklevel=2)
return list(
filter(lambda x: x.IsTestableFile(),
self.AffectedFiles(include_deletes=False, **kwargs)))
def AffectedTextFiles(self, include_deletes=None):
"""An alias to AffectedTestableFiles for backwards compatibility."""
return self.AffectedTestableFiles(include_deletes=include_deletes)
def LocalPaths(self):
"""Convenience function."""
return [af.LocalPath() for af in self.AffectedFiles()]
def LocalSubmodules(self):
"""Returns local paths for affected submodules."""
return [af.LocalPath() for af in self.AffectedSubmodules()]
def AbsoluteLocalPaths(self):
"""Convenience function."""
return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
def AbsoluteLocalSubmodules(self):
"""Returns absolute local paths for affected submodules"""
return [af.AbsoluteLocalPath() for af in self.AffectedSubmodules()]
def RightHandSideLines(self):
"""An iterator over all text lines in 'new' version of changed files.
Lists lines from new or modified text files in the change.
This is useful for doing line-by-line regex checks, like checking for
trailing whitespace.
Yields:
a 3 tuple:
the AffectedFile instance of the current file;
integer line number (1-based); and
the contents of the line as a string.
"""
return _RightHandSideLinesImpl(
x for x in self.AffectedFiles(include_deletes=False)
if x.IsTestableFile())
def OriginalOwnersFiles(self):
"""A map from path names of affected OWNERS files to their old content."""
def owners_file_filter(f):
return 'OWNERS' in os.path.split(f.LocalPath())[1]
files = self.AffectedFiles(file_filter=owners_file_filter)
return {f.LocalPath(): f.OldContents() for f in files}
def _repo_submodules(self):
"""Returns submodule paths for current change's repo."""
if not self._submodules:
self._submodules = scm.GIT.ListSubmodules(self.RepositoryRoot())
return self._submodules
class GitChange(Change):
_AFFECTED_FILES = GitAffectedFile
scm = 'git'
def __init__(self, *args, upstream, **kwargs):
self._upstream = upstream
super(GitChange, self).__init__(*args)
def _diff_cache(self):
return self._AFFECTED_FILES.DIFF_CACHE(self._upstream)
def UpstreamBranch(self):
"""Returns the upstream branch for the change."""
return self._upstream
def AllFiles(self, root=None):
"""List all files under source control in the repo."""
root = root or self.RepositoryRoot()
return subprocess.check_output(
['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'],
cwd=root).decode('utf-8', 'ignore').splitlines()
def AffectedFiles(self, include_deletes=True, file_filter=None):
"""Returns a list of AffectedFile instances for all files in the change.
Args:
include_deletes: If false, deleted files will be filtered out.
file_filter: An additional filter to apply.
Returns:
[AffectedFile(path, action), AffectedFile(path, action)]
"""
files = [
af for af in self._affected_files
if af.LocalPath() not in self._repo_submodules()
]
affected = list(filter(file_filter, files))
if include_deletes:
return affected
return list(filter(lambda x: x.Action() != 'D', affected))
class ProvidedDiffChange(Change):
_AFFECTED_FILES = ProvidedDiffAffectedFile
def __init__(self, *args, diff, **kwargs):
self._diff = diff
super(ProvidedDiffChange, self).__init__(*args)
def _diff_cache(self):
return self._AFFECTED_FILES.DIFF_CACHE(self._diff)
def AllFiles(self, root=None):
"""List all files under source control in the repo."""
root = root or self.RepositoryRoot()
return scm.DIFF.GetAllFiles(root)
def ListRelevantPresubmitFiles(files, root):
"""Finds all presubmit files that apply to a given set of source files.
If inherit-review-settings-ok is present right under root, looks for
PRESUBMIT.py in directories enclosing root.
Args:
files: An iterable container containing file paths.
root: Path where to stop searching.
Return:
List of absolute paths of the existing PRESUBMIT.py scripts.
"""
files = [normpath(os.path.join(root, f)) for f in files]
# List all the individual directories containing files.
directories = {os.path.dirname(f) for f in files}
# Ignore root if inherit-review-settings-ok is present.
if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
root = None
# Collect all unique directories that may contain PRESUBMIT.py.
candidates = set()
for directory in directories:
while True:
if directory in candidates:
break
candidates.add(directory)
if directory == root:
break
parent_dir = os.path.dirname(directory)
if parent_dir == directory:
# We hit the system root directory.
break
directory = parent_dir
# Look for PRESUBMIT.py in all candidate directories.
results = []
for directory in sorted(list(candidates)):
try:
for f in os.listdir(directory):
p = os.path.join(directory, f)
if os.path.isfile(p) and re.match(
r'PRESUBMIT.*\.py$',
f) and not f.startswith('PRESUBMIT_test'):
results.append(p)
except OSError:
pass
logging.debug('Presubmit files: %s', ','.join(results))
return results
class GetPostUploadExecuter(object):
def __init__(self, change, gerrit_obj):
"""
Args:
change: The Change object.
gerrit_obj: provides basic Gerrit codereview functionality.
"""
self.change = change
self.gerrit = gerrit_obj
def ExecPresubmitScript(self, script_text, presubmit_path):
"""Executes PostUploadHook() from a single presubmit script.
Caller is responsible for validating whether the hook should be executed
and should only call this function if it should be.
Args:
script_text: The text of the presubmit script.
presubmit_path: Project script to run.
Return:
A list of results objects.
"""
# Change to the presubmit file's directory to support local imports.
presubmit_dir = os.path.dirname(presubmit_path)
main_path = os.getcwd()
try:
os.chdir(presubmit_dir)
return self._execute_with_local_working_directory(
script_text, presubmit_dir, presubmit_path)
finally:
# Return the process to the original working directory.
os.chdir(main_path)
def _execute_with_local_working_directory(self, script_text, presubmit_dir,
presubmit_path):
context = {}
try:
exec(
compile(script_text, presubmit_path, 'exec', dont_inherit=True),
context)
except Exception as e:
raise PresubmitFailure('"%s" had an exception.\n%s' %
(presubmit_path, e))
function_name = 'PostUploadHook'
if function_name not in context:
return {}
post_upload_hook = context[function_name]
if not len(inspect.getfullargspec(post_upload_hook)[0]) == 3:
raise PresubmitFailure(
'Expected function "PostUploadHook" to take three arguments.')
return post_upload_hook(self.gerrit, self.change, OutputApi(False))
def DoPostUploadExecuter(change, gerrit_obj, verbose):
"""Execute the post upload hook.
Args:
change: The Change object.
gerrit_obj: The GerritAccessor object.
verbose: Prints debug info.
"""
python_version = 'Python %s' % sys.version_info.major
sys.stdout.write('Running %s post upload checks ...\n' % python_version)
presubmit_files = ListRelevantPresubmitFiles(
change.LocalPaths() + change.LocalSubmodules(), change.RepositoryRoot())
if not presubmit_files and verbose:
sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
results = []
executer = GetPostUploadExecuter(change, gerrit_obj)
# The root presubmit file should be executed after the ones in
# subdirectories. i.e. the specific post upload hooks should run before the
# general ones. Thus, reverse the order provided by
# ListRelevantPresubmitFiles.
presubmit_files.reverse()
for filename in presubmit_files:
filename = os.path.abspath(filename)
# Accept CRLF presubmit script.
presubmit_script = gclient_utils.FileRead(filename).replace(
'\r\n', '\n')
if verbose:
sys.stdout.write('Running %s\n' % filename)
results.extend(executer.ExecPresubmitScript(presubmit_script, filename))
if not results:
return 0
sys.stdout.write('\n')
sys.stdout.write('** Post Upload Hook Messages **\n')
exit_code = 0
for result in results:
if result.fatal:
exit_code = 1
result.handle()
sys.stdout.write('\n')
return exit_code
class PresubmitExecuter(object):
def __init__(self,
change,
committing,
verbose,
gerrit_obj,
dry_run=None,
thread_pool=None,
parallel=False,
no_diffs=False):
"""
Args:
change: The Change object.
committing: True if 'git cl land' is running, False if
'git cl upload' is.
gerrit_obj: provides basic Gerrit codereview functionality.
dry_run: if true, some Checks will be skipped.
parallel: if true, all tests reported via input_api.RunTests for all
PRESUBMIT files will be run in parallel.
no_diffs: if true, implies that --files or --all was specified so some
checks can be skipped, and some errors will be messages.
"""
self.change = change
self.committing = committing
self.gerrit = gerrit_obj
self.verbose = verbose
self.dry_run = dry_run
self.more_cc = []
self.thread_pool = thread_pool
self.parallel = parallel
self.no_diffs = no_diffs
def ExecPresubmitScript(self, script_text, presubmit_path):
"""Executes a single presubmit script.
Caller is responsible for validating whether the hook should be executed
and should only call this function if it should be.
Args:
script_text: The text of the presubmit script.
presubmit_path: The path to the presubmit file (this will be
reported via input_api.PresubmitLocalPath()).
Return:
A list of result objects, empty if no problems.
"""
# Change to the presubmit file's directory to support local imports.
presubmit_dir = os.path.dirname(presubmit_path)
main_path = os.getcwd()
try:
os.chdir(presubmit_dir)
return self._execute_with_local_working_directory(
script_text, presubmit_dir, presubmit_path)
finally:
# Return the process to the original working directory.
os.chdir(main_path)
def _execute_with_local_working_directory(self, script_text, presubmit_dir,
presubmit_path):
# Load the presubmit script into context.
input_api = InputApi(self.change,
presubmit_path,
self.committing,
self.verbose,
gerrit_obj=self.gerrit,
dry_run=self.dry_run,
thread_pool=self.thread_pool,
parallel=self.parallel,
no_diffs=self.no_diffs)
output_api = OutputApi(self.committing)
context = {}
try:
exec(
compile(script_text, presubmit_path, 'exec', dont_inherit=True),
context)
except Exception as e:
raise PresubmitFailure('"%s" had an exception.\n%s' %
(presubmit_path, e))
context['__args'] = (input_api, output_api)
# Get path of presubmit directory relative to repository root.
# Always use forward slashes, so that path is same in *nix and Windows
root = input_api.change.RepositoryRoot()
rel_path = os.path.relpath(presubmit_dir, root)
rel_path = rel_path.replace(os.path.sep, '/')
# Get the URL of git remote origin and use it to identify host and
# project
host = project = ''
if self.gerrit:
host = self.gerrit.host or ''
project = self.gerrit.project or ''
# Prefix for test names
prefix = 'presubmit:%s/%s:%s/' % (host, project, rel_path)
# Perform all the desired presubmit checks.
results = []
try:
version = [
int(x)
for x in context.get('PRESUBMIT_VERSION', '0.0.0').split('.')
]
with rdb_wrapper.client(prefix) as sink:
if version >= [2, 0, 0]:
# Copy the keys to prevent "dictionary changed size during
# iteration" exception if checks add globals to context.
# E.g. sometimes the Python runtime will add
# __warningregistry__.
for function_name in list(context.keys()):
if not function_name.startswith('Check'):
continue
if function_name.endswith(
'Commit') and not self.committing:
continue
if function_name.endswith('Upload') and self.committing:
continue
logging.debug('Running %s in %s', function_name,
presubmit_path)
results.extend(
self._run_check_function(function_name, context,
sink, presubmit_path))
logging.debug('Running %s done.', function_name)
self.more_cc.extend(output_api.more_cc)
# Clear the CC list between running each presubmit check
# to prevent CCs from being repeatedly appended.
output_api.more_cc = []
else: # Old format
if self.committing:
function_name = 'CheckChangeOnCommit'
else:
function_name = 'CheckChangeOnUpload'
if function_name in list(context.keys()):
logging.debug('Running %s in %s', function_name,
presubmit_path)
results.extend(
self._run_check_function(function_name, context,
sink, presubmit_path))
logging.debug('Running %s done.', function_name)
self.more_cc.extend(output_api.more_cc)
# Clear the CC list between running each presubmit check
# to prevent CCs from being repeatedly appended.
output_api.more_cc = []
finally:
for f in input_api._named_temporary_files:
os.remove(f)
self.more_cc = sorted(set(self.more_cc))
return results
def _run_check_function(self, function_name, context, sink, presubmit_path):
"""Evaluates and returns the result of a given presubmit function.
If sink is given, the result of the presubmit function will be reported
to the ResultSink.
Args:
function_name: the name of the presubmit function to evaluate
context: a context dictionary in which the function will be evaluated
sink: an instance of ResultSink. None, by default.
Returns:
the result of the presubmit function call.
"""
start_time = time_time()
try:
result = eval(function_name + '(*__args)', context)
self._check_result_type(result)
except Exception:
_, e_value, _ = sys.exc_info()
result = [
OutputApi.PresubmitError(
'Evaluation of %s failed: %s, %s' %
(function_name, e_value, traceback.format_exc()))
]
elapsed_time = time_time() - start_time
if elapsed_time > 10.0:
sys.stdout.write('%6.1fs to run %s from %s.\n' %
(elapsed_time, function_name, presubmit_path))
if sink:
status, failure_reason = RDBStatusFrom(result)
sink.report(function_name, status, elapsed_time, failure_reason)
return result
def _check_result_type(self, result):
"""Helper function which ensures result is a list, and all elements are
instances of OutputApi.PresubmitResult"""
if not isinstance(result, (tuple, list)):
raise PresubmitFailure(
'Presubmit functions must return a tuple or list')
if not all(
isinstance(res, OutputApi.PresubmitResult) for res in result):
raise PresubmitFailure(
'All presubmit results must be of types derived from '
'output_api.PresubmitResult')
def RDBStatusFrom(result):
"""Returns the status and failure reason for a PresubmitResult."""
failure_reason = None
status = rdb_wrapper.STATUS_PASS
if any(r.fatal for r in result):
status = rdb_wrapper.STATUS_FAIL
failure_reasons = []
for r in result:
fields = r.json_format()
message = fields['message']
items = '\n'.join(' %s' % item for item in fields['items'])
failure_reasons.append('\n'.join([message, items]))
if failure_reasons:
failure_reason = '\n'.join(failure_reasons)
return status, failure_reason
def DoPresubmitChecks(change,
committing,
verbose,
default_presubmit,
may_prompt,
gerrit_obj,
dry_run=None,
parallel=False,
json_output=None,
no_diffs=False):
"""Runs all presubmit checks that apply to the files in the change.
This finds all PRESUBMIT.py files in directories enclosing the files in the
change (up to the repository root) and calls the relevant entrypoint function
depending on whether the change is being committed or uploaded.
Prints errors, warnings and notifications. Prompts the user for warnings
when needed.
Args:
change: The Change object.
committing: True if 'git cl land' is running, False if 'git cl upload' is.
verbose: Prints debug info.
default_presubmit: A default presubmit script to execute in any case.
may_prompt: Enable (y/n) questions on warning or error. If False,
any questions are answered with yes by default.
gerrit_obj: provides basic Gerrit codereview functionality.
dry_run: if true, some Checks will be skipped.
parallel: if true, all tests specified by input_api.RunTests in all
PRESUBMIT files will be run in parallel.
no_diffs: if true, implies that --files or --all was specified so some
checks can be skipped, and some errors will be messages.
Return:
1 if presubmit checks failed or 0 otherwise.
"""
with setup_environ({'PYTHONDONTWRITEBYTECODE': '1'}):
python_version = 'Python %s' % sys.version_info.major
if committing:
sys.stdout.write('Running %s presubmit commit checks ...\n' %
python_version)
else:
sys.stdout.write('Running %s presubmit upload checks ...\n' %
python_version)
start_time = time_time()
presubmit_files = ListRelevantPresubmitFiles(
change.AbsoluteLocalPaths() + change.AbsoluteLocalSubmodules(),
change.RepositoryRoot())
if not presubmit_files and verbose:
sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
results = []
thread_pool = ThreadPool()
executer = PresubmitExecuter(change, committing, verbose, gerrit_obj,
dry_run, thread_pool, parallel, no_diffs)
if default_presubmit:
if verbose:
sys.stdout.write('Running default presubmit script.\n')
fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
results += executer.ExecPresubmitScript(default_presubmit,
fake_path)
for filename in presubmit_files:
filename = os.path.abspath(filename)
# Accept CRLF presubmit script.
presubmit_script = gclient_utils.FileRead(filename).replace(
'\r\n', '\n')
if verbose:
sys.stdout.write('Running %s\n' % filename)
results += executer.ExecPresubmitScript(presubmit_script, filename)
results += thread_pool.RunAsync()
messages = {}
should_prompt = False
presubmits_failed = False
for result in results:
if result.fatal:
presubmits_failed = True
messages.setdefault('ERRORS', []).append(result)
elif result.should_prompt:
should_prompt = True
messages.setdefault('Warnings', []).append(result)
else:
messages.setdefault('Messages', []).append(result)
# Print the different message types in a consistent order. ERRORS go
# last so that they will be most visible in the local-presubmit output.
for name in ['Messages', 'Warnings', 'ERRORS']:
if name in messages:
items = messages[name]
sys.stdout.write('** Presubmit %s: %d **\n' %
(name, len(items)))
for item in items:
item.handle()
sys.stdout.write('\n')
total_time = time_time() - start_time
if total_time > 1.0:
sys.stdout.write('Presubmit checks took %.1fs to calculate.\n' %
total_time)
if not should_prompt and not presubmits_failed:
sys.stdout.write('%s presubmit checks passed.\n\n' % python_version)
elif should_prompt and not presubmits_failed:
sys.stdout.write('There were %s presubmit warnings. ' %
python_version)
if may_prompt:
presubmits_failed = not prompt_should_continue(
'Are you sure you wish to continue? (y/N): ')
else:
sys.stdout.write('\n')
else:
sys.stdout.write('There were %s presubmit errors.\n' %
python_version)
if json_output:
# Write the presubmit results to json output
presubmit_results = {
'errors':
[error.json_format() for error in messages.get('ERRORS', [])],
'notifications': [
notification.json_format()
for notification in messages.get('Messages', [])
],
'warnings': [
warning.json_format()
for warning in messages.get('Warnings', [])
],
'more_cc':
executer.more_cc,
}
gclient_utils.FileWrite(
json_output, json.dumps(presubmit_results, sort_keys=True))
global _ASKED_FOR_FEEDBACK
# Ask for feedback one time out of 5.
if (results and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
sys.stdout.write(
'Was the presubmit check useful? If not, run "git cl presubmit -v"\n'
'to figure out which PRESUBMIT.py was run, then run git blame\n'
'on the file to figure out who to ask for help.\n')
_ASKED_FOR_FEEDBACK = True
return 1 if presubmits_failed else 0
def _scan_sub_dirs(mask, recursive):
if not recursive:
return [x for x in glob.glob(mask) if x not in ('.svn', '.git')]
results = []
for root, dirs, files in os.walk('.'):
if '.svn' in dirs:
dirs.remove('.svn')
if '.git' in dirs:
dirs.remove('.git')
for name in files:
if fnmatch.fnmatch(name, mask):
results.append(os.path.join(root, name))
return results
def _parse_files(args, recursive):
logging.debug('Searching for %s', args)
files = []
for arg in args:
files.extend([('M', f) for f in _scan_sub_dirs(arg, recursive)])
return files
def _parse_change(parser, options):
"""Process change options.
Args:
parser: The parser used to parse the arguments from command line.
options: The arguments parsed from command line.
Returns:
A GitChange if the change root is a git repository, a ProvidedDiffChange
if a diff file is specified, or a Change otherwise.
"""
if options.all_files:
if options.files:
parser.error('<files> cannot be specified when --all-files is set.')
if options.diff_file:
parser.error(
'<diff_file> cannot be specified when --all-files is set.')
if options.diff_file and options.generate_diff:
parser.error(
'<diff_file> cannot be specified when <generate_diff> is set.')
change_scm = scm.determine_scm(options.root)
if change_scm == 'diff' and not (options.files or options.all_files
or options.diff_file):
parser.error('unversioned directories must specify '
'<files>, <all_files>, or <diff_file>.')
diff = None
if options.files:
if options.source_controlled_only:
# Get the filtered set of files from SCM.
change_files = []
for name in scm.GIT.GetAllFiles(options.root):
for mask in options.files:
if fnmatch.fnmatch(name, mask):
change_files.append(('M', name))
break
elif options.generate_diff:
gerrit_url = urlparse.urlparse(options.gerrit_url).netloc
diffs = presubmit_diff.create_diffs(
host=gerrit_url.split('-review')[0],
repo=options.gerrit_project,
ref=options.upstream,
root=options.root,
files=options.files,
)
diff = '\n'.join(diffs.values())
change_files = _diffs_to_change_files(diffs)
else:
# Get the filtered set of files from a directory scan.
change_files = _parse_files(options.files, options.recursive)
elif options.all_files:
if change_scm == 'git':
all_files = scm.GIT.GetAllFiles(options.root)
else:
all_files = scm.DIFF.GetAllFiles(options.root)
change_files = [('M', f) for f in all_files]
elif options.diff_file:
diff, change_files = _process_diff_file(options.diff_file)
else:
change_files = scm.GIT.CaptureStatus(options.root,
options.upstream or None,
ignore_submodules=False)
logging.info('Found %d file(s).', len(change_files))
change_args = [
options.name, options.description, options.root, change_files,
options.issue, options.patchset, options.author
]
if diff:
return ProvidedDiffChange(*change_args, diff=diff)
if change_scm == 'git':
return GitChange(*change_args, upstream=options.upstream)
return Change(*change_args)
def _parse_gerrit_options(parser, options):
"""Process gerrit options.
SIDE EFFECTS: Modifies options.author and options.description from Gerrit if
options.gerrit_fetch is set.
Args:
parser: The parser used to parse the arguments from command line.
options: The arguments parsed from command line.
Returns:
A GerritAccessor object if options.gerrit_url is set, or None otherwise.
"""
gerrit_obj = None
if options.gerrit_url:
gerrit_obj = GerritAccessor(url=options.gerrit_url,
project=options.gerrit_project,
branch=options.gerrit_branch)
if not options.gerrit_fetch:
return gerrit_obj
if not options.gerrit_url or not options.issue or not options.patchset:
parser.error(
'--gerrit_fetch requires --gerrit_url, --issue and --patchset.')
options.author = gerrit_obj.GetChangeOwner(options.issue)
options.description = gerrit_obj.GetChangeDescription(
options.issue, options.patchset)
logging.info('Got author: "%s"', options.author)
logging.info('Got description: """\n%s\n"""', options.description)
return gerrit_obj
def _parse_unified_diff(diff):
"""Parses a unified git diff and returns a list of (path, diff) tuples."""
diffs = {}
# This regex matches the path twice, separated by a space. Note that
# filename itself may contain spaces.
file_marker = re.compile(
'^diff --git (?:a/)?(?P<filename>.*) (?:b/)?(?P=filename)$')
current_diff = []
keep_line_endings = True
for x in diff.splitlines(keep_line_endings):
match = file_marker.match(x)
if match:
# Marks the start of a new per-file section.
diffs[match.group('filename')] = current_diff = [x]
elif x.startswith('diff --git'):
raise PresubmitFailure('Unexpected diff line: %s' % x)
else:
current_diff.append(x)
return dict((normpath(path), ''.join(diff)) for path, diff in diffs.items())
def _process_diff_file(diff_file):
diff = gclient_utils.FileRead(diff_file)
if not diff:
raise PresubmitFailure('diff file is empty')
return diff, _diffs_to_change_files(_parse_unified_diff(diff))
def _diffs_to_change_files(diffs):
"""Validates a dict of diffs and processes it into a list of change files.
Each change file is a tuple of (action, path) where action is one of:
* A: newly added file
* M: modified file
* D: deleted file
Args:
diffs: Dict of (path, diff) tuples.
Returns:
A list of change file tuples from the diffs.
Raises:
PresubmitFailure: If a diff is invalid.
"""
change_files = []
for file, file_diff in diffs.items():
if not file_diff:
# If a file is modified such that its contents are the same as the
# upstream commit, it may not have a diff. For example, if you added
# a newline to a file in PS1, then deleted it in PS2, the diff will
# be empty. Add this to change_files as modified anyway.
change_files.append(('M', file))
continue
header_line = file_diff.splitlines()[1]
if not header_line:
raise PresubmitFailure('diff header is empty')
if header_line.startswith('new'):
action = 'A'
elif header_line.startswith('deleted'):
action = 'D'
else:
action = 'M'
change_files.append((action, file))
return change_files
@contextlib.contextmanager
def setup_environ(kv: Mapping[str, str]):
"""Update environment while in context, and reset back to original on exit.
Example usage:
with setup_environ({"key": "value"}):
# os.environ now has key set to value.
pass
"""
old_kv = {}
for k, v in kv.items():
old_kv[k] = os.environ.get(k, None)
os.environ[k] = v
yield
for k, v in old_kv.items():
if v:
os.environ[k] = v
else:
os.environ.pop(k, None)
@contextlib.contextmanager
def canned_check_filter(method_names):
filtered = {}
try:
for method_name in method_names:
if not hasattr(presubmit_canned_checks, method_name):
logging.warning('Skipping unknown "canned" check %s' %
method_name)
continue
filtered[method_name] = getattr(presubmit_canned_checks,
method_name)
setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: [])
yield
finally:
for name, method in filtered.items():
setattr(presubmit_canned_checks, name, method)
def main(argv=None):
parser = argparse.ArgumentParser(usage='%(prog)s [options] <files...>')
hooks = parser.add_mutually_exclusive_group()
hooks.add_argument('-c',
'--commit',
action='store_true',
help='Use commit instead of upload checks.')
hooks.add_argument('-u',
'--upload',
action='store_false',
dest='commit',
help='Use upload instead of commit checks.')
hooks.add_argument('--post_upload',
action='store_true',
help='Run post-upload commit hooks.')
parser.add_argument('-r',
'--recursive',
action='store_true',
help='Act recursively.')
parser.add_argument('-v',
'--verbose',
action='count',
default=0,
help='Use 2 times for more debug info.')
parser.add_argument('--name', default='no name')
parser.add_argument('--author')
desc = parser.add_mutually_exclusive_group()
desc.add_argument('--description',
default='',
help='The change description.')
desc.add_argument('--description_file',
help='File to read change description from.')
parser.add_argument('--diff_file', help='File to read change diff from.')
parser.add_argument('--generate_diff',
action='store_true',
help='Create a diff using upstream server as base.')
parser.add_argument('--issue', type=int, default=0)
parser.add_argument('--patchset', type=int, default=0)
parser.add_argument('--root',
default=os.getcwd(),
help='Search for PRESUBMIT.py up to this directory. '
'If inherit-review-settings-ok is present in this '
'directory, parent directories up to the root file '
'system directories will also be searched.')
parser.add_argument('--upstream',
help='The base ref or upstream branch against '
'which the diff should be computed.')
parser.add_argument('--default_presubmit')
parser.add_argument('--may_prompt', action='store_true', default=False)
parser.add_argument(
'--skip_canned',
action='append',
default=[],
help='A list of checks to skip which appear in '
'presubmit_canned_checks. Can be provided multiple times '
'to skip multiple canned checks.')
parser.add_argument('--dry_run',
action='store_true',
help=argparse.SUPPRESS)
parser.add_argument('--gerrit_url', help=argparse.SUPPRESS)
parser.add_argument('--gerrit_project', help=argparse.SUPPRESS)
parser.add_argument('--gerrit_branch', help=argparse.SUPPRESS)
parser.add_argument('--gerrit_fetch',
action='store_true',
help=argparse.SUPPRESS)
parser.add_argument('--parallel',
action='store_true',
help='Run all tests specified by input_api.RunTests in '
'all PRESUBMIT files in parallel.')
parser.add_argument('--json_output',
help='Write presubmit errors to json output.')
parser.add_argument('--all_files',
action='store_true',
help='Mark all files under source control as modified.')
parser.add_argument('files',
nargs='*',
help='List of files to be marked as modified when '
'executing presubmit or post-upload hooks. fnmatch '
'wildcards can also be used.')
parser.add_argument('--source_controlled_only',
action='store_true',
help='Constrain \'files\' to those in source control.')
parser.add_argument('--no_diffs',
action='store_true',
help='Assume that all "modified" files have no diffs.')
options = parser.parse_args(argv)
log_level = logging.ERROR
if options.verbose >= 2:
log_level = logging.DEBUG
elif options.verbose:
log_level = logging.INFO
log_format = ('[%(levelname).1s%(asctime)s %(process)d %(thread)d '
'%(filename)s] %(message)s')
logging.basicConfig(format=log_format, level=log_level)
# Print call stacks when _PresubmitResult objects are created with -v -v is
# specified. This helps track down where presubmit messages are coming from.
if options.verbose >= 2:
global _SHOW_CALLSTACKS
_SHOW_CALLSTACKS = True
if options.description_file:
options.description = gclient_utils.FileRead(options.description_file)
gerrit_obj = _parse_gerrit_options(parser, options)
change = _parse_change(parser, options)
try:
if options.post_upload:
return DoPostUploadExecuter(change, gerrit_obj, options.verbose)
with canned_check_filter(options.skip_canned):
return DoPresubmitChecks(change, options.commit, options.verbose,
options.default_presubmit,
options.may_prompt, gerrit_obj,
options.dry_run, options.parallel,
options.json_output, options.no_diffs)
except PresubmitFailure as e:
import utils
print(e, file=sys.stderr)
print('Maybe your depot_tools is out of date?', file=sys.stderr)
print('depot_tools version: %s' % utils.depot_tools_version(),
file=sys.stderr)
return 2
if __name__ == '__main__':
try:
sys.exit(main())
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
sys.exit(2)