From 5fea38772fed9dd66cfe51b84afd0a5d2418f7e7 Mon Sep 17 00:00:00 2001 From: "thakis@chromium.org" Date: Wed, 10 Feb 2016 16:51:43 +0000 Subject: [PATCH] Revert of Finally get rid of depot_tools' breakpad. (patchset #2 id:20001 of https://codereview.chromium.org/1689633002/ ) Reason for revert: All chromium bots depend on this: ________ running '/usr/bin/python src/build/android/play_services/update.py download' in '/b/build/slave/linux/build' Traceback (most recent call last): File "src/build/android/play_services/update.py", line 29, in import find_depot_tools # pylint: disable=import-error,unused-import File "/b/build/slave/linux/build/src/build/find_depot_tools.py", line 49, in import breakpad ImportError: No module named breakpad https://www.google.com/url?hl=en&q=http://build.chromium.org/p/tryserver.chromium.linux/builders/chromium_presubmit/builds/144739&source=gmail&ust=1455209366639000&usg=AFQjCNH42SEVcJg4J6dX0J9HF0Rcqv81eA Original issue's description: > Finally get rid of depot_tools' breakpad. > > R=maruel@chromium.org > BUG= > > Committed: http://src.chromium.org/viewvc/chrome?view=rev&revision=298710 TBR=maruel@chromium.org,tandrii@chromium.org # Skipping CQ checks because original CL landed less than 1 days ago. NOPRESUBMIT=true NOTREECHECKS=true NOTRY=true BUG= Review URL: https://codereview.chromium.org/1683173002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@298712 0039d316-1c4b-4281-b951-d872f2087c98 --- apply_issue.py | 1 + breakpad.py | 147 ++++++++++++++++++++++++++++++++++++ commit_queue.py | 2 + drover.py | 5 +- gcl.py | 8 ++ gclient.py | 2 + git_cl.py | 7 ++ git_try.py | 2 + tests/breakpad_unittest.py | 123 ++++++++++++++++++++++++++++++ tests/git_cl_test.py | 6 ++ tests/trychange_unittest.py | 1 + trychange.py | 2 + 12 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 breakpad.py create mode 100755 tests/breakpad_unittest.py diff --git a/apply_issue.py b/apply_issue.py index b0d917631..7899fa89a 100755 --- a/apply_issue.py +++ b/apply_issue.py @@ -14,6 +14,7 @@ import subprocess import sys import urllib2 +import breakpad # pylint: disable=W0611 import annotated_gclient import auth diff --git a/breakpad.py b/breakpad.py new file mode 100644 index 000000000..2248e35c7 --- /dev/null +++ b/breakpad.py @@ -0,0 +1,147 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Breakpad for Python. + +Sends a notification when a process stops on an exception. + +It is only enabled when all these conditions are met: + 1. hostname finishes with '.google.com' or 'chromium.org' + 2. main module name doesn't contain the word 'test' + 3. no NO_BREAKPAD environment variable is defined +""" + +import atexit +import getpass +import os +import socket +import sys +import time +import traceback +import urllib +import urllib2 + + +# Configure these values. +DEFAULT_URL = 'https://chromium-status.appspot.com' + +# Global variable to prevent double registration. +_REGISTERED = False + +_TIME_STARTED = time.time() + +_HOST_NAME = socket.getfqdn() + +# Skip unit tests and we don't want anything from non-googler. +IS_ENABLED = ( + not 'test' in getattr(sys.modules['__main__'], '__file__', '') and + not 'NO_BREAKPAD' in os.environ and + _HOST_NAME.endswith(('.google.com', '.chromium.org'))) + + +def post(url, params): + """HTTP POST with timeout when it's supported.""" + if not IS_ENABLED: + # Make sure to not send anything for non googler. + return + kwargs = {} + if (sys.version_info[0] * 10 + sys.version_info[1]) >= 26: + kwargs['timeout'] = 4 + try: + request = urllib2.urlopen(url, urllib.urlencode(params), **kwargs) + out = request.read() + request.close() + return out + except IOError: + return 'There was a failure while trying to send the stack trace. Too bad.' + + +def FormatException(e): + """Returns a human readable form of an exception. + + Adds the maximum number of interesting information in the safest way.""" + try: + out = repr(e) + except Exception: + out = '' + try: + out = str(e) + if isinstance(e, Exception): + # urllib exceptions, usually the HTTP headers. + if hasattr(e, 'headers'): + out += '\nHeaders: %s' % e.headers + if hasattr(e, 'url'): + out += '\nUrl: %s' % e.url + if hasattr(e, 'msg'): + out += '\nMsg: %s' % e.msg + # The web page in some urllib exceptions. + if hasattr(e, 'read') and callable(e.read): + out += '\nread(): %s' % e.read() + if hasattr(e, 'info') and callable(e.info): + out += '\ninfo(): %s' % e.info() + except Exception: + pass + return out + + +def SendStack(last_tb, stack, url=None, maxlen=50, verbose=True): + """Sends the stack trace to the breakpad server.""" + if not IS_ENABLED: + return + def p(o): + if verbose: + print(o) + p('Sending crash report ...') + params = { + 'args': sys.argv, + 'cwd': os.getcwd(), + 'exception': FormatException(last_tb), + 'host': _HOST_NAME, + 'stack': stack[0:4096], + 'user': getpass.getuser(), + 'version': sys.version, + } + p('\n'.join(' %s: %s' % (k, params[k][0:maxlen]) for k in sorted(params))) + p(post(url or DEFAULT_URL + '/breakpad', params)) + + +def SendProfiling(duration, url=None): + params = { + 'argv': ' '.join(sys.argv), + # Strip the hostname. + 'domain': _HOST_NAME.split('.', 1)[-1], + 'duration': duration, + 'platform': sys.platform, + } + post(url or DEFAULT_URL + '/profiling', params) + + +def CheckForException(): + """Runs at exit. Look if there was an exception active.""" + last_value = getattr(sys, 'last_value', None) + if last_value: + if not isinstance(last_value, KeyboardInterrupt): + last_tb = getattr(sys, 'last_traceback', None) + if last_tb: + SendStack(last_value, ''.join(traceback.format_tb(last_tb))) + else: + duration = time.time() - _TIME_STARTED + if duration > 90: + SendProfiling(duration) + + +def Register(): + """Registers the callback at exit. Calling it multiple times is no-op.""" + global _REGISTERED + if _REGISTERED: + return + _REGISTERED = True + atexit.register(CheckForException) + + +if IS_ENABLED: + Register() + +# Uncomment this line if you want to test it out. +#Register() diff --git a/commit_queue.py b/commit_queue.py index eb667a49a..4e624773a 100755 --- a/commit_queue.py +++ b/commit_queue.py @@ -16,6 +16,8 @@ import os import sys import urllib2 +import breakpad # pylint: disable=W0611 + import auth import fix_encoding import rietveld diff --git a/drover.py b/drover.py index a73f1d347..87025300d 100755 --- a/drover.py +++ b/drover.py @@ -10,6 +10,7 @@ import re import sys import urlparse +import breakpad # pylint: disable=W0611 import gclient_utils import subprocess2 @@ -376,7 +377,7 @@ def getSVNAuthInfo(folder=None): try: for auth_file in os.listdir(svn_simple_folder): # Read the SVN auth file, convert it into a dictionary, and store it. - results[auth_file] = dict(re.findall(r'K [0-9]+\n(.*)\nV [0-9]+\n(.*)\n', + results[auth_file] = dict(re.findall(r'K [0-9]+\n(.*)\nV [0-9]+\n(.*)\n', open(os.path.join(svn_simple_folder, auth_file)).read())) except Exception as _: pass @@ -390,7 +391,7 @@ def getCurrentSVNUsers(url): auth_infos = getSVNAuthInfo() results = [] for _, auth_info in auth_infos.iteritems(): - if ('svn:realmstring' in auth_info + if ('svn:realmstring' in auth_info and netloc in auth_info['svn:realmstring']): username = auth_info['username'] results.append(username) diff --git a/gcl.py b/gcl.py index a80939673..1f88025dd 100755 --- a/gcl.py +++ b/gcl.py @@ -20,6 +20,8 @@ import tempfile import time import urllib2 +import breakpad # pylint: disable=W0611 + import auth import fix_encoding @@ -741,6 +743,12 @@ def GetTreeStatus(): def OptionallyDoPresubmitChecks(change_info, committing, args): if FilterFlag(args, "--no_presubmit") or FilterFlag(args, "--force"): + breakpad.SendStack( + breakpad.DEFAULT_URL + '/breakpad', + 'GclHooksBypassedCommit', + 'Issue %s/%s bypassed hook when committing (tree status was "%s")' % + (change_info.rietveld, change_info.issue, GetTreeStatus()), + verbose=False) return presubmit_support.PresubmitOutput() return DoPresubmitChecks(change_info, committing, True) diff --git a/gclient.py b/gclient.py index 7ba378a7d..507721f6f 100755 --- a/gclient.py +++ b/gclient.py @@ -95,6 +95,8 @@ import time import urllib import urlparse +import breakpad # pylint: disable=W0611 + import fix_encoding import gclient_scm import gclient_utils diff --git a/git_cl.py b/git_cl.py index 46e60fb20..5b5872bbd 100755 --- a/git_cl.py +++ b/git_cl.py @@ -40,6 +40,7 @@ from third_party import httplib2 from third_party import upload import auth from luci_hacks import trigger_luci_job as luci_trigger +import breakpad # pylint: disable=W0611 import clang_format import commit_queue import dart_format @@ -2618,6 +2619,12 @@ def SendUpstream(parser, args, cmd): print('Unable to determine tree status. Please verify manually and ' 'use "git cl %s --bypass-hooks" to commit on a closed tree.' % cmd) return 1 + else: + breakpad.SendStack( + 'GitClHooksBypassedCommit', + 'Issue %s/%s bypassed hook when committing (tree status was "%s")' % + (cl.GetRietveldServer(), cl.GetIssue(), GetTreeStatus()), + verbose=False) change_desc = ChangeDescription(options.message) if not change_desc.description and cl.GetIssue(): diff --git a/git_try.py b/git_try.py index e0fb7f6d6..f8f6d306a 100755 --- a/git_try.py +++ b/git_try.py @@ -7,6 +7,8 @@ import logging import sys +import breakpad # pylint: disable=W0611 + from scm import GIT import subprocess2 import third_party.upload diff --git a/tests/breakpad_unittest.py b/tests/breakpad_unittest.py new file mode 100755 index 000000000..72a7c11ca --- /dev/null +++ b/tests/breakpad_unittest.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for breakpad.py.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from testing_support.super_mox import SuperMoxTestBase + +import breakpad + + +class Breakpad(SuperMoxTestBase): + """Setups and tear downs the mocks but doesn't test anything as-is.""" + def setUp(self): + super(Breakpad, self).setUp() + self.mox.StubOutWithMock(breakpad.atexit, 'register') + self.mox.StubOutWithMock(breakpad.getpass, 'getuser') + self.mox.StubOutWithMock(breakpad.urllib2, 'urlopen') + breakpad._HOST_NAME = 'bozo' + self.assertEquals(False, breakpad.IS_ENABLED) + breakpad.IS_ENABLED = True + self._old_sys_argv = breakpad.sys.argv + breakpad.sys.argv = ['my_test'] + self._old_sys_version = breakpad.sys.version + breakpad.sys.version = 'random python' + + def tearDown(self): + breakpad.IS_ENABLED = False + breakpad.sys.version = self._old_sys_version + breakpad.sys.argv = self._old_sys_argv + super(Breakpad, self).tearDown() + + def testMembersChanged(self): + members = [ + 'CheckForException', 'DEFAULT_URL', 'FormatException', 'IS_ENABLED', + 'Register', 'SendProfiling', 'SendStack', + 'atexit', 'getpass', 'os', 'post', 'socket', 'sys', 'time', 'traceback', + 'urllib', 'urllib2', + ] + # If this test fails, you should add the relevant test. + self.compareMembers(breakpad, members) + + def _part_1_setup_mocks(self, exception): + breakpad.os.getcwd().AndReturn('/tmp/asdf') + breakpad.getpass.getuser().AndReturn('georges') + obj = self.mox.CreateMockAnything() + kwargs = {} + if (breakpad.sys.version_info[0] * 10 + breakpad.sys.version_info[1]) >= 26: + kwargs['timeout'] = 4 + breakpad.urllib2.urlopen( + 'https://chromium-status.appspot.com/breakpad', + breakpad.urllib.urlencode([('exception', exception)]) + ( + '&args=%5B%27my_test%27%5D' + '&stack=bar' + '&host=bozo' + '&version=random+python' + '&user=georges' + '&cwd=%2Ftmp%2Fasdf'), + **kwargs).AndReturn(obj) + obj.read().AndReturn('ok') + obj.close() + + def _part_2_verify_stdout(self, exception): + self.checkstdout( + ( "Sending crash report ...\n" + " args: ['my_test']\n" + " cwd: /tmp/asdf\n" + " exception: %s\n" + " host: bozo\n" + " stack: bar\n" + " user: georges\n" + " version: random python\n" + "ok\n") % exception) + + def _check(self, obj, result): + self._part_1_setup_mocks(result) + self.mox.ReplayAll() + breakpad.SendStack(obj, 'bar') + self._part_2_verify_stdout(result) + + def testSendBase(self): + self._check('foo', 'foo') + + def testSendReprThrows(self): + class Throws(object): + def __repr__(self): + raise NotImplementedError() + def __str__(self): + return '[foo]' + self._check(Throws(), '[foo]') + + def testSendStrThrows(self): + class Throws(object): + def __repr__(self): + return '[foo]' + def __str__(self): + raise NotImplementedError() + self._check(Throws(), '[foo]') + + def testSendBoth(self): + class Both(object): + def __repr__(self): + return '[foo]' + def __str__(self): + return '[bar]' + self._check(Both(), '[bar]') + + def testSendException(self): + obj = Exception('foo') + obj.msg = 'a message' + self._check(obj, 'foo\nMsg: a message') + + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/tests/git_cl_test.py b/tests/git_cl_test.py index c24a4d8fe..c62c1ed7b 100755 --- a/tests/git_cl_test.py +++ b/tests/git_cl_test.py @@ -88,6 +88,8 @@ class TestGitCl(TestCase): self.mock(git_cl, 'BranchExists', lambda _: True) self.mock(git_cl, 'FindCodereviewSettingsFile', lambda: '') self.mock(git_cl, 'ask_for_data', self._mocked_call) + self.mock(git_cl.breakpad, 'post', self._mocked_call) + self.mock(git_cl.breakpad, 'SendStack', self._mocked_call) self.mock(git_cl.presubmit_support, 'DoPresubmitChecks', PresubmitMock) self.mock(git_cl.rietveld, 'Rietveld', RietveldMock) self.mock(git_cl.rietveld, 'CachingRietveld', RietveldMock) @@ -331,6 +333,10 @@ class TestGitCl(TestCase): 'config', 'branch.working.rietveldissue'],), '12345'), ((['git', 'config', 'branch.working.rietveldserver'],), 'codereview.example.com'), + ((['git', 'config', 'rietveld.tree-status-url'],), ''), + (('GitClHooksBypassedCommit', + 'Issue https://codereview.example.com/12345 bypassed hook when ' + 'committing (tree status was "unset")'), None), ] @classmethod diff --git a/tests/trychange_unittest.py b/tests/trychange_unittest.py index c4917c0f5..d4e54e3ab 100755 --- a/tests/trychange_unittest.py +++ b/tests/trychange_unittest.py @@ -51,6 +51,7 @@ class TryChangeUnittest(TryChangeTestsBase): 'HELP_STRING', 'Error', 'InvalidScript', 'NoTryServerAccess', 'OptionParser', 'PrintSuccess', 'RunCommand', 'RunGit', 'SCM', 'SVN', 'TryChange', 'USAGE', 'contextlib', + 'breakpad', 'datetime', 'errno', 'fix_encoding', 'gcl', 'gclient_utils', 'gerrit_util', 'gen_parser', 'getpass', 'itertools', 'json', 'logging', 'optparse', 'os', 'posixpath', diff --git a/trychange.py b/trychange.py index 27a9e931f..03e59b37f 100755 --- a/trychange.py +++ b/trychange.py @@ -26,6 +26,8 @@ import urllib import urllib2 import urlparse +import breakpad # pylint: disable=W0611 + import fix_encoding import gcl import gclient_utils