# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Defines class Rietveld to easily access a rietveld instance.
Security implications:
The following hypothesis are made:
- Rietveld enforces:
- Nobody else than issue owner can upload a patch set
- Verifies the issue owner credentials when creating new issues
- A issue owner can't change once the issue is created
- A patch set cannot be modified
"""
import logging
import os
import re
import sys
import time
import urllib2
try:
import simplejson as json # pylint: disable=F0401
except ImportError:
try:
import json # pylint: disable=F0401
except ImportError:
# Import the one included in depot_tools.
sys.path.append(os.path.join(os.path.dirname(__file__), 'third_party'))
import simplejson as json # pylint: disable=F0401
from third_party import upload
import patch
# Hack out upload logging.info()
upload.logging = logging.getLogger('upload')
# Mac pylint choke on this line.
upload.logging.setLevel(logging.WARNING) # pylint: disable=E1103
class Rietveld(object):
"""Accesses rietveld."""
def __init__(self, url, email, password, extra_headers=None):
self.url = url.rstrip('/')
# TODO(maruel): It's not awesome but maybe necessary to retrieve the value.
# It happens when the presubmit check is ran out of process, the cookie
# needed to be recreated from the credentials. Instead, it should pass the
# email and the cookie.
self.email = email
self.password = password
if email and password:
get_creds = lambda: (email, password)
self.rpc_server = upload.HttpRpcServer(
self.url,
get_creds,
extra_headers=extra_headers or {})
else:
self.rpc_server = upload.GetRpcServer(url, email)
self._xsrf_token = None
self._xsrf_token_time = None
def xsrf_token(self):
if (not self._xsrf_token_time or
(time.time() - self._xsrf_token_time) > 30*60):
self._xsrf_token_time = time.time()
self._xsrf_token = self.get(
'/xsrf_token',
extra_headers={'X-Requesting-XSRF-Token': '1'})
return self._xsrf_token
def get_pending_issues(self):
"""Returns an array of dict of all the pending issues on the server."""
return json.loads(self.get(
'/search?format=json&commit=2&closed=3&keys_only=True&limit=1000')
)['results']
def close_issue(self, issue):
"""Closes the Rietveld issue for this changelist."""
logging.info('closing issue %s' % issue)
self.post("/%d/close" % issue, [('xsrf_token', self.xsrf_token())])
def get_description(self, issue):
"""Returns the issue's description."""
return self.get('/%d/description' % issue)
def get_issue_properties(self, issue, messages):
"""Returns all the issue's metadata as a dictionary."""
url = '/api/%s' % issue
if messages:
url += '?messages=true'
return json.loads(self.get(url))
def get_patchset_properties(self, issue, patchset):
"""Returns the patchset properties."""
url = '/api/%s/%s' % (issue, patchset)
return json.loads(self.get(url))
def get_file_content(self, issue, patchset, item):
"""Returns the content of a new file.
Throws HTTP 302 exception if the file doesn't exist or is not a binary file.
"""
# content = 0 is the old file, 1 is the new file.
content = 1
url = '/%s/image/%s/%s/%s' % (issue, patchset, item, content)
return self.get(url)
def get_file_diff(self, issue, patchset, item):
"""Returns the diff of the file.
Returns a useless diff for binary files.
"""
url = '/download/issue%s_%s_%s.diff' % (issue, patchset, item)
return self.get(url)
def get_patch(self, issue, patchset):
"""Returns a PatchSet object containing the details to apply this patch."""
props = self.get_patchset_properties(issue, patchset) or {}
out = []
for filename, state in props.get('files', {}).iteritems():
logging.debug('%s' % filename)
status = state.get('status')
if not status:
raise patch.UnsupportedPatchFormat(
filename, 'File\'s status is None, patchset upload is incomplete.')
if status[0] not in ('A', 'D', 'M'):
raise patch.UnsupportedPatchFormat(
filename, 'Change with status \'%s\' is not supported.' % status)
svn_props = self.parse_svn_properties(
state.get('property_changes', ''), filename)
if state.get('is_binary'):
if status[0] == 'D':
if status[0] != status.strip():
raise patch.UnsupportedPatchFormat(
filename, 'Deleted file shouldn\'t have property change.')
out.append(patch.FilePatchDelete(filename, state['is_binary']))
else:
out.append(patch.FilePatchBinary(
filename,
self.get_file_content(issue, patchset, state['id']),
svn_props,
is_new=(status[0] == 'A')))
continue
try:
diff = self.get_file_diff(issue, patchset, state['id'])
except urllib2.HTTPError, e:
if e.code == 404:
raise patch.UnsupportedPatchFormat(
filename, 'File doesn\'t have a diff.')
raise
# FilePatchDiff() will detect file deletion automatically.
p = patch.FilePatchDiff(filename, diff, svn_props)
out.append(p)
if status[0] == 'A':
# It won't be set for empty file.
p.is_new = True
if (len(status) > 1 and
status[1] == '+' and
not (p.source_filename or p.svn_properties)):
raise patch.UnsupportedPatchFormat(
filename, 'Failed to process the svn properties')
return patch.PatchSet(out)
@staticmethod
def parse_svn_properties(rietveld_svn_props, filename):
"""Returns a list of tuple [('property', 'newvalue')].
rietveld_svn_props is the exact format from 'svn diff'.
"""
rietveld_svn_props = rietveld_svn_props.splitlines()
svn_props = []
if not rietveld_svn_props:
return svn_props
# 1. Ignore svn:mergeinfo.
# 2. Accept svn:eol-style and svn:executable.
# 3. Refuse any other.
# \n
# Added: svn:ignore\n
# + LF\n
spacer = rietveld_svn_props.pop(0)
if spacer or not rietveld_svn_props:
# svn diff always put a spacer between the unified diff and property
# diff
raise patch.UnsupportedPatchFormat(
filename, 'Failed to parse svn properties.')
while rietveld_svn_props:
# Something like 'Added: svn:eol-style'. Note the action is localized.
# *sigh*.
action = rietveld_svn_props.pop(0)
match = re.match(r'^(\w+): (.+)$', action)
if not match or not rietveld_svn_props:
raise patch.UnsupportedPatchFormat(
filename, 'Failed to parse svn properties.')
if match.group(2) == 'svn:mergeinfo':
# Silently ignore the content.
rietveld_svn_props.pop(0)
continue
if match.group(1) not in ('Added', 'Modified'):
# Will fail for our French friends.
raise patch.UnsupportedPatchFormat(
filename, 'Unsupported svn property operation.')
if match.group(2) in ('svn:eol-style', 'svn:executable'):
# ' + foo' where foo is the new value. That's fragile.
content = rietveld_svn_props.pop(0)
match2 = re.match(r'^ \+ (.*)$', content)
if not match2:
raise patch.UnsupportedPatchFormat(
filename, 'Unsupported svn property format.')
svn_props.append((match.group(2), match2.group(1)))
return svn_props
def update_description(self, issue, description):
"""Sets the description for an issue on Rietveld."""
logging.info('new description for issue %s' % issue)
self.post('/%s/description' % issue, [
('description', description),
('xsrf_token', self.xsrf_token())])
def add_comment(self, issue, message):
logging.info('issue %s; comment: %s' % (issue, message))
return self.post('/%s/publish' % issue, [
('xsrf_token', self.xsrf_token()),
('message', message),
('message_only', 'True'),
('send_mail', 'True'),
('no_redirect', 'True')])
def set_flag(self, issue, patchset, flag, value):
return self.post('/%s/edit_flags' % issue, [
('last_patchset', str(patchset)),
('xsrf_token', self.xsrf_token()),
(flag, value)])
def search(
self,
owner=None, reviewer=None,
base=None,
closed=None, private=None, commit=None,
created_before=None, created_after=None,
modified_before=None, modified_after=None,
per_request=None, keys_only=False,
with_messages=False):
"""Yields search results."""
# These are expected to be strings.
string_keys = {
'owner': owner,
'reviewer': reviewer,
'base': base,
'created_before': created_before,
'created_after': created_after,
'modified_before': modified_before,
'modified_after': modified_after,
}
# These are either None, False or True.
three_state_keys = {
'closed': closed,
'private': private,
'commit': commit,
}
url = '/search?format=json'
# Sort the keys mainly to ease testing.
for key in sorted(string_keys):
value = string_keys[key]
if value:
url += '&%s=%s' % (key, urllib2.quote(value))
for key in sorted(three_state_keys):
value = three_state_keys[key]
if value is not None:
url += '&%s=%d' % (key, int(value) + 1)
if keys_only:
url += '&keys_only=True'
if with_messages:
url += '&with_messages=True'
if per_request:
url += '&limit=%d' % per_request
cursor = ''
while True:
output = self.get(url + cursor)
if output.startswith('<'):
# It's an error message. Return as no result.
break
data = json.loads(output) or {}
if not data.get('results'):
break
for i in data['results']:
yield i
cursor = '&cursor=%s' % data['cursor']
def get(self, request_path, **kwargs):
kwargs.setdefault('payload', None)
return self._send(request_path, **kwargs)
def post(self, request_path, data, **kwargs):
ctype, body = upload.EncodeMultipartFormData(data, [])
return self._send(request_path, payload=body, content_type=ctype, **kwargs)
def _send(self, request_path, **kwargs):
"""Sends a POST/GET to Rietveld. Returns the response body."""
maxtries = 5
for retry in xrange(maxtries):
try:
logging.debug('%s' % request_path)
result = self.rpc_server.Send(request_path, **kwargs)
# Sometimes GAE returns a HTTP 200 but with HTTP 500 as the content. How
# nice.
return result
except urllib2.HTTPError, e:
if retry >= (maxtries - 1):
raise
if e.code not in (500, 502, 503):
raise
except urllib2.URLError, e:
if retry >= (maxtries - 1):
raise
if not 'Name or service not known' in e.reason:
# Usually internal GAE flakiness.
raise
# If reaching this line, loop again. Uses a small backoff.
time.sleep(1+maxtries*2)
# DEPRECATED.
Send = get