You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
66 lines
1.9 KiB
Python
66 lines
1.9 KiB
Python
# Copyright 2014 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
from recipe_engine import recipe_api
|
|
|
|
import urllib
|
|
|
|
class UrlApi(recipe_api.RecipeApi):
|
|
quote = staticmethod(urllib.quote)
|
|
urlencode = staticmethod(urllib.urlencode)
|
|
|
|
@recipe_api.non_step
|
|
def join(self, *parts):
|
|
return '/'.join(str(x).strip('/') for x in parts)
|
|
|
|
def fetch(self, url, step_name=None, attempts=None, headers=None, **kwargs):
|
|
"""Fetches data at given URL and returns it as str.
|
|
|
|
Args:
|
|
url: URL to fetch.
|
|
step_name: optional step name, 'fetch <url>' by default.
|
|
attempts: how many attempts to make (1 by default).
|
|
headers: a {header_name: value} dictionary for HTTP headers.
|
|
|
|
Returns:
|
|
Fetched data as string.
|
|
"""
|
|
fetch_result = self.fetch_to_file(
|
|
url, None, step_name=step_name, attempts=attempts, headers=headers,
|
|
**kwargs)
|
|
return fetch_result.raw_io.output_text
|
|
|
|
def fetch_to_file(self, url, path, step_name=None, attempts=None,
|
|
headers=None, **kwargs):
|
|
"""Fetches data at given URL and writes it to file.
|
|
|
|
Args:
|
|
url: URL to fetch.
|
|
path: path to write the fetched file to. You are responsible for deleting
|
|
it later if not needed.
|
|
step_name: optional step name, 'fetch <url>' by default.
|
|
attempts: how many attempts to make (1 by default).
|
|
headers: a {header_name: value} dictionary for HTTP headers.
|
|
|
|
Returns:
|
|
Step.
|
|
"""
|
|
if not step_name:
|
|
step_name = 'fetch %s' % url
|
|
args = [
|
|
url,
|
|
'--outfile', self.m.raw_io.output_text(leak_to=path),
|
|
]
|
|
|
|
if headers:
|
|
args.extend(['--headers-json', self.m.json.input(headers)])
|
|
|
|
if attempts:
|
|
args.extend(['--attempts', attempts])
|
|
return self.m.python(
|
|
name=step_name,
|
|
script=self.resource('pycurl.py'),
|
|
args=args,
|
|
**kwargs)
|