You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
538 lines
17 KiB
Python
538 lines
17 KiB
Python
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
|
|
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
|
|
#
|
|
# This file is part of logilab-common.
|
|
#
|
|
# logilab-common is free software: you can redistribute it and/or modify it under
|
|
# the terms of the GNU Lesser General Public License as published by the Free
|
|
# Software Foundation, either version 2.1 of the License, or (at your option) any
|
|
# later version.
|
|
#
|
|
# logilab-common is distributed in the hope that it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
|
|
# details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public License along
|
|
# with logilab-common. If not, see <http://www.gnu.org/licenses/>.
|
|
"""Some text manipulation utility functions.
|
|
|
|
|
|
:group text formatting: normalize_text, normalize_paragraph, pretty_match,\
|
|
unquote, colorize_ansi
|
|
:group text manipulation: searchall, splitstrip
|
|
:sort: text formatting, text manipulation
|
|
|
|
:type ANSI_STYLES: dict(str)
|
|
:var ANSI_STYLES: dictionary mapping style identifier to ANSI terminal code
|
|
|
|
:type ANSI_COLORS: dict(str)
|
|
:var ANSI_COLORS: dictionary mapping color identifier to ANSI terminal code
|
|
|
|
:type ANSI_PREFIX: str
|
|
:var ANSI_PREFIX:
|
|
ANSI terminal code notifying the start of an ANSI escape sequence
|
|
|
|
:type ANSI_END: str
|
|
:var ANSI_END:
|
|
ANSI terminal code notifying the end of an ANSI escape sequence
|
|
|
|
:type ANSI_RESET: str
|
|
:var ANSI_RESET:
|
|
ANSI terminal code resetting format defined by a previous ANSI escape sequence
|
|
"""
|
|
__docformat__ = "restructuredtext en"
|
|
|
|
import sys
|
|
import re
|
|
import os.path as osp
|
|
from warnings import warn
|
|
from unicodedata import normalize as _uninormalize
|
|
try:
|
|
from os import linesep
|
|
except ImportError:
|
|
linesep = '\n' # gae
|
|
|
|
from logilab.common.deprecation import deprecated
|
|
|
|
MANUAL_UNICODE_MAP = {
|
|
u'\xa1': u'!', # INVERTED EXCLAMATION MARK
|
|
u'\u0142': u'l', # LATIN SMALL LETTER L WITH STROKE
|
|
u'\u2044': u'/', # FRACTION SLASH
|
|
u'\xc6': u'AE', # LATIN CAPITAL LETTER AE
|
|
u'\xa9': u'(c)', # COPYRIGHT SIGN
|
|
u'\xab': u'"', # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
|
|
u'\xe6': u'ae', # LATIN SMALL LETTER AE
|
|
u'\xae': u'(r)', # REGISTERED SIGN
|
|
u'\u0153': u'oe', # LATIN SMALL LIGATURE OE
|
|
u'\u0152': u'OE', # LATIN CAPITAL LIGATURE OE
|
|
u'\xd8': u'O', # LATIN CAPITAL LETTER O WITH STROKE
|
|
u'\xf8': u'o', # LATIN SMALL LETTER O WITH STROKE
|
|
u'\xbb': u'"', # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
|
|
u'\xdf': u'ss', # LATIN SMALL LETTER SHARP S
|
|
}
|
|
|
|
def unormalize(ustring, ignorenonascii=None, substitute=None):
|
|
"""replace diacritical characters with their corresponding ascii characters
|
|
|
|
Convert the unicode string to its long normalized form (unicode character
|
|
will be transform into several characters) and keep the first one only.
|
|
The normal form KD (NFKD) will apply the compatibility decomposition, i.e.
|
|
replace all compatibility characters with their equivalents.
|
|
|
|
:type substitute: str
|
|
:param substitute: replacement character to use if decomposition fails
|
|
|
|
:see: Another project about ASCII transliterations of Unicode text
|
|
http://pypi.python.org/pypi/Unidecode
|
|
"""
|
|
# backward compatibility, ignorenonascii was a boolean
|
|
if ignorenonascii is not None:
|
|
warn("ignorenonascii is deprecated, use substitute named parameter instead",
|
|
DeprecationWarning, stacklevel=2)
|
|
if ignorenonascii:
|
|
substitute = ''
|
|
res = []
|
|
for letter in ustring[:]:
|
|
try:
|
|
replacement = MANUAL_UNICODE_MAP[letter]
|
|
except KeyError:
|
|
replacement = _uninormalize('NFKD', letter)[0]
|
|
if ord(replacement) >= 2 ** 7:
|
|
if substitute is None:
|
|
raise ValueError("can't deal with non-ascii based characters")
|
|
replacement = substitute
|
|
res.append(replacement)
|
|
return u''.join(res)
|
|
|
|
def unquote(string):
|
|
"""remove optional quotes (simple or double) from the string
|
|
|
|
:type string: str or unicode
|
|
:param string: an optionally quoted string
|
|
|
|
:rtype: str or unicode
|
|
:return: the unquoted string (or the input string if it wasn't quoted)
|
|
"""
|
|
if not string:
|
|
return string
|
|
if string[0] in '"\'':
|
|
string = string[1:]
|
|
if string[-1] in '"\'':
|
|
string = string[:-1]
|
|
return string
|
|
|
|
|
|
_BLANKLINES_RGX = re.compile('\r?\n\r?\n')
|
|
_NORM_SPACES_RGX = re.compile('\s+')
|
|
|
|
def normalize_text(text, line_len=80, indent='', rest=False):
|
|
"""normalize a text to display it with a maximum line size and
|
|
optionally arbitrary indentation. Line jumps are normalized but blank
|
|
lines are kept. The indentation string may be used to insert a
|
|
comment (#) or a quoting (>) mark for instance.
|
|
|
|
:type text: str or unicode
|
|
:param text: the input text to normalize
|
|
|
|
:type line_len: int
|
|
:param line_len: expected maximum line's length, default to 80
|
|
|
|
:type indent: str or unicode
|
|
:param indent: optional string to use as indentation
|
|
|
|
:rtype: str or unicode
|
|
:return:
|
|
the input text normalized to fit on lines with a maximized size
|
|
inferior to `line_len`, and optionally prefixed by an
|
|
indentation string
|
|
"""
|
|
if rest:
|
|
normp = normalize_rest_paragraph
|
|
else:
|
|
normp = normalize_paragraph
|
|
result = []
|
|
for text in _BLANKLINES_RGX.split(text):
|
|
result.append(normp(text, line_len, indent))
|
|
return ('%s%s%s' % (linesep, indent, linesep)).join(result)
|
|
|
|
|
|
def normalize_paragraph(text, line_len=80, indent=''):
|
|
"""normalize a text to display it with a maximum line size and
|
|
optionally arbitrary indentation. Line jumps are normalized. The
|
|
indentation string may be used top insert a comment mark for
|
|
instance.
|
|
|
|
:type text: str or unicode
|
|
:param text: the input text to normalize
|
|
|
|
:type line_len: int
|
|
:param line_len: expected maximum line's length, default to 80
|
|
|
|
:type indent: str or unicode
|
|
:param indent: optional string to use as indentation
|
|
|
|
:rtype: str or unicode
|
|
:return:
|
|
the input text normalized to fit on lines with a maximized size
|
|
inferior to `line_len`, and optionally prefixed by an
|
|
indentation string
|
|
"""
|
|
text = _NORM_SPACES_RGX.sub(' ', text)
|
|
line_len = line_len - len(indent)
|
|
lines = []
|
|
while text:
|
|
aline, text = splittext(text.strip(), line_len)
|
|
lines.append(indent + aline)
|
|
return linesep.join(lines)
|
|
|
|
def normalize_rest_paragraph(text, line_len=80, indent=''):
|
|
"""normalize a ReST text to display it with a maximum line size and
|
|
optionally arbitrary indentation. Line jumps are normalized. The
|
|
indentation string may be used top insert a comment mark for
|
|
instance.
|
|
|
|
:type text: str or unicode
|
|
:param text: the input text to normalize
|
|
|
|
:type line_len: int
|
|
:param line_len: expected maximum line's length, default to 80
|
|
|
|
:type indent: str or unicode
|
|
:param indent: optional string to use as indentation
|
|
|
|
:rtype: str or unicode
|
|
:return:
|
|
the input text normalized to fit on lines with a maximized size
|
|
inferior to `line_len`, and optionally prefixed by an
|
|
indentation string
|
|
"""
|
|
toreport = ''
|
|
lines = []
|
|
line_len = line_len - len(indent)
|
|
for line in text.splitlines():
|
|
line = toreport + _NORM_SPACES_RGX.sub(' ', line.strip())
|
|
toreport = ''
|
|
while len(line) > line_len:
|
|
# too long line, need split
|
|
line, toreport = splittext(line, line_len)
|
|
lines.append(indent + line)
|
|
if toreport:
|
|
line = toreport + ' '
|
|
toreport = ''
|
|
else:
|
|
line = ''
|
|
if line:
|
|
lines.append(indent + line.strip())
|
|
return linesep.join(lines)
|
|
|
|
|
|
def splittext(text, line_len):
|
|
"""split the given text on space according to the given max line size
|
|
|
|
return a 2-uple:
|
|
* a line <= line_len if possible
|
|
* the rest of the text which has to be reported on another line
|
|
"""
|
|
if len(text) <= line_len:
|
|
return text, ''
|
|
pos = min(len(text)-1, line_len)
|
|
while pos > 0 and text[pos] != ' ':
|
|
pos -= 1
|
|
if pos == 0:
|
|
pos = min(len(text), line_len)
|
|
while len(text) > pos and text[pos] != ' ':
|
|
pos += 1
|
|
return text[:pos], text[pos+1:].strip()
|
|
|
|
|
|
def splitstrip(string, sep=','):
|
|
"""return a list of stripped string by splitting the string given as
|
|
argument on `sep` (',' by default). Empty string are discarded.
|
|
|
|
>>> splitstrip('a, b, c , 4,,')
|
|
['a', 'b', 'c', '4']
|
|
>>> splitstrip('a')
|
|
['a']
|
|
>>>
|
|
|
|
:type string: str or unicode
|
|
:param string: a csv line
|
|
|
|
:type sep: str or unicode
|
|
:param sep: field separator, default to the comma (',')
|
|
|
|
:rtype: str or unicode
|
|
:return: the unquoted string (or the input string if it wasn't quoted)
|
|
"""
|
|
return [word.strip() for word in string.split(sep) if word.strip()]
|
|
|
|
get_csv = deprecated('get_csv is deprecated, use splitstrip')(splitstrip)
|
|
|
|
|
|
def split_url_or_path(url_or_path):
|
|
"""return the latest component of a string containing either an url of the
|
|
form <scheme>://<path> or a local file system path
|
|
"""
|
|
if '://' in url_or_path:
|
|
return url_or_path.rstrip('/').rsplit('/', 1)
|
|
return osp.split(url_or_path.rstrip(osp.sep))
|
|
|
|
|
|
def text_to_dict(text):
|
|
"""parse multilines text containing simple 'key=value' lines and return a
|
|
dict of {'key': 'value'}. When the same key is encountered multiple time,
|
|
value is turned into a list containing all values.
|
|
|
|
>>> d = text_to_dict('''multiple=1
|
|
... multiple= 2
|
|
... single =3
|
|
... ''')
|
|
>>> d['single']
|
|
'3'
|
|
>>> d['multiple']
|
|
['1', '2']
|
|
|
|
"""
|
|
res = {}
|
|
if not text:
|
|
return res
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith('#'):
|
|
key, value = [w.strip() for w in line.split('=', 1)]
|
|
if key in res:
|
|
try:
|
|
res[key].append(value)
|
|
except AttributeError:
|
|
res[key] = [res[key], value]
|
|
else:
|
|
res[key] = value
|
|
return res
|
|
|
|
|
|
_BLANK_URE = r'(\s|,)+'
|
|
_BLANK_RE = re.compile(_BLANK_URE)
|
|
__VALUE_URE = r'-?(([0-9]+\.[0-9]*)|((0x?)?[0-9]+))'
|
|
__UNITS_URE = r'[a-zA-Z]+'
|
|
_VALUE_RE = re.compile(r'(?P<value>%s)(?P<unit>%s)?'%(__VALUE_URE, __UNITS_URE))
|
|
_VALIDATION_RE = re.compile(r'^((%s)(%s))*(%s)?$' % (__VALUE_URE, __UNITS_URE,
|
|
__VALUE_URE))
|
|
|
|
BYTE_UNITS = {
|
|
"b": 1,
|
|
"kb": 1024,
|
|
"mb": 1024 ** 2,
|
|
"gb": 1024 ** 3,
|
|
"tb": 1024 ** 4,
|
|
}
|
|
|
|
TIME_UNITS = {
|
|
"ms": 0.0001,
|
|
"s": 1,
|
|
"min": 60,
|
|
"h": 60 * 60,
|
|
"d": 60 * 60 *24,
|
|
}
|
|
|
|
def apply_units(string, units, inter=None, final=float, blank_reg=_BLANK_RE,
|
|
value_reg=_VALUE_RE):
|
|
"""Parse the string applying the units defined in units
|
|
(e.g.: "1.5m",{'m',60} -> 80).
|
|
|
|
:type string: str or unicode
|
|
:param string: the string to parse
|
|
|
|
:type units: dict (or any object with __getitem__ using basestring key)
|
|
:param units: a dict mapping a unit string repr to its value
|
|
|
|
:type inter: type
|
|
:param inter: used to parse every intermediate value (need __sum__)
|
|
|
|
:type blank_reg: regexp
|
|
:param blank_reg: should match every blank char to ignore.
|
|
|
|
:type value_reg: regexp with "value" and optional "unit" group
|
|
:param value_reg: match a value and it's unit into the
|
|
"""
|
|
if inter is None:
|
|
inter = final
|
|
fstring = _BLANK_RE.sub('', string)
|
|
if not (fstring and _VALIDATION_RE.match(fstring)):
|
|
raise ValueError("Invalid unit string: %r." % string)
|
|
values = []
|
|
for match in value_reg.finditer(fstring):
|
|
dic = match.groupdict()
|
|
lit, unit = dic["value"], dic.get("unit")
|
|
value = inter(lit)
|
|
if unit is not None:
|
|
try:
|
|
value *= units[unit.lower()]
|
|
except KeyError:
|
|
raise KeyError('invalid unit %s. valid units are %s' %
|
|
(unit, units.keys()))
|
|
values.append(value)
|
|
return final(sum(values))
|
|
|
|
|
|
_LINE_RGX = re.compile('\r\n|\r+|\n')
|
|
|
|
def pretty_match(match, string, underline_char='^'):
|
|
"""return a string with the match location underlined:
|
|
|
|
>>> import re
|
|
>>> print(pretty_match(re.search('mange', 'il mange du bacon'), 'il mange du bacon'))
|
|
il mange du bacon
|
|
^^^^^
|
|
>>>
|
|
|
|
:type match: _sre.SRE_match
|
|
:param match: object returned by re.match, re.search or re.finditer
|
|
|
|
:type string: str or unicode
|
|
:param string:
|
|
the string on which the regular expression has been applied to
|
|
obtain the `match` object
|
|
|
|
:type underline_char: str or unicode
|
|
:param underline_char:
|
|
character to use to underline the matched section, default to the
|
|
carret '^'
|
|
|
|
:rtype: str or unicode
|
|
:return:
|
|
the original string with an inserted line to underline the match
|
|
location
|
|
"""
|
|
start = match.start()
|
|
end = match.end()
|
|
string = _LINE_RGX.sub(linesep, string)
|
|
start_line_pos = string.rfind(linesep, 0, start)
|
|
if start_line_pos == -1:
|
|
start_line_pos = 0
|
|
result = []
|
|
else:
|
|
result = [string[:start_line_pos]]
|
|
start_line_pos += len(linesep)
|
|
offset = start - start_line_pos
|
|
underline = ' ' * offset + underline_char * (end - start)
|
|
end_line_pos = string.find(linesep, end)
|
|
if end_line_pos == -1:
|
|
string = string[start_line_pos:]
|
|
result.append(string)
|
|
result.append(underline)
|
|
else:
|
|
end = string[end_line_pos + len(linesep):]
|
|
string = string[start_line_pos:end_line_pos]
|
|
result.append(string)
|
|
result.append(underline)
|
|
result.append(end)
|
|
return linesep.join(result).rstrip()
|
|
|
|
|
|
# Ansi colorization ###########################################################
|
|
|
|
ANSI_PREFIX = '\033['
|
|
ANSI_END = 'm'
|
|
ANSI_RESET = '\033[0m'
|
|
ANSI_STYLES = {
|
|
'reset': "0",
|
|
'bold': "1",
|
|
'italic': "3",
|
|
'underline': "4",
|
|
'blink': "5",
|
|
'inverse': "7",
|
|
'strike': "9",
|
|
}
|
|
ANSI_COLORS = {
|
|
'reset': "0",
|
|
'black': "30",
|
|
'red': "31",
|
|
'green': "32",
|
|
'yellow': "33",
|
|
'blue': "34",
|
|
'magenta': "35",
|
|
'cyan': "36",
|
|
'white': "37",
|
|
}
|
|
|
|
def _get_ansi_code(color=None, style=None):
|
|
"""return ansi escape code corresponding to color and style
|
|
|
|
:type color: str or None
|
|
:param color:
|
|
the color name (see `ANSI_COLORS` for available values)
|
|
or the color number when 256 colors are available
|
|
|
|
:type style: str or None
|
|
:param style:
|
|
style string (see `ANSI_COLORS` for available values). To get
|
|
several style effects at the same time, use a coma as separator.
|
|
|
|
:raise KeyError: if an unexistent color or style identifier is given
|
|
|
|
:rtype: str
|
|
:return: the built escape code
|
|
"""
|
|
ansi_code = []
|
|
if style:
|
|
style_attrs = splitstrip(style)
|
|
for effect in style_attrs:
|
|
ansi_code.append(ANSI_STYLES[effect])
|
|
if color:
|
|
if color.isdigit():
|
|
ansi_code.extend(['38', '5'])
|
|
ansi_code.append(color)
|
|
else:
|
|
ansi_code.append(ANSI_COLORS[color])
|
|
if ansi_code:
|
|
return ANSI_PREFIX + ';'.join(ansi_code) + ANSI_END
|
|
return ''
|
|
|
|
def colorize_ansi(msg, color=None, style=None):
|
|
"""colorize message by wrapping it with ansi escape codes
|
|
|
|
:type msg: str or unicode
|
|
:param msg: the message string to colorize
|
|
|
|
:type color: str or None
|
|
:param color:
|
|
the color identifier (see `ANSI_COLORS` for available values)
|
|
|
|
:type style: str or None
|
|
:param style:
|
|
style string (see `ANSI_COLORS` for available values). To get
|
|
several style effects at the same time, use a coma as separator.
|
|
|
|
:raise KeyError: if an unexistent color or style identifier is given
|
|
|
|
:rtype: str or unicode
|
|
:return: the ansi escaped string
|
|
"""
|
|
# If both color and style are not defined, then leave the text as is
|
|
if color is None and style is None:
|
|
return msg
|
|
escape_code = _get_ansi_code(color, style)
|
|
# If invalid (or unknown) color, don't wrap msg with ansi codes
|
|
if escape_code:
|
|
return '%s%s%s' % (escape_code, msg, ANSI_RESET)
|
|
return msg
|
|
|
|
DIFF_STYLE = {'separator': 'cyan', 'remove': 'red', 'add': 'green'}
|
|
|
|
def diff_colorize_ansi(lines, out=sys.stdout, style=DIFF_STYLE):
|
|
for line in lines:
|
|
if line[:4] in ('--- ', '+++ '):
|
|
out.write(colorize_ansi(line, style['separator']))
|
|
elif line[0] == '-':
|
|
out.write(colorize_ansi(line, style['remove']))
|
|
elif line[0] == '+':
|
|
out.write(colorize_ansi(line, style['add']))
|
|
elif line[:4] == '--- ':
|
|
out.write(colorize_ansi(line, style['separator']))
|
|
elif line[:4] == '+++ ':
|
|
out.write(colorize_ansi(line, style['separator']))
|
|
else:
|
|
out.write(line)
|
|
|