Imported from SVN by Bitbucket

This commit is contained in:
2015-03-31 20:26:20 +00:00
committed by bitbucket
commit ceb7984dec
212 changed files with 49537 additions and 0 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,167 @@
"""A more or less complete user-defined wrapper around dictionary objects."""
class UserDict:
def __init__(self, dict=None, **kwargs):
self.data = {}
if dict is not None:
if not hasattr(dict,'keys'):
dict = type({})(dict) # make mapping from a sequence
self.update(dict)
if len(kwargs):
self.update(kwargs)
def __repr__(self): return repr(self.data)
def __cmp__(self, dict):
if isinstance(dict, UserDict):
return cmp(self.data, dict.data)
else:
return cmp(self.data, dict)
def __len__(self): return len(self.data)
def __getitem__(self, key): return self.data[key]
def __setitem__(self, key, item): self.data[key] = item
def __delitem__(self, key): del self.data[key]
def clear(self): self.data.clear()
def copy(self):
if self.__class__ is UserDict:
return UserDict(self.data)
import copy
data = self.data
try:
self.data = {}
c = copy.copy(self)
finally:
self.data = data
c.update(self)
return c
def keys(self): return self.data.keys()
def items(self): return self.data.items()
def iteritems(self): return self.data.iteritems()
def iterkeys(self): return self.data.iterkeys()
def itervalues(self): return self.data.itervalues()
def values(self): return self.data.values()
def has_key(self, key): return self.data.has_key(key)
def update(self, dict):
if isinstance(dict, UserDict):
self.data.update(dict.data)
elif isinstance(dict, type(self.data)):
self.data.update(dict)
else:
for k, v in dict.items():
self[k] = v
def get(self, key, failobj=None):
if not self.has_key(key):
return failobj
return self[key]
def setdefault(self, key, failobj=None):
if not self.has_key(key):
self[key] = failobj
return self[key]
def pop(self, key, *args):
return self.data.pop(key, *args)
def popitem(self):
return self.data.popitem()
def __contains__(self, key):
return key in self.data
def fromkeys(cls, iterable, value=None):
d = cls()
for key in iterable:
d[key] = value
return d
fromkeys = classmethod(fromkeys)
class IterableUserDict(UserDict):
def __iter__(self):
return iter(self.data)
class DictMixin:
# Mixin defining all dictionary methods for classes that already have
# a minimum dictionary interface including getitem, setitem, delitem,
# and keys. Without knowledge of the subclass constructor, the mixin
# does not define __init__() or copy(). In addition to the four base
# methods, progressively more efficiency comes with defining
# __contains__(), __iter__(), and iteritems().
# second level definitions support higher levels
def __iter__(self):
for k in self.keys():
yield k
def has_key(self, key):
try:
value = self[key]
except KeyError:
return False
return True
def __contains__(self, key):
return self.has_key(key)
# third level takes advantage of second level definitions
def iteritems(self):
for k in self:
yield (k, self[k])
def iterkeys(self):
return self.__iter__()
# fourth level uses definitions from lower levels
def itervalues(self):
for _, v in self.iteritems():
yield v
def values(self):
return [v for _, v in self.iteritems()]
def items(self):
return list(self.iteritems())
def clear(self):
for key in self.keys():
del self[key]
def setdefault(self, key, default):
try:
return self[key]
except KeyError:
self[key] = default
return default
def pop(self, key, *args):
if len(args) > 1:
raise TypeError, "pop expected at most 2 arguments, got "\
+ repr(1 + len(args))
try:
value = self[key]
except KeyError:
if args:
return args[0]
raise
del self[key]
return value
def popitem(self):
try:
k, v = self.iteritems().next()
except StopIteration:
raise KeyError, 'container is empty'
del self[k]
return (k, v)
def update(self, other):
# Make progressively weaker assumptions about "other"
if hasattr(other, 'iteritems'): # iteritems saves memory and lookups
for k, v in other.iteritems():
self[k] = v
elif hasattr(other, '__iter__'): # iter saves memory
for k in other:
self[k] = other[k]
else:
for k in other.keys():
self[k] = other[k]
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
def __repr__(self):
return repr(dict(self.iteritems()))
def __cmp__(self, other):
if other is None:
return 1
if isinstance(other, DictMixin):
other = dict(other.iteritems())
return cmp(dict(self.iteritems()), other)
def __len__(self):
return len(self.keys())
def __nonzero__(self):
return bool(self.iteritems())

View File

@@ -0,0 +1,4 @@
"""
Package for miscellaneous routines that do not depend on other parts
of Paste
"""

View File

@@ -0,0 +1,42 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
class ClassInitMeta(type):
def __new__(meta, class_name, bases, new_attrs):
cls = type.__new__(meta, class_name, bases, new_attrs)
if (new_attrs.has_key('__classinit__')
and not isinstance(cls.__classinit__, staticmethod)):
setattr(cls, '__classinit__',
staticmethod(cls.__classinit__.im_func))
if hasattr(cls, '__classinit__'):
cls.__classinit__(cls, new_attrs)
return cls
def build_properties(cls, new_attrs):
"""
Given a class and a new set of attributes (as passed in by
__classinit__), create or modify properties based on functions
with special names ending in __get, __set, and __del.
"""
for name, value in new_attrs.items():
if (name.endswith('__get') or name.endswith('__set')
or name.endswith('__del')):
base = name[:-5]
if hasattr(cls, base):
old_prop = getattr(cls, base)
if not isinstance(old_prop, property):
raise ValueError(
"Attribute %s is a %s, not a property; function %s is named like a property"
% (base, type(old_prop), name))
attrs = {'fget': old_prop.fget,
'fset': old_prop.fset,
'fdel': old_prop.fdel,
'doc': old_prop.__doc__}
else:
attrs = {}
attrs['f' + name[-3:]] = value
if name.endswith('__get') and value.__doc__:
attrs['doc'] = value.__doc__
new_prop = property(**attrs)
setattr(cls, base, new_prop)

View File

@@ -0,0 +1,38 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
class classinstancemethod(object):
"""
Acts like a class method when called from a class, like an
instance method when called by an instance. The method should
take two arguments, 'self' and 'cls'; one of these will be None
depending on how the method was called.
"""
def __init__(self, func):
self.func = func
self.__doc__ = func.__doc__
def __get__(self, obj, type=None):
return _methodwrapper(self.func, obj=obj, type=type)
class _methodwrapper(object):
def __init__(self, func, obj, type):
self.func = func
self.obj = obj
self.type = type
def __call__(self, *args, **kw):
assert not kw.has_key('self') and not kw.has_key('cls'), (
"You cannot use 'self' or 'cls' arguments to a "
"classinstancemethod")
return self.func(*((self.obj, self.type) + args), **kw)
def __repr__(self):
if self.obj is None:
return ('<bound class method %s.%s>'
% (self.type.__name__, self.func.func_name))
else:
return ('<bound method %s.%s of %r>'
% (self.type.__name__, self.func.func_name, self.obj))

View File

@@ -0,0 +1,26 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
def asbool(obj):
if isinstance(obj, (str, unicode)):
obj = obj.strip().lower()
if obj in ['true', 'yes', 'on', 'y', 't', '1']:
return True
elif obj in ['false', 'no', 'off', 'n', 'f', '0']:
return False
else:
raise ValueError(
"String is not true/false: %r" % obj)
return bool(obj)
def aslist(obj, sep=None, strip=True):
if isinstance(obj, (str, unicode)):
lst = obj.split(sep)
if strip:
lst = [v.strip() for v in lst]
return lst
elif isinstance(obj, (list, tuple)):
return obj
elif obj is None:
return []
else:
return [obj]

View File

@@ -0,0 +1,103 @@
"""
DateInterval.py
Convert interval strings (in the form of 1w2d, etc) to
seconds, and back again. Is not exactly about months or
years (leap years in particular).
Accepts (y)ear, (b)month, (w)eek, (d)ay, (h)our, (m)inute, (s)econd.
Exports only timeEncode and timeDecode functions.
"""
import re
__all__ = ['interval_decode', 'interval_encode']
second = 1
minute = second*60
hour = minute*60
day = hour*24
week = day*7
month = day*30
year = day*365
timeValues = {
'y': year,
'b': month,
'w': week,
'd': day,
'h': hour,
'm': minute,
's': second,
}
timeOrdered = timeValues.items()
timeOrdered.sort(lambda a, b: -cmp(a[1], b[1]))
def interval_encode(seconds, include_sign=False):
"""Encodes a number of seconds (representing a time interval)
into a form like 1h2d3s.
>>> interval_encode(10)
'10s'
>>> interval_encode(493939)
'5d17h12m19s'
"""
s = ''
orig = seconds
seconds = abs(seconds)
for char, amount in timeOrdered:
if seconds >= amount:
i, seconds = divmod(seconds, amount)
s += '%i%s' % (i, char)
if orig < 0:
s = '-' + s
elif not orig:
return '0'
elif include_sign:
s = '+' + s
return s
_timeRE = re.compile(r'[0-9]+[a-zA-Z]')
def interval_decode(s):
"""Decodes a number in the format 1h4d3m (1 hour, 3 days, 3 minutes)
into a number of seconds
>>> interval_decode('40s')
40
>>> interval_decode('10000s')
10000
>>> interval_decode('3d1w45s')
864045
"""
time = 0
sign = 1
s = s.strip()
if s.startswith('-'):
s = s[1:]
sign = -1
elif s.startswith('+'):
s = s[1:]
for match in allMatches(s, _timeRE):
char = match.group(0)[-1].lower()
if not timeValues.has_key(char):
# @@: should signal error
continue
time += int(match.group(0)[:-1]) * timeValues[char]
return time
# @@-sgd 2002-12-23 - this function does not belong in this module, find a better place.
def allMatches(source, regex):
"""Return a list of matches for regex in source
"""
pos = 0
end = len(source)
rv = []
match = regex.search(source, pos)
while match:
rv.append(match)
match = regex.search(source, match.end() )
return rv
if __name__ == '__main__':
import doctest
doctest.testmod()

View File

@@ -0,0 +1,361 @@
# (c) 2005 Clark C. Evans and contributors
# This module is part of the Python Paste Project and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
# Some of this code was funded by: http://prometheusresearch.com
"""
Date, Time, and Timespan Parsing Utilities
This module contains parsing support to create "human friendly"
``datetime`` object parsing. The explicit goal of these routines is
to provide a multi-format date/time support not unlike that found in
Microsoft Excel. In most approaches, the input is very "strict" to
prevent errors -- however, this approach is much more liberal since we
are assuming the user-interface is parroting back the normalized value
and thus the user has immediate feedback if the data is not typed in
correctly.
``parse_date`` and ``normalize_date``
These functions take a value like '9 jan 2007' and returns either an
``date`` object, or an ISO 8601 formatted date value such
as '2007-01-09'. There is an option to provide an Oracle database
style output as well, ``09 JAN 2007``, but this is not the default.
This module always treats '/' delimiters as using US date order
(since the author's clients are US based), hence '1/9/2007' is
January 9th. Since this module treats the '-' as following
European order this supports both modes of data-entry; together
with immediate parroting back the result to the screen, the author
has found this approach to work well in pratice.
``parse_time`` and ``normalize_time``
These functions take a value like '1 pm' and returns either an
``time`` object, or an ISO 8601 formatted 24h clock time
such as '13:00'. There is an option to provide for US style time
values, '1:00 PM', however this is not the default.
``parse_datetime`` and ``normalize_datetime``
These functions take a value like '9 jan 2007 at 1 pm' and returns
either an ``datetime`` object, or an ISO 8601 formatted
return (without the T) such as '2007-01-09 13:00'. There is an
option to provide for Oracle / US style, '09 JAN 2007 @ 1:00 PM',
however this is not the default.
``parse_delta`` and ``normalize_delta``
These functions take a value like '1h 15m' and returns either an
``timedelta`` object, or an 2-decimal fixed-point
numerical value in hours, such as '1.25'. The rationale is to
support meeting or time-billing lengths, not to be an accurate
representation in mili-seconds. As such not all valid
``timedelta`` values will have a normalized representation.
"""
from datetime import timedelta, time, date
from time import localtime
import string
__all__ = ['parse_timedelta', 'normalize_timedelta',
'parse_time', 'normalize_time',
'parse_date', 'normalize_date']
def _number(val):
try:
return string.atoi(val)
except:
return None
#
# timedelta
#
def parse_timedelta(val):
"""
returns a ``timedelta`` object, or None
"""
if not val:
return None
val = string.lower(val)
if "." in val:
val = float(val)
return timedelta(hours=int(val), minutes=60*(val % 1.0))
fHour = ("h" in val or ":" in val)
fMin = ("m" in val or ":" in val)
fFraction = "." in val
for noise in "minu:teshour()":
val = string.replace(val, noise, ' ')
val = string.strip(val)
val = string.split(val)
hr = 0.0
mi = 0
val.reverse()
if fHour:
hr = int(val.pop())
if fMin:
mi = int(val.pop())
if len(val) > 0 and not hr:
hr = int(val.pop())
return timedelta(hours=hr, minutes=mi)
def normalize_timedelta(val):
"""
produces a normalized string value of the timedelta
This module returns a normalized time span value consisting of the
number of hours in fractional form. For example '1h 15min' is
formatted as 01.25.
"""
if type(val) == str:
val = parse_timedelta(val)
if not val:
return ''
hr = val.seconds/3600
mn = (val.seconds % 3600)/60
return "%d.%02d" % (hr, mn * 100/60)
#
# time
#
def parse_time(val):
if not val:
return None
hr = mi = 0
val = string.lower(val)
amflag = (-1 != string.find(val, 'a')) # set if AM is found
pmflag = (-1 != string.find(val, 'p')) # set if PM is found
for noise in ":amp.":
val = string.replace(val, noise, ' ')
val = string.split(val)
if len(val) > 1:
hr = int(val[0])
mi = int(val[1])
else:
val = val[0]
if len(val) < 1:
pass
elif 'now' == val:
tm = localtime()
hr = tm[3]
mi = tm[4]
elif 'noon' == val:
hr = 12
elif len(val) < 3:
hr = int(val)
if not amflag and not pmflag and hr < 7:
hr += 12
elif len(val) < 5:
hr = int(val[:-2])
mi = int(val[-2:])
else:
hr = int(val[:1])
if amflag and hr >= 12:
hr = hr - 12
if pmflag and hr < 12:
hr = hr + 12
return time(hr, mi)
def normalize_time(value, ampm):
if not value:
return ''
if type(value) == str:
value = parse_time(value)
if not ampm:
return "%02d:%02d" % (value.hour, value.minute)
hr = value.hour
am = "AM"
if hr < 1 or hr > 23:
hr = 12
elif hr >= 12:
am = "PM"
if hr > 12:
hr = hr - 12
return "%02d:%02d %s" % (hr, value.minute, am)
#
# Date Processing
#
_one_day = timedelta(days=1)
_str2num = {'jan':1, 'feb':2, 'mar':3, 'apr':4, 'may':5, 'jun':6,
'jul':7, 'aug':8, 'sep':9, 'oct':10, 'nov':11, 'dec':12 }
def _month(val):
for (key, mon) in _str2num.items():
if key in val:
return mon
raise TypeError("unknown month '%s'" % val)
_days_in_month = {1: 31, 2: 28, 3: 31, 4: 30, 5: 31, 6: 30,
7: 31, 8: 31, 9: 30, 10: 31, 11: 30, 12: 31,
}
_num2str = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun',
7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec',
}
_wkdy = ("mon", "tue", "wed", "thu", "fri", "sat", "sun")
def parse_date(val):
if not(val):
return None
val = string.lower(val)
now = None
# optimized check for YYYY-MM-DD
strict = val.split("-")
if len(strict) == 3:
(y, m, d) = strict
if "+" in d:
d = d.split("+")[0]
if " " in d:
d = d.split(" ")[0]
try:
now = date(int(y), int(m), int(d))
val = "xxx" + val[10:]
except ValueError:
pass
# allow for 'now', 'mon', 'tue', etc.
if not now:
chk = val[:3]
if chk in ('now','tod'):
now = date.today()
elif chk in _wkdy:
now = date.today()
idx = list(_wkdy).index(chk) + 1
while now.isoweekday() != idx:
now += _one_day
# allow dates to be modified via + or - /w number of days, so
# that now+3 is three days from now
if now:
tail = val[3:].strip()
tail = tail.replace("+"," +").replace("-"," -")
for item in tail.split():
try:
days = int(item)
except ValueError:
pass
else:
now += timedelta(days=days)
return now
# ok, standard parsing
yr = mo = dy = None
for noise in ('/', '-', ',', '*'):
val = string.replace(val, noise, ' ')
for noise in _wkdy:
val = string.replace(val, noise, ' ')
out = []
last = False
ldig = False
for ch in val:
if ch.isdigit():
if last and not ldig:
out.append(' ')
last = ldig = True
else:
if ldig:
out.append(' ')
ldig = False
last = True
out.append(ch)
val = string.split("".join(out))
if 3 == len(val):
a = _number(val[0])
b = _number(val[1])
c = _number(val[2])
if len(val[0]) == 4:
yr = a
if b: # 1999 6 23
mo = b
dy = c
else: # 1999 Jun 23
mo = _month(val[1])
dy = c
elif a > 0:
yr = c
if len(val[2]) < 4:
raise TypeError("four digit year required")
if b: # 6 23 1999
dy = b
mo = a
else: # 23 Jun 1999
dy = a
mo = _month(val[1])
else: # Jun 23, 2000
dy = b
yr = c
if len(val[2]) < 4:
raise TypeError("four digit year required")
mo = _month(val[0])
elif 2 == len(val):
a = _number(val[0])
b = _number(val[1])
if a > 999:
yr = a
dy = 1
if b > 0: # 1999 6
mo = b
else: # 1999 Jun
mo = _month(val[1])
elif a > 0:
if b > 999: # 6 1999
mo = a
yr = b
dy = 1
elif b > 0: # 6 23
mo = a
dy = b
else: # 23 Jun
dy = a
mo = _month(val[1])
else:
if b > 999: # Jun 2001
yr = b
dy = 1
else: # Jun 23
dy = b
mo = _month(val[0])
elif 1 == len(val):
val = val[0]
if not val.isdigit():
mo = _month(val)
if mo is not None:
dy = 1
else:
v = _number(val)
val = str(v)
if 8 == len(val): # 20010623
yr = _number(val[:4])
mo = _number(val[4:6])
dy = _number(val[6:])
elif len(val) in (3,4):
if v > 1300: # 2004
yr = v
mo = 1
dy = 1
else: # 1202
mo = _number(val[:-2])
dy = _number(val[-2:])
elif v < 32:
dy = v
else:
raise TypeError("four digit year required")
tm = localtime()
if mo is None:
mo = tm[1]
if dy is None:
dy = tm[2]
if yr is None:
yr = tm[0]
return date(yr, mo, dy)
def normalize_date(val, iso8601=True):
if not val:
return ''
if type(val) == str:
val = parse_date(val)
if iso8601:
return "%4d-%02d-%02d" % (val.year, val.month, val.day)
return "%02d %s %4d" % (val.day, _num2str[val.month], val.year)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,53 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
class FileMixin(object):
"""
Used to provide auxiliary methods to objects simulating files.
Objects must implement write, and read if they are input files.
Also they should implement close.
Other methods you may wish to override:
* flush()
* seek(offset[, whence])
* tell()
* truncate([size])
Attributes you may wish to provide:
* closed
* encoding (you should also respect that in write())
* mode
* newlines (hard to support)
* softspace
"""
def flush(self):
pass
def next(self):
return self.readline()
def readline(self, size=None):
# @@: This is a lame implementation; but a buffer would probably
# be necessary for a better implementation
output = []
while 1:
next = self.read(1)
if not next:
return ''.join(output)
output.append(next)
if size and size > 0 and len(output) >= size:
return ''.join(output)
if next == '\n':
# @@: also \r?
return ''.join(output)
def xreadlines(self):
return self
def writelines(self, lines):
for line in lines:
self.write(line)

View File

@@ -0,0 +1,99 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
# Note: you may want to copy this into your setup.py file verbatim, as
# you can't import this from another package, when you don't know if
# that package is installed yet.
import os
import sys
from fnmatch import fnmatchcase
from distutils.util import convert_path
# Provided as an attribute, so you can append to these instead
# of replicating them:
standard_exclude = ('*.py', '*.pyc', '*$py.class', '*~', '.*', '*.bak')
standard_exclude_directories = ('.*', 'CVS', '_darcs', './build',
'./dist', 'EGG-INFO', '*.egg-info')
def find_package_data(
where='.', package='',
exclude=standard_exclude,
exclude_directories=standard_exclude_directories,
only_in_packages=True,
show_ignored=False):
"""
Return a dictionary suitable for use in ``package_data``
in a distutils ``setup.py`` file.
The dictionary looks like::
{'package': [files]}
Where ``files`` is a list of all the files in that package that
don't match anything in ``exclude``.
If ``only_in_packages`` is true, then top-level directories that
are not packages won't be included (but directories under packages
will).
Directories matching any pattern in ``exclude_directories`` will
be ignored; by default directories with leading ``.``, ``CVS``,
and ``_darcs`` will be ignored.
If ``show_ignored`` is true, then all the files that aren't
included in package data are shown on stderr (for debugging
purposes).
Note patterns use wildcards, or can be exact paths (including
leading ``./``), and all searching is case-insensitive.
"""
out = {}
stack = [(convert_path(where), '', package, only_in_packages)]
while stack:
where, prefix, package, only_in_packages = stack.pop(0)
for name in os.listdir(where):
fn = os.path.join(where, name)
if os.path.isdir(fn):
bad_name = False
for pattern in exclude_directories:
if (fnmatchcase(name, pattern)
or fn.lower() == pattern.lower()):
bad_name = True
if show_ignored:
print >> sys.stderr, (
"Directory %s ignored by pattern %s"
% (fn, pattern))
break
if bad_name:
continue
if (os.path.isfile(os.path.join(fn, '__init__.py'))
and not prefix):
if not package:
new_package = name
else:
new_package = package + '.' + name
stack.append((fn, '', new_package, False))
else:
stack.append((fn, prefix + name + '/', package, only_in_packages))
elif package or not only_in_packages:
# is a file
bad_name = False
for pattern in exclude:
if (fnmatchcase(name, pattern)
or fn.lower() == pattern.lower()):
bad_name = True
if show_ignored:
print >> sys.stderr, (
"File %s ignored by pattern %s"
% (fn, pattern))
break
if bad_name:
continue
out.setdefault(package, []).append(prefix+name)
return out
if __name__ == '__main__':
import pprint
pprint.pprint(
find_package_data(show_ignored=True))

View File

@@ -0,0 +1,26 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
import sys
import os
def find_package(dir):
"""
Given a directory, finds the equivalent package name. If it
is directly in sys.path, returns ''.
"""
dir = os.path.abspath(dir)
orig_dir = dir
path = map(os.path.abspath, sys.path)
packages = []
last_dir = None
while 1:
if dir in path:
return '.'.join(packages)
packages.insert(0, os.path.basename(dir))
dir = os.path.dirname(dir)
if last_dir == dir:
raise ValueError(
"%s is not under any path found in sys.path" % orig_dir)
last_dir = dir

View File

@@ -0,0 +1,95 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
"""
'imports' a string -- converts a string to a Python object, importing
any necessary modules and evaluating the expression. Everything
before the : in an import expression is the module path; everything
after is an expression to be evaluated in the namespace of that
module.
Alternately, if no : is present, then import the modules and get the
attributes as necessary. Arbitrary expressions are not allowed in
that case.
"""
def eval_import(s):
"""
Import a module, or import an object from a module.
A module name like ``foo.bar:baz()`` can be used, where
``foo.bar`` is the module, and ``baz()`` is an expression
evaluated in the context of that module. Note this is not safe on
arbitrary strings because of the eval.
"""
if ':' not in s:
return simple_import(s)
module_name, expr = s.split(':', 1)
module = import_module(module_name)
obj = eval(expr, module.__dict__)
return obj
def simple_import(s):
"""
Import a module, or import an object from a module.
A name like ``foo.bar.baz`` can be a module ``foo.bar.baz`` or a
module ``foo.bar`` with an object ``baz`` in it, or a module
``foo`` with an object ``bar`` with an attribute ``baz``.
"""
parts = s.split('.')
module = import_module(parts[0])
name = parts[0]
parts = parts[1:]
last_import_error = None
while parts:
name += '.' + parts[0]
try:
module = import_module(name)
parts = parts[1:]
except ImportError, e:
last_import_error = e
break
obj = module
while parts:
try:
obj = getattr(module, parts[0])
except AttributeError:
raise ImportError(
"Cannot find %s in module %r (stopped importing modules with error %s)" % (parts[0], module, last_import_error))
parts = parts[1:]
return obj
def import_module(s):
"""
Import a module.
"""
mod = __import__(s)
parts = s.split('.')
for part in parts[1:]:
mod = getattr(mod, part)
return mod
def try_import_module(module_name):
"""
Imports a module, but catches import errors. Only catches errors
when that module doesn't exist; if that module itself has an
import error it will still get raised. Returns None if the module
doesn't exist.
"""
try:
return import_module(module_name)
except ImportError, e:
if not getattr(e, 'args', None):
raise
desc = e.args[0]
if not desc.startswith('No module named '):
raise
desc = desc[len('No module named '):]
# If you import foo.bar.baz, the bad import could be any
# of foo.bar.baz, bar.baz, or baz; we'll test them all:
parts = module_name.split('.')
for i in range(len(parts)):
if desc == '.'.join(parts[i:]):
return None
raise

View File

@@ -0,0 +1,511 @@
# -*- coding: iso-8859-15 -*-
"""Immutable integer set type.
Integer set class.
Copyright (C) 2006, Heiko Wundram.
Released under the MIT license.
"""
# Version information
# -------------------
__author__ = "Heiko Wundram <me@modelnine.org>"
__version__ = "0.2"
__revision__ = "6"
__date__ = "2006-01-20"
# Utility classes
# ---------------
class _Infinity(object):
"""Internal type used to represent infinity values."""
__slots__ = ["_neg"]
def __init__(self,neg):
self._neg = neg
def __lt__(self,value):
if not isinstance(value,(int,long,_Infinity)):
return NotImplemented
return ( self._neg and
not ( isinstance(value,_Infinity) and value._neg ) )
def __le__(self,value):
if not isinstance(value,(int,long,_Infinity)):
return NotImplemented
return self._neg
def __gt__(self,value):
if not isinstance(value,(int,long,_Infinity)):
return NotImplemented
return not ( self._neg or
( isinstance(value,_Infinity) and not value._neg ) )
def __ge__(self,value):
if not isinstance(value,(int,long,_Infinity)):
return NotImplemented
return not self._neg
def __eq__(self,value):
if not isinstance(value,(int,long,_Infinity)):
return NotImplemented
return isinstance(value,_Infinity) and self._neg == value._neg
def __ne__(self,value):
if not isinstance(value,(int,long,_Infinity)):
return NotImplemented
return not isinstance(value,_Infinity) or self._neg <> value._neg
def __repr__(self):
return "None"
# Constants
# ---------
_MININF = _Infinity(True)
_MAXINF = _Infinity(False)
# Integer set class
# -----------------
class IntSet(object):
"""Integer set class with efficient storage in a RLE format of ranges.
Supports minus and plus infinity in the range."""
__slots__ = ["_ranges","_min","_max","_hash"]
def __init__(self,*args,**kwargs):
"""Initialize an integer set. The constructor accepts an unlimited
number of arguments that may either be tuples in the form of
(start,stop) where either start or stop may be a number or None to
represent maximum/minimum in that direction. The range specified by
(start,stop) is always inclusive (differing from the builtin range
operator).
Keyword arguments that can be passed to an integer set are min and
max, which specify the minimum and maximum number in the set,
respectively. You can also pass None here to represent minus or plus
infinity, which is also the default.
"""
# Special case copy constructor.
if len(args) == 1 and isinstance(args[0],IntSet):
if kwargs:
raise ValueError("No keyword arguments for copy constructor.")
self._min = args[0]._min
self._max = args[0]._max
self._ranges = args[0]._ranges
self._hash = args[0]._hash
return
# Initialize set.
self._ranges = []
# Process keyword arguments.
self._min = kwargs.pop("min",_MININF)
self._max = kwargs.pop("max",_MAXINF)
if self._min is None:
self._min = _MININF
if self._max is None:
self._max = _MAXINF
# Check keyword arguments.
if kwargs:
raise ValueError("Invalid keyword argument.")
if not ( isinstance(self._min,(int,long)) or self._min is _MININF ):
raise TypeError("Invalid type of min argument.")
if not ( isinstance(self._max,(int,long)) or self._max is _MAXINF ):
raise TypeError("Invalid type of max argument.")
if ( self._min is not _MININF and self._max is not _MAXINF and
self._min > self._max ):
raise ValueError("Minimum is not smaller than maximum.")
if isinstance(self._max,(int,long)):
self._max += 1
# Process arguments.
for arg in args:
if isinstance(arg,(int,long)):
start, stop = arg, arg+1
elif isinstance(arg,tuple):
if len(arg) <> 2:
raise ValueError("Invalid tuple, must be (start,stop).")
# Process argument.
start, stop = arg
if start is None:
start = self._min
if stop is None:
stop = self._max
# Check arguments.
if not ( isinstance(start,(int,long)) or start is _MININF ):
raise TypeError("Invalid type of tuple start.")
if not ( isinstance(stop,(int,long)) or stop is _MAXINF ):
raise TypeError("Invalid type of tuple stop.")
if ( start is not _MININF and stop is not _MAXINF and
start > stop ):
continue
if isinstance(stop,(int,long)):
stop += 1
else:
raise TypeError("Invalid argument.")
if start > self._max:
continue
elif start < self._min:
start = self._min
if stop < self._min:
continue
elif stop > self._max:
stop = self._max
self._ranges.append((start,stop))
# Normalize set.
self._normalize()
# Utility functions for set operations
# ------------------------------------
def _iterranges(self,r1,r2,minval=_MININF,maxval=_MAXINF):
curval = minval
curstates = {"r1":False,"r2":False}
imax, jmax = 2*len(r1), 2*len(r2)
i, j = 0, 0
while i < imax or j < jmax:
if i < imax and ( ( j < jmax and
r1[i>>1][i&1] < r2[j>>1][j&1] ) or
j == jmax ):
cur_r, newname, newstate = r1[i>>1][i&1], "r1", not (i&1)
i += 1
else:
cur_r, newname, newstate = r2[j>>1][j&1], "r2", not (j&1)
j += 1
if curval < cur_r:
if cur_r > maxval:
break
yield curstates, (curval,cur_r)
curval = cur_r
curstates[newname] = newstate
if curval < maxval:
yield curstates, (curval,maxval)
def _normalize(self):
self._ranges.sort()
i = 1
while i < len(self._ranges):
if self._ranges[i][0] < self._ranges[i-1][1]:
self._ranges[i-1] = (self._ranges[i-1][0],
max(self._ranges[i-1][1],
self._ranges[i][1]))
del self._ranges[i]
else:
i += 1
self._ranges = tuple(self._ranges)
self._hash = hash(self._ranges)
def __coerce__(self,other):
if isinstance(other,IntSet):
return self, other
elif isinstance(other,(int,long,tuple)):
try:
return self, self.__class__(other)
except TypeError:
# Catch a type error, in that case the structure specified by
# other is something we can't coerce, return NotImplemented.
# ValueErrors are not caught, they signal that the data was
# invalid for the constructor. This is appropriate to signal
# as a ValueError to the caller.
return NotImplemented
elif isinstance(other,list):
try:
return self, self.__class__(*other)
except TypeError:
# See above.
return NotImplemented
return NotImplemented
# Set function definitions
# ------------------------
def _make_function(name,type,doc,pall,pany=None):
"""Makes a function to match two ranges. Accepts two types: either
'set', which defines a function which returns a set with all ranges
matching pall (pany is ignored), or 'bool', which returns True if pall
matches for all ranges and pany matches for any one range. doc is the
dostring to give this function. pany may be none to ignore the any
match.
The predicates get a dict with two keys, 'r1', 'r2', which denote
whether the current range is present in range1 (self) and/or range2
(other) or none of the two, respectively."""
if type == "set":
def f(self,other):
coerced = self.__coerce__(other)
if coerced is NotImplemented:
return NotImplemented
other = coerced[1]
newset = self.__class__.__new__(self.__class__)
newset._min = min(self._min,other._min)
newset._max = max(self._max,other._max)
newset._ranges = []
for states, (start,stop) in \
self._iterranges(self._ranges,other._ranges,
newset._min,newset._max):
if pall(states):
if newset._ranges and newset._ranges[-1][1] == start:
newset._ranges[-1] = (newset._ranges[-1][0],stop)
else:
newset._ranges.append((start,stop))
newset._ranges = tuple(newset._ranges)
newset._hash = hash(self._ranges)
return newset
elif type == "bool":
def f(self,other):
coerced = self.__coerce__(other)
if coerced is NotImplemented:
return NotImplemented
other = coerced[1]
_min = min(self._min,other._min)
_max = max(self._max,other._max)
found = not pany
for states, (start,stop) in \
self._iterranges(self._ranges,other._ranges,_min,_max):
if not pall(states):
return False
found = found or pany(states)
return found
else:
raise ValueError("Invalid type of function to create.")
try:
f.func_name = name
except TypeError:
pass
f.func_doc = doc
return f
# Intersection.
__and__ = _make_function("__and__","set",
"Intersection of two sets as a new set.",
lambda s: s["r1"] and s["r2"])
__rand__ = _make_function("__rand__","set",
"Intersection of two sets as a new set.",
lambda s: s["r1"] and s["r2"])
intersection = _make_function("intersection","set",
"Intersection of two sets as a new set.",
lambda s: s["r1"] and s["r2"])
# Union.
__or__ = _make_function("__or__","set",
"Union of two sets as a new set.",
lambda s: s["r1"] or s["r2"])
__ror__ = _make_function("__ror__","set",
"Union of two sets as a new set.",
lambda s: s["r1"] or s["r2"])
union = _make_function("union","set",
"Union of two sets as a new set.",
lambda s: s["r1"] or s["r2"])
# Difference.
__sub__ = _make_function("__sub__","set",
"Difference of two sets as a new set.",
lambda s: s["r1"] and not s["r2"])
__rsub__ = _make_function("__rsub__","set",
"Difference of two sets as a new set.",
lambda s: s["r2"] and not s["r1"])
difference = _make_function("difference","set",
"Difference of two sets as a new set.",
lambda s: s["r1"] and not s["r2"])
# Symmetric difference.
__xor__ = _make_function("__xor__","set",
"Symmetric difference of two sets as a new set.",
lambda s: s["r1"] ^ s["r2"])
__rxor__ = _make_function("__rxor__","set",
"Symmetric difference of two sets as a new set.",
lambda s: s["r1"] ^ s["r2"])
symmetric_difference = _make_function("symmetric_difference","set",
"Symmetric difference of two sets as a new set.",
lambda s: s["r1"] ^ s["r2"])
# Containership testing.
__contains__ = _make_function("__contains__","bool",
"Returns true if self is superset of other.",
lambda s: s["r1"] or not s["r2"])
issubset = _make_function("issubset","bool",
"Returns true if self is subset of other.",
lambda s: s["r2"] or not s["r1"])
istruesubset = _make_function("istruesubset","bool",
"Returns true if self is true subset of other.",
lambda s: s["r2"] or not s["r1"],
lambda s: s["r2"] and not s["r1"])
issuperset = _make_function("issuperset","bool",
"Returns true if self is superset of other.",
lambda s: s["r1"] or not s["r2"])
istruesuperset = _make_function("istruesuperset","bool",
"Returns true if self is true superset of other.",
lambda s: s["r1"] or not s["r2"],
lambda s: s["r1"] and not s["r2"])
overlaps = _make_function("overlaps","bool",
"Returns true if self overlaps with other.",
lambda s: True,
lambda s: s["r1"] and s["r2"])
# Comparison.
__eq__ = _make_function("__eq__","bool",
"Returns true if self is equal to other.",
lambda s: not ( s["r1"] ^ s["r2"] ))
__ne__ = _make_function("__ne__","bool",
"Returns true if self is different to other.",
lambda s: True,
lambda s: s["r1"] ^ s["r2"])
# Clean up namespace.
del _make_function
# Define other functions.
def inverse(self):
"""Inverse of set as a new set."""
newset = self.__class__.__new__(self.__class__)
newset._min = self._min
newset._max = self._max
newset._ranges = []
laststop = self._min
for r in self._ranges:
if laststop < r[0]:
newset._ranges.append((laststop,r[0]))
laststop = r[1]
if laststop < self._max:
newset._ranges.append((laststop,self._max))
return newset
__invert__ = inverse
# Hashing
# -------
def __hash__(self):
"""Returns a hash value representing this integer set. As the set is
always stored normalized, the hash value is guaranteed to match for
matching ranges."""
return self._hash
# Iterating
# ---------
def __len__(self):
"""Get length of this integer set. In case the length is larger than
2**31 (including infinitely sized integer sets), it raises an
OverflowError. This is due to len() restricting the size to
0 <= len < 2**31."""
if not self._ranges:
return 0
if self._ranges[0][0] is _MININF or self._ranges[-1][1] is _MAXINF:
raise OverflowError("Infinitely sized integer set.")
rlen = 0
for r in self._ranges:
rlen += r[1]-r[0]
if rlen >= 2**31:
raise OverflowError("Integer set bigger than 2**31.")
return rlen
def len(self):
"""Returns the length of this integer set as an integer. In case the
length is infinite, returns -1. This function exists because of a
limitation of the builtin len() function which expects values in
the range 0 <= len < 2**31. Use this function in case your integer
set might be larger."""
if not self._ranges:
return 0
if self._ranges[0][0] is _MININF or self._ranges[-1][1] is _MAXINF:
return -1
rlen = 0
for r in self._ranges:
rlen += r[1]-r[0]
return rlen
def __nonzero__(self):
"""Returns true if this integer set contains at least one item."""
return bool(self._ranges)
def __iter__(self):
"""Iterate over all values in this integer set. Iteration always starts
by iterating from lowest to highest over the ranges that are bounded.
After processing these, all ranges that are unbounded (maximum 2) are
yielded intermixed."""
ubranges = []
for r in self._ranges:
if r[0] is _MININF:
if r[1] is _MAXINF:
ubranges.extend(([0,1],[-1,-1]))
else:
ubranges.append([r[1]-1,-1])
elif r[1] is _MAXINF:
ubranges.append([r[0],1])
else:
for val in xrange(r[0],r[1]):
yield val
if ubranges:
while True:
for ubrange in ubranges:
yield ubrange[0]
ubrange[0] += ubrange[1]
# Printing
# --------
def __repr__(self):
"""Return a representation of this integer set. The representation is
executable to get an equal integer set."""
rv = []
for start, stop in self._ranges:
if ( isinstance(start,(int,long)) and isinstance(stop,(int,long))
and stop-start == 1 ):
rv.append("%r" % start)
elif isinstance(stop,(int,long)):
rv.append("(%r,%r)" % (start,stop-1))
else:
rv.append("(%r,%r)" % (start,stop))
if self._min is not _MININF:
rv.append("min=%r" % self._min)
if self._max is not _MAXINF:
rv.append("max=%r" % self._max)
return "%s(%s)" % (self.__class__.__name__,",".join(rv))
if __name__ == "__main__":
# Little test script demonstrating functionality.
x = IntSet((10,20),30)
y = IntSet((10,20))
z = IntSet((10,20),30,(15,19),min=0,max=40)
print x
print x&110
print x|110
print x^(15,25)
print x-12
print 12 in x
print x.issubset(x)
print y.issubset(x)
print x.istruesubset(x)
print y.istruesubset(x)
for val in x:
print val
print x.inverse()
print x == z
print x == y
print x <> y
print hash(x)
print hash(z)
print len(x)
print x.len()

View File

@@ -0,0 +1,273 @@
# -*- coding: iso-8859-15 -*-
"""IP4 address range set implementation.
Implements an IPv4-range type.
Copyright (C) 2006, Heiko Wundram.
Released under the MIT-license.
"""
# Version information
# -------------------
__author__ = "Heiko Wundram <me@modelnine.org>"
__version__ = "0.2"
__revision__ = "3"
__date__ = "2006-01-20"
# Imports
# -------
import intset
import socket
# IP4Range class
# --------------
class IP4Range(intset.IntSet):
"""IP4 address range class with efficient storage of address ranges.
Supports all set operations."""
_MINIP4 = 0
_MAXIP4 = (1<<32) - 1
_UNITYTRANS = "".join([chr(n) for n in range(256)])
_IPREMOVE = "0123456789."
def __init__(self,*args):
"""Initialize an ip4range class. The constructor accepts an unlimited
number of arguments that may either be tuples in the form (start,stop),
integers, longs or strings, where start and stop in a tuple may
also be of the form integer, long or string.
Passing an integer or long means passing an IPv4-address that's already
been converted to integer notation, whereas passing a string specifies
an address where this conversion still has to be done. A string
address may be in the following formats:
- 1.2.3.4 - a plain address, interpreted as a single address
- 1.2.3 - a set of addresses, interpreted as 1.2.3.0-1.2.3.255
- localhost - hostname to look up, interpreted as single address
- 1.2.3<->5 - a set of addresses, interpreted as 1.2.3.0-1.2.5.255
- 1.2.0.0/16 - a set of addresses, interpreted as 1.2.0.0-1.2.255.255
Only the first three notations are valid if you use a string address in
a tuple, whereby notation 2 is interpreted as 1.2.3.0 if specified as
lower bound and 1.2.3.255 if specified as upper bound, not as a range
of addresses.
Specifying a range is done with the <-> operator. This is necessary
because '-' might be present in a hostname. '<->' shouldn't be, ever.
"""
# Special case copy constructor.
if len(args) == 1 and isinstance(args[0],IP4Range):
super(IP4Range,self).__init__(args[0])
return
# Convert arguments to tuple syntax.
args = list(args)
for i in range(len(args)):
argval = args[i]
if isinstance(argval,str):
if "<->" in argval:
# Type 4 address.
args[i] = self._parseRange(*argval.split("<->",1))
continue
elif "/" in argval:
# Type 5 address.
args[i] = self._parseMask(*argval.split("/",1))
else:
# Type 1, 2 or 3.
args[i] = self._parseAddrRange(argval)
elif isinstance(argval,tuple):
if len(tuple) <> 2:
raise ValueError("Tuple is of invalid length.")
addr1, addr2 = argval
if isinstance(addr1,str):
addr1 = self._parseAddrRange(addr1)[0]
elif not isinstance(addr1,(int,long)):
raise TypeError("Invalid argument.")
if isinstance(addr2,str):
addr2 = self._parseAddrRange(addr2)[1]
elif not isinstance(addr2,(int,long)):
raise TypeError("Invalid argument.")
args[i] = (addr1,addr2)
elif not isinstance(argval,(int,long)):
raise TypeError("Invalid argument.")
# Initialize the integer set.
super(IP4Range,self).__init__(min=self._MINIP4,max=self._MAXIP4,*args)
# Parsing functions
# -----------------
def _parseRange(self,addr1,addr2):
naddr1, naddr1len = _parseAddr(addr1)
naddr2, naddr2len = _parseAddr(addr2)
if naddr2len < naddr1len:
naddr2 += naddr1&(((1<<((naddr1len-naddr2len)*8))-1)<<
(naddr2len*8))
naddr2len = naddr1len
elif naddr2len > naddr1len:
raise ValueError("Range has more dots than address.")
naddr1 <<= (4-naddr1len)*8
naddr2 <<= (4-naddr2len)*8
naddr2 += (1<<((4-naddr2len)*8))-1
return (naddr1,naddr2)
def _parseMask(self,addr,mask):
naddr, naddrlen = _parseAddr(addr)
naddr <<= (4-naddrlen)*8
try:
if not mask:
masklen = 0
else:
masklen = int(mask)
if not 0 <= masklen <= 32:
raise ValueError
except ValueError:
try:
mask = _parseAddr(mask,False)
except ValueError:
raise ValueError("Mask isn't parseable.")
remaining = 0
masklen = 0
if not mask:
masklen = 0
else:
while not (mask&1):
remaining += 1
while (mask&1):
mask >>= 1
masklen += 1
if remaining+masklen <> 32:
raise ValueError("Mask isn't a proper host mask.")
naddr1 = naddr & (((1<<masklen)-1)<<(32-masklen))
naddr2 = naddr1 + (1<<(32-masklen)) - 1
return (naddr1,naddr2)
def _parseAddrRange(self,addr):
naddr, naddrlen = _parseAddr(addr)
naddr1 = naddr<<((4-naddrlen)*8)
naddr2 = ( (naddr<<((4-naddrlen)*8)) +
(1<<((4-naddrlen)*8)) - 1 )
return (naddr1,naddr2)
# Utility functions
# -----------------
def _int2ip(self,num):
rv = []
for i in range(4):
rv.append(str(num&255))
num >>= 8
return ".".join(reversed(rv))
# Iterating
# ---------
def iteraddresses(self):
"""Returns an iterator which iterates over ips in this iprange. An
IP is returned in string form (e.g. '1.2.3.4')."""
for v in super(IP4Range,self).__iter__():
yield self._int2ip(v)
def iterranges(self):
"""Returns an iterator which iterates over ip-ip ranges which build
this iprange if combined. An ip-ip pair is returned in string form
(e.g. '1.2.3.4-2.3.4.5')."""
for r in self._ranges:
if r[1]-r[0] == 1:
yield self._int2ip(r[0])
else:
yield '%s-%s' % (self._int2ip(r[0]),self._int2ip(r[1]-1))
def itermasks(self):
"""Returns an iterator which iterates over ip/mask pairs which build
this iprange if combined. An IP/Mask pair is returned in string form
(e.g. '1.2.3.0/24')."""
for r in self._ranges:
for v in self._itermasks(r):
yield v
def _itermasks(self,r):
ranges = [r]
while ranges:
cur = ranges.pop()
curmask = 0
while True:
curmasklen = 1<<(32-curmask)
start = (cur[0]+curmasklen-1)&(((1<<curmask)-1)<<(32-curmask))
if start >= cur[0] and start+curmasklen <= cur[1]:
break
else:
curmask += 1
yield "%s/%s" % (self._int2ip(start),curmask)
if cur[0] < start:
ranges.append((cur[0],start))
if cur[1] > start+curmasklen:
ranges.append((start+curmasklen,cur[1]))
__iter__ = iteraddresses
# Printing
# --------
def __repr__(self):
"""Returns a string which can be used to reconstruct this iprange."""
rv = []
for start, stop in self._ranges:
if stop-start == 1:
rv.append("%r" % (self._int2ip(start),))
else:
rv.append("(%r,%r)" % (self._int2ip(start),
self._int2ip(stop-1)))
return "%s(%s)" % (self.__class__.__name__,",".join(rv))
def _parseAddr(addr,lookup=True):
if lookup and addr.translate(IP4Range._UNITYTRANS, IP4Range._IPREMOVE):
try:
addr = socket.gethostbyname(addr)
except socket.error:
raise ValueError("Invalid Hostname as argument.")
naddr = 0
for naddrpos, part in enumerate(addr.split(".")):
if naddrpos >= 4:
raise ValueError("Address contains more than four parts.")
try:
if not part:
part = 0
else:
part = int(part)
if not 0 <= part < 256:
raise ValueError
except ValueError:
raise ValueError("Address part out of range.")
naddr <<= 8
naddr += part
return naddr, naddrpos+1
def ip2int(addr, lookup=True):
return _parseAddr(addr, lookup=lookup)[0]
if __name__ == "__main__":
# Little test script.
x = IP4Range("172.22.162.250/24")
y = IP4Range("172.22.162.250","172.22.163.250","172.22.163.253<->255")
print x
for val in x.itermasks():
print val
for val in y.itermasks():
print val
for val in (x|y).itermasks():
print val
for val in (x^y).iterranges():
print val
for val in x:
print val

View File

@@ -0,0 +1,30 @@
"""
Kill a thread, from http://sebulba.wikispaces.com/recipe+thread2
"""
import types
try:
import ctypes
except ImportError:
raise ImportError(
"You cannot use paste.util.killthread without ctypes installed")
if not hasattr(ctypes, 'pythonapi'):
raise ImportError(
"You cannot use paste.util.killthread without ctypes.pythonapi")
def async_raise(tid, exctype):
"""raises the exception, performs cleanup if needed.
tid is the value given by thread.get_ident() (an integer).
Raise SystemExit to kill a thread."""
if not isinstance(exctype, (types.ClassType, type)):
raise TypeError("Only types can be raised (not instances)")
if not isinstance(tid, int):
raise TypeError("tid must be an integer")
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid), ctypes.py_object(exctype))
if res == 0:
raise ValueError("invalid thread id")
elif res != 1:
# """if it returns a number greater than one, you're in trouble,
# and you should call it again with exc=NULL to revert the effect"""
ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid), 0)
raise SystemError("PyThreadState_SetAsyncExc failed")

View File

@@ -0,0 +1,152 @@
"""
Helper for looping over sequences, particular in templates.
Often in a loop in a template it's handy to know what's next up,
previously up, if this is the first or last item in the sequence, etc.
These can be awkward to manage in a normal Python loop, but using the
looper you can get a better sense of the context. Use like::
>>> for loop, item in looper(['a', 'b', 'c']):
... print loop.number, item
... if not loop.last:
... print '---'
1 a
---
2 b
---
3 c
"""
__all__ = ['looper']
class looper(object):
"""
Helper for looping (particularly in templates)
Use this like::
for loop, item in looper(seq):
if loop.first:
...
"""
def __init__(self, seq):
self.seq = seq
def __iter__(self):
return looper_iter(self.seq)
def __repr__(self):
return '<%s for %r>' % (
self.__class__.__name__, self.seq)
class looper_iter(object):
def __init__(self, seq):
self.seq = list(seq)
self.pos = 0
def __iter__(self):
return self
def next(self):
if self.pos >= len(self.seq):
raise StopIteration
result = loop_pos(self.seq, self.pos), self.seq[self.pos]
self.pos += 1
return result
class loop_pos(object):
def __init__(self, seq, pos):
self.seq = seq
self.pos = pos
def __repr__(self):
return '<loop pos=%r at %r>' % (
self.seq[pos], pos)
def index(self):
return self.pos
index = property(index)
def number(self):
return self.pos + 1
number = property(number)
def item(self):
return self.seq[self.pos]
item = property(item)
def next(self):
try:
return self.seq[self.pos+1]
except IndexError:
return None
next = property(next)
def previous(self):
if self.pos == 0:
return None
return self.seq[self.pos-1]
previous = property(previous)
def odd(self):
return not self.pos % 2
odd = property(odd)
def even(self):
return self.pos % 2
even = property(even)
def first(self):
return self.pos == 0
first = property(first)
def last(self):
return self.pos == len(self.seq)-1
last = property(last)
def length(self):
return len(self.seq)
length = property(length)
def first_group(self, getter=None):
"""
Returns true if this item is the start of a new group,
where groups mean that some attribute has changed. The getter
can be None (the item itself changes), an attribute name like
``'.attr'``, a function, or a dict key or list index.
"""
if self.first:
return True
return self._compare_group(self.item, self.previous, getter)
def last_group(self, getter=None):
"""
Returns true if this item is the end of a new group,
where groups mean that some attribute has changed. The getter
can be None (the item itself changes), an attribute name like
``'.attr'``, a function, or a dict key or list index.
"""
if self.last:
return True
return self._compare_group(self.item, self.next, getter)
def _compare_group(self, item, other, getter):
if getter is None:
return item != other
elif (isinstance(getter, basestring)
and getter.startswith('.')):
getter = getter[1:]
if getter.endswith('()'):
getter = getter[:-2]
return getattr(item, getter)() != getattr(other, getter)()
else:
return getattr(item, getter) != getattr(other, getter)
elif callable(getter):
return getter(item) != getter(other)
else:
return item[getter] != other[getter]

View File

@@ -0,0 +1,160 @@
"""MIME-Type Parser
This module provides basic functions for handling mime-types. It can handle
matching mime-types against a list of media-ranges. See section 14.1 of
the HTTP specification [RFC 2616] for a complete explanation.
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
Based on mimeparse 0.1.2 by Joe Gregorio:
http://code.google.com/p/mimeparse/
Contents:
- parse_mime_type(): Parses a mime-type into its component parts.
- parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q' quality parameter.
- quality(): Determines the quality ('q') of a mime-type when compared against a list of media-ranges.
- quality_parsed(): Just like quality() except the second parameter must be pre-parsed.
- best_match(): Choose the mime-type with the highest quality ('q') from a list of candidates.
- desired_matches(): Filter against a list of desired mime-types in the order the server prefers.
"""
def parse_mime_type(mime_type):
"""Carves up a mime-type and returns a tuple of the
(type, subtype, params) where 'params' is a dictionary
of all the parameters for the media range.
For example, the media range 'application/xhtml;q=0.5' would
get parsed into:
('application', 'xhtml', {'q', '0.5'})
"""
type = mime_type.split(';')
type, plist = type[0], type[1:]
try:
type, subtype = type.split('/', 1)
except ValueError:
type, subtype = type.strip() or '*', '*'
else:
type = type.strip() or '*'
subtype = subtype.strip() or '*'
params = {}
for param in plist:
param = param.split('=', 1)
if len(param) == 2:
key, value = param[0].strip(), param[1].strip()
if key and value:
params[key] = value
return type, subtype, params
def parse_media_range(range):
"""Carves up a media range and returns a tuple of the
(type, subtype, params) where 'params' is a dictionary
of all the parameters for the media range.
For example, the media range 'application/*;q=0.5' would
get parsed into:
('application', '*', {'q', '0.5'})
In addition this function also guarantees that there
is a value for 'q' in the params dictionary, filling it
in with a proper default if necessary.
"""
type, subtype, params = parse_mime_type(range)
try:
if not 0 <= float(params['q']) <= 1:
raise ValueError
except (KeyError, ValueError):
params['q'] = '1'
return type, subtype, params
def fitness_and_quality_parsed(mime_type, parsed_ranges):
"""Find the best match for a given mime-type against
a list of media_ranges that have already been
parsed by parse_media_range(). Returns a tuple of
the fitness value and the value of the 'q' quality
parameter of the best match, or (-1, 0) if no match
was found. Just as for quality_parsed(), 'parsed_ranges'
must be a list of parsed media ranges."""
best_fitness, best_fit_q = -1, 0
target_type, target_subtype, target_params = parse_media_range(mime_type)
for type, subtype, params in parsed_ranges:
if (type == target_type
or type == '*' or target_type == '*') and (
subtype == target_subtype
or subtype == '*' or target_subtype == '*'):
fitness = 0
if type == target_type:
fitness += 100
if subtype == target_subtype:
fitness += 10
for key in target_params:
if key != 'q' and key in params:
if params[key] == target_params[key]:
fitness += 1
if fitness > best_fitness:
best_fitness = fitness
best_fit_q = params['q']
return best_fitness, float(best_fit_q)
def quality_parsed(mime_type, parsed_ranges):
"""Find the best match for a given mime-type against
a list of media_ranges that have already been
parsed by parse_media_range(). Returns the
'q' quality parameter of the best match, 0 if no
match was found. This function behaves the same as quality()
except that 'parsed_ranges' must be a list of
parsed media ranges."""
return fitness_and_quality_parsed(mime_type, parsed_ranges)[1]
def quality(mime_type, ranges):
"""Returns the quality 'q' of a mime-type when compared
against the media-ranges in ranges. For example:
>>> quality('text/html','text/*;q=0.3, text/html;q=0.7, text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5')
0.7
"""
parsed_ranges = map(parse_media_range, ranges.split(','))
return quality_parsed(mime_type, parsed_ranges)
def best_match(supported, header):
"""Takes a list of supported mime-types and finds the best
match for all the media-ranges listed in header. In case of
ambiguity, whatever comes first in the list will be chosen.
The value of header must be a string that conforms to the format
of the HTTP Accept: header. The value of 'supported' is a list
of mime-types.
>>> best_match(['application/xbel+xml', 'text/xml'], 'text/*;q=0.5,*/*; q=0.1')
'text/xml'
"""
if not supported:
return ''
parsed_header = map(parse_media_range, header.split(','))
best_type = max([
(fitness_and_quality_parsed(mime_type, parsed_header), -n)
for n, mime_type in enumerate(supported)])
return best_type[0][1] and supported[-best_type[1]] or ''
def desired_matches(desired, header):
"""Takes a list of desired mime-types in the order the server prefers to
send them regardless of the browsers preference.
Browsers (such as Firefox) technically want XML over HTML depending on how
one reads the specification. This function is provided for a server to
declare a set of desired mime-types it supports, and returns a subset of
the desired list in the same order should each one be Accepted by the
browser.
>>> desired_matches(['text/html', 'application/xml'], \
... 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png')
['text/html', 'application/xml']
>>> desired_matches(['text/html', 'application/xml'], 'application/xml,application/json')
['application/xml']
"""
parsed_ranges = map(parse_media_range, header.split(','))
return [mimetype for mimetype in desired
if quality_parsed(mimetype, parsed_ranges)]

View File

@@ -0,0 +1,397 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
import cgi
import copy
import sys
from UserDict import DictMixin
class MultiDict(DictMixin):
"""
An ordered dictionary that can have multiple values for each key.
Adds the methods getall, getone, mixed, and add to the normal
dictionary interface.
"""
def __init__(self, *args, **kw):
if len(args) > 1:
raise TypeError(
"MultiDict can only be called with one positional argument")
if args:
if hasattr(args[0], 'iteritems'):
items = list(args[0].iteritems())
elif hasattr(args[0], 'items'):
items = args[0].items()
else:
items = list(args[0])
self._items = items
else:
self._items = []
self._items.extend(kw.iteritems())
def __getitem__(self, key):
for k, v in self._items:
if k == key:
return v
raise KeyError(repr(key))
def __setitem__(self, key, value):
try:
del self[key]
except KeyError:
pass
self._items.append((key, value))
def add(self, key, value):
"""
Add the key and value, not overwriting any previous value.
"""
self._items.append((key, value))
def getall(self, key):
"""
Return a list of all values matching the key (may be an empty list)
"""
result = []
for k, v in self._items:
if key == k:
result.append(v)
return result
def getone(self, key):
"""
Get one value matching the key, raising a KeyError if multiple
values were found.
"""
v = self.getall(key)
if not v:
raise KeyError('Key not found: %r' % key)
if len(v) > 1:
raise KeyError('Multiple values match %r: %r' % (key, v))
return v[0]
def mixed(self):
"""
Returns a dictionary where the values are either single
values, or a list of values when a key/value appears more than
once in this dictionary. This is similar to the kind of
dictionary often used to represent the variables in a web
request.
"""
result = {}
multi = {}
for key, value in self._items:
if key in result:
# We do this to not clobber any lists that are
# *actual* values in this dictionary:
if key in multi:
result[key].append(value)
else:
result[key] = [result[key], value]
multi[key] = None
else:
result[key] = value
return result
def dict_of_lists(self):
"""
Returns a dictionary where each key is associated with a
list of values.
"""
result = {}
for key, value in self._items:
if key in result:
result[key].append(value)
else:
result[key] = [value]
return result
def __delitem__(self, key):
items = self._items
found = False
for i in range(len(items)-1, -1, -1):
if items[i][0] == key:
del items[i]
found = True
if not found:
raise KeyError(repr(key))
def __contains__(self, key):
for k, v in self._items:
if k == key:
return True
return False
has_key = __contains__
def clear(self):
self._items = []
def copy(self):
return MultiDict(self)
def setdefault(self, key, default=None):
for k, v in self._items:
if key == k:
return v
self._items.append((key, default))
return default
def pop(self, key, *args):
if len(args) > 1:
raise TypeError, "pop expected at most 2 arguments, got "\
+ repr(1 + len(args))
for i in range(len(self._items)):
if self._items[i][0] == key:
v = self._items[i][1]
del self._items[i]
return v
if args:
return args[0]
else:
raise KeyError(repr(key))
def popitem(self):
return self._items.pop()
def update(self, other=None, **kwargs):
if other is None:
pass
elif hasattr(other, 'items'):
self._items.extend(other.items())
elif hasattr(other, 'keys'):
for k in other.keys():
self._items.append((k, other[k]))
else:
for k, v in other:
self._items.append((k, v))
if kwargs:
self.update(kwargs)
def __repr__(self):
items = ', '.join(['(%r, %r)' % v for v in self._items])
return '%s([%s])' % (self.__class__.__name__, items)
def __len__(self):
return len(self._items)
##
## All the iteration:
##
def keys(self):
return [k for k, v in self._items]
def iterkeys(self):
for k, v in self._items:
yield k
__iter__ = iterkeys
def items(self):
return self._items[:]
def iteritems(self):
return iter(self._items)
def values(self):
return [v for k, v in self._items]
def itervalues(self):
for k, v in self._items:
yield v
class UnicodeMultiDict(DictMixin):
"""
A MultiDict wrapper that decodes returned values to unicode on the
fly. Decoding is not applied to assigned values.
The key/value contents are assumed to be ``str``/``strs`` or
``str``/``FieldStorages`` (as is returned by the ``paste.request.parse_``
functions).
Can optionally also decode keys when the ``decode_keys`` argument is
True.
``FieldStorage`` instances are cloned, and the clone's ``filename``
variable is decoded. Its ``name`` variable is decoded when ``decode_keys``
is enabled.
"""
def __init__(self, multi=None, encoding=None, errors='strict',
decode_keys=False):
self.multi = multi
if encoding is None:
encoding = sys.getdefaultencoding()
self.encoding = encoding
self.errors = errors
self.decode_keys = decode_keys
def _decode_key(self, key):
if self.decode_keys:
try:
key = key.decode(self.encoding, self.errors)
except AttributeError:
pass
return key
def _decode_value(self, value):
"""
Decode the specified value to unicode. Assumes value is a ``str`` or
`FieldStorage`` object.
``FieldStorage`` objects are specially handled.
"""
if isinstance(value, cgi.FieldStorage):
# decode FieldStorage's field name and filename
value = copy.copy(value)
if self.decode_keys:
value.name = value.name.decode(self.encoding, self.errors)
value.filename = value.filename.decode(self.encoding, self.errors)
else:
try:
value = value.decode(self.encoding, self.errors)
except AttributeError:
pass
return value
def __getitem__(self, key):
return self._decode_value(self.multi.__getitem__(key))
def __setitem__(self, key, value):
self.multi.__setitem__(key, value)
def add(self, key, value):
"""
Add the key and value, not overwriting any previous value.
"""
self.multi.add(key, value)
def getall(self, key):
"""
Return a list of all values matching the key (may be an empty list)
"""
return [self._decode_value(v) for v in self.multi.getall(key)]
def getone(self, key):
"""
Get one value matching the key, raising a KeyError if multiple
values were found.
"""
return self._decode_value(self.multi.getone(key))
def mixed(self):
"""
Returns a dictionary where the values are either single
values, or a list of values when a key/value appears more than
once in this dictionary. This is similar to the kind of
dictionary often used to represent the variables in a web
request.
"""
unicode_mixed = {}
for key, value in self.multi.mixed().iteritems():
if isinstance(value, list):
value = [self._decode_value(value) for value in value]
else:
value = self._decode_value(value)
unicode_mixed[self._decode_key(key)] = value
return unicode_mixed
def dict_of_lists(self):
"""
Returns a dictionary where each key is associated with a
list of values.
"""
unicode_dict = {}
for key, value in self.multi.dict_of_lists().iteritems():
value = [self._decode_value(value) for value in value]
unicode_dict[self._decode_key(key)] = value
return unicode_dict
def __delitem__(self, key):
self.multi.__delitem__(key)
def __contains__(self, key):
return self.multi.__contains__(key)
has_key = __contains__
def clear(self):
self.multi.clear()
def copy(self):
return UnicodeMultiDict(self.multi.copy(), self.encoding, self.errors)
def setdefault(self, key, default=None):
return self._decode_value(self.multi.setdefault(key, default))
def pop(self, key, *args):
return self._decode_value(self.multi.pop(key, *args))
def popitem(self):
k, v = self.multi.popitem()
return (self._decode_key(k), self._decode_value(v))
def __repr__(self):
items = ', '.join(['(%r, %r)' % v for v in self.items()])
return '%s([%s])' % (self.__class__.__name__, items)
def __len__(self):
return self.multi.__len__()
##
## All the iteration:
##
def keys(self):
return [self._decode_key(k) for k in self.multi.iterkeys()]
def iterkeys(self):
for k in self.multi.iterkeys():
yield self._decode_key(k)
__iter__ = iterkeys
def items(self):
return [(self._decode_key(k), self._decode_value(v)) for \
k, v in self.multi.iteritems()]
def iteritems(self):
for k, v in self.multi.iteritems():
yield (self._decode_key(k), self._decode_value(v))
def values(self):
return [self._decode_value(v) for v in self.multi.itervalues()]
def itervalues(self):
for v in self.multi.itervalues():
yield self._decode_value(v)
__test__ = {
'general': """
>>> d = MultiDict(a=1, b=2)
>>> d['a']
1
>>> d.getall('c')
[]
>>> d.add('a', 2)
>>> d['a']
1
>>> d.getall('a')
[1, 2]
>>> d['b'] = 4
>>> d.getall('b')
[4]
>>> d.keys()
['a', 'a', 'b']
>>> d.items()
[('a', 1), ('a', 2), ('b', 4)]
>>> d.mixed()
{'a': [1, 2], 'b': 4}
>>> MultiDict([('a', 'b')], c=2)
MultiDict([('a', 'b'), ('c', 2)])
"""}
if __name__ == '__main__':
import doctest
doctest.testmod()

View File

@@ -0,0 +1,98 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
import cgi
import htmlentitydefs
import urllib
import re
__all__ = ['html_quote', 'html_unquote', 'url_quote', 'url_unquote',
'strip_html']
default_encoding = 'UTF-8'
def html_quote(v, encoding=None):
r"""
Quote the value (turned to a string) as HTML. This quotes <, >,
and quotes:
>>> html_quote(1)
'1'
>>> html_quote(None)
''
>>> html_quote('<hey!>')
'&lt;hey!&gt;'
>>> html_quote(u'\u1029')
'\xe1\x80\xa9'
"""
encoding = encoding or default_encoding
if v is None:
return ''
elif isinstance(v, str):
return cgi.escape(v, 1)
elif isinstance(v, unicode):
return cgi.escape(v.encode(encoding), 1)
else:
return cgi.escape(unicode(v).encode(encoding), 1)
_unquote_re = re.compile(r'&([a-zA-Z]+);')
def _entity_subber(match, name2c=htmlentitydefs.name2codepoint):
code = name2c.get(match.group(1))
if code:
return unichr(code)
else:
return match.group(0)
def html_unquote(s, encoding=None):
r"""
Decode the value.
>>> html_unquote('&lt;hey&nbsp;you&gt;')
u'<hey\xa0you>'
>>> html_unquote('')
u''
>>> html_unquote('&blahblah;')
u'&blahblah;'
>>> html_unquote('\xe1\x80\xa9')
u'\u1029'
"""
if isinstance(s, str):
if s == '':
# workaround re.sub('', '', u'') returning '' < 2.5.2
# instead of u'' >= 2.5.2
return u''
s = s.decode(encoding or default_encoding)
return _unquote_re.sub(_entity_subber, s)
def strip_html(s):
# should this use html_unquote?
s = re.sub('<.*?>', '', s)
s = html_unquote(s)
return s
def no_quote(s):
"""
Quoting that doesn't do anything
"""
return s
_comment_quote_re = re.compile(r'\-\s*\>')
# Everything but \r, \n, \t:
_bad_chars_re = re.compile('[\x00-\x08\x0b-\x0c\x0e-\x1f]')
def comment_quote(s):
"""
Quote that makes sure text can't escape a comment
"""
comment = str(s)
#comment = _bad_chars_re.sub('', comment)
#print 'in ', repr(str(s))
#print 'out', repr(comment)
comment = _comment_quote_re.sub('-&gt;', comment)
return comment
url_quote = urllib.quote
url_unquote = urllib.unquote
if __name__ == '__main__':
import doctest
doctest.testmod()

View File

@@ -0,0 +1,171 @@
"""
SCGI-->WSGI application proxy, "SWAP".
(Originally written by Titus Brown.)
This lets an SCGI front-end like mod_scgi be used to execute WSGI
application objects. To use it, subclass the SWAP class like so::
class TestAppHandler(swap.SWAP):
def __init__(self, *args, **kwargs):
self.prefix = '/canal'
self.app_obj = TestAppClass
swap.SWAP.__init__(self, *args, **kwargs)
where 'TestAppClass' is the application object from WSGI and '/canal'
is the prefix for what is served by the SCGI Web-server-side process.
Then execute the SCGI handler "as usual" by doing something like this::
scgi_server.SCGIServer(TestAppHandler, port=4000).serve()
and point mod_scgi (or whatever your SCGI front end is) at port 4000.
Kudos to the WSGI folk for writing a nice PEP & the Quixote folk for
writing a nice extensible SCGI server for Python!
"""
import sys
import time
from scgi import scgi_server
def debug(msg):
timestamp = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))
sys.stderr.write("[%s] %s\n" % (timestamp, msg))
class SWAP(scgi_server.SCGIHandler):
"""
SCGI->WSGI application proxy: let an SCGI server execute WSGI
application objects.
"""
app_obj = None
prefix = None
def __init__(self, *args, **kwargs):
assert self.app_obj, "must set app_obj"
assert self.prefix is not None, "must set prefix"
args = (self,) + args
scgi_server.SCGIHandler.__init__(*args, **kwargs)
def handle_connection(self, conn):
"""
Handle an individual connection.
"""
input = conn.makefile("r")
output = conn.makefile("w")
environ = self.read_env(input)
environ['wsgi.input'] = input
environ['wsgi.errors'] = sys.stderr
environ['wsgi.version'] = (1, 0)
environ['wsgi.multithread'] = False
environ['wsgi.multiprocess'] = True
environ['wsgi.run_once'] = False
# dunno how SCGI does HTTPS signalling; can't test it myself... @CTB
if environ.get('HTTPS','off') in ('on','1'):
environ['wsgi.url_scheme'] = 'https'
else:
environ['wsgi.url_scheme'] = 'http'
## SCGI does some weird environ manglement. We need to set
## SCRIPT_NAME from 'prefix' and then set PATH_INFO from
## REQUEST_URI.
prefix = self.prefix
path = environ['REQUEST_URI'][len(prefix):].split('?', 1)[0]
environ['SCRIPT_NAME'] = prefix
environ['PATH_INFO'] = path
headers_set = []
headers_sent = []
chunks = []
def write(data):
chunks.append(data)
def start_response(status, response_headers, exc_info=None):
if exc_info:
try:
if headers_sent:
# Re-raise original exception if headers sent
raise exc_info[0], exc_info[1], exc_info[2]
finally:
exc_info = None # avoid dangling circular ref
elif headers_set:
raise AssertionError("Headers already set!")
headers_set[:] = [status, response_headers]
return write
###
result = self.app_obj(environ, start_response)
try:
for data in result:
chunks.append(data)
# Before the first output, send the stored headers
if not headers_set:
# Error -- the app never called start_response
status = '500 Server Error'
response_headers = [('Content-type', 'text/html')]
chunks = ["XXX start_response never called"]
else:
status, response_headers = headers_sent[:] = headers_set
output.write('Status: %s\r\n' % status)
for header in response_headers:
output.write('%s: %s\r\n' % header)
output.write('\r\n')
for data in chunks:
output.write(data)
finally:
if hasattr(result,'close'):
result.close()
# SCGI backends use connection closing to signal 'fini'.
try:
input.close()
output.close()
conn.close()
except IOError, err:
debug("IOError while closing connection ignored: %s" % err)
def serve_application(application, prefix, port=None, host=None, max_children=None):
"""
Serve the specified WSGI application via SCGI proxy.
``application``
The WSGI application to serve.
``prefix``
The prefix for what is served by the SCGI Web-server-side process.
``port``
Optional port to bind the SCGI proxy to. Defaults to SCGIServer's
default port value.
``host``
Optional host to bind the SCGI proxy to. Defaults to SCGIServer's
default host value.
``host``
Optional maximum number of child processes the SCGIServer will
spawn. Defaults to SCGIServer's default max_children value.
"""
class SCGIAppHandler(SWAP):
def __init__ (self, *args, **kwargs):
self.prefix = prefix
self.app_obj = application
SWAP.__init__(self, *args, **kwargs)
kwargs = dict(handler_class=SCGIAppHandler)
for kwarg in ('host', 'port', 'max_children'):
if locals()[kwarg] is not None:
kwargs[kwarg] = locals()[kwarg]
scgi_server.SCGIServer(**kwargs).serve()

View File

@@ -0,0 +1,531 @@
"""A collection of string operations (most are no longer used).
Warning: most of the code you see here isn't normally used nowadays.
Beginning with Python 1.6, many of these functions are implemented as
methods on the standard string object. They used to be implemented by
a built-in module called strop, but strop is now obsolete itself.
Public module variables:
whitespace -- a string containing all characters considered whitespace
lowercase -- a string containing all characters considered lowercase letters
uppercase -- a string containing all characters considered uppercase letters
letters -- a string containing all characters considered letters
digits -- a string containing all characters considered decimal digits
hexdigits -- a string containing all characters considered hexadecimal digits
octdigits -- a string containing all characters considered octal digits
punctuation -- a string containing all characters considered punctuation
printable -- a string containing all characters considered printable
"""
# Some strings for ctype-style character classification
whitespace = ' \t\n\r\v\f'
lowercase = 'abcdefghijklmnopqrstuvwxyz'
uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
letters = lowercase + uppercase
ascii_lowercase = lowercase
ascii_uppercase = uppercase
ascii_letters = ascii_lowercase + ascii_uppercase
digits = '0123456789'
hexdigits = digits + 'abcdef' + 'ABCDEF'
octdigits = '01234567'
punctuation = """!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
printable = digits + letters + punctuation + whitespace
# Case conversion helpers
# Use str to convert Unicode literal in case of -U
# Note that Cookie.py bogusly uses _idmap :(
l = map(chr, xrange(256))
_idmap = str('').join(l)
del l
# Functions which aren't available as string methods.
# Capitalize the words in a string, e.g. " aBc dEf " -> "Abc Def".
# See also regsub.capwords().
def capwords(s, sep=None):
"""capwords(s, [sep]) -> string
Split the argument into words using split, capitalize each
word using capitalize, and join the capitalized words using
join. Note that this replaces runs of whitespace characters by
a single space.
"""
return (sep or ' ').join([x.capitalize() for x in s.split(sep)])
# Construct a translation string
_idmapL = None
def maketrans(fromstr, tostr):
"""maketrans(frm, to) -> string
Return a translation table (a string of 256 bytes long)
suitable for use in string.translate. The strings frm and to
must be of the same length.
"""
if len(fromstr) != len(tostr):
raise ValueError, "maketrans arguments must have same length"
global _idmapL
if not _idmapL:
_idmapL = map(None, _idmap)
L = _idmapL[:]
fromstr = map(ord, fromstr)
for i in range(len(fromstr)):
L[fromstr[i]] = tostr[i]
return ''.join(L)
####################################################################
import re as _re
class _multimap:
"""Helper class for combining multiple mappings.
Used by .{safe_,}substitute() to combine the mapping and keyword
arguments.
"""
def __init__(self, primary, secondary):
self._primary = primary
self._secondary = secondary
def __getitem__(self, key):
try:
return self._primary[key]
except KeyError:
return self._secondary[key]
class _TemplateMetaclass(type):
pattern = r"""
%(delim)s(?:
(?P<escaped>%(delim)s) | # Escape sequence of two delimiters
(?P<named>%(id)s) | # delimiter and a Python identifier
{(?P<braced>%(id)s)} | # delimiter and a braced identifier
(?P<invalid>) # Other ill-formed delimiter exprs
)
"""
def __init__(cls, name, bases, dct):
super(_TemplateMetaclass, cls).__init__(name, bases, dct)
if 'pattern' in dct:
pattern = cls.pattern
else:
pattern = _TemplateMetaclass.pattern % {
'delim' : _re.escape(cls.delimiter),
'id' : cls.idpattern,
}
cls.pattern = _re.compile(pattern, _re.IGNORECASE | _re.VERBOSE)
class Template:
"""A string class for supporting $-substitutions."""
__metaclass__ = _TemplateMetaclass
delimiter = '$'
idpattern = r'[_a-z][_a-z0-9]*'
def __init__(self, template):
self.template = template
# Search for $$, $identifier, ${identifier}, and any bare $'s
def _invalid(self, mo):
i = mo.start('invalid')
lines = self.template[:i].splitlines(True)
if not lines:
colno = 1
lineno = 1
else:
colno = i - len(''.join(lines[:-1]))
lineno = len(lines)
raise ValueError('Invalid placeholder in string: line %d, col %d' %
(lineno, colno))
def substitute(self, *args, **kws):
if len(args) > 1:
raise TypeError('Too many positional arguments')
if not args:
mapping = kws
elif kws:
mapping = _multimap(kws, args[0])
else:
mapping = args[0]
# Helper function for .sub()
def convert(mo):
# Check the most common path first.
named = mo.group('named') or mo.group('braced')
if named is not None:
val = mapping[named]
# We use this idiom instead of str() because the latter will
# fail if val is a Unicode containing non-ASCII characters.
return '%s' % val
if mo.group('escaped') is not None:
return self.delimiter
if mo.group('invalid') is not None:
self._invalid(mo)
raise ValueError('Unrecognized named group in pattern',
self.pattern)
return self.pattern.sub(convert, self.template)
def safe_substitute(self, *args, **kws):
if len(args) > 1:
raise TypeError('Too many positional arguments')
if not args:
mapping = kws
elif kws:
mapping = _multimap(kws, args[0])
else:
mapping = args[0]
# Helper function for .sub()
def convert(mo):
named = mo.group('named')
if named is not None:
try:
# We use this idiom instead of str() because the latter
# will fail if val is a Unicode containing non-ASCII
return '%s' % mapping[named]
except KeyError:
return self.delimiter + named
braced = mo.group('braced')
if braced is not None:
try:
return '%s' % mapping[braced]
except KeyError:
return self.delimiter + '{' + braced + '}'
if mo.group('escaped') is not None:
return self.delimiter
if mo.group('invalid') is not None:
return self.delimiter
raise ValueError('Unrecognized named group in pattern',
self.pattern)
return self.pattern.sub(convert, self.template)
####################################################################
# NOTE: Everything below here is deprecated. Use string methods instead.
# This stuff will go away in Python 3.0.
# Backward compatible names for exceptions
index_error = ValueError
atoi_error = ValueError
atof_error = ValueError
atol_error = ValueError
# convert UPPER CASE letters to lower case
def lower(s):
"""lower(s) -> string
Return a copy of the string s converted to lowercase.
"""
return s.lower()
# Convert lower case letters to UPPER CASE
def upper(s):
"""upper(s) -> string
Return a copy of the string s converted to uppercase.
"""
return s.upper()
# Swap lower case letters and UPPER CASE
def swapcase(s):
"""swapcase(s) -> string
Return a copy of the string s with upper case characters
converted to lowercase and vice versa.
"""
return s.swapcase()
# Strip leading and trailing tabs and spaces
def strip(s, chars=None):
"""strip(s [,chars]) -> string
Return a copy of the string s with leading and trailing
whitespace removed.
If chars is given and not None, remove characters in chars instead.
If chars is unicode, S will be converted to unicode before stripping.
"""
return s.strip(chars)
# Strip leading tabs and spaces
def lstrip(s, chars=None):
"""lstrip(s [,chars]) -> string
Return a copy of the string s with leading whitespace removed.
If chars is given and not None, remove characters in chars instead.
"""
return s.lstrip(chars)
# Strip trailing tabs and spaces
def rstrip(s, chars=None):
"""rstrip(s [,chars]) -> string
Return a copy of the string s with trailing whitespace removed.
If chars is given and not None, remove characters in chars instead.
"""
return s.rstrip(chars)
# Split a string into a list of space/tab-separated words
def split(s, sep=None, maxsplit=-1):
"""split(s [,sep [,maxsplit]]) -> list of strings
Return a list of the words in the string s, using sep as the
delimiter string. If maxsplit is given, splits at no more than
maxsplit places (resulting in at most maxsplit+1 words). If sep
is not specified or is None, any whitespace string is a separator.
(split and splitfields are synonymous)
"""
return s.split(sep, maxsplit)
splitfields = split
# Split a string into a list of space/tab-separated words
def rsplit(s, sep=None, maxsplit=-1):
"""rsplit(s [,sep [,maxsplit]]) -> list of strings
Return a list of the words in the string s, using sep as the
delimiter string, starting at the end of the string and working
to the front. If maxsplit is given, at most maxsplit splits are
done. If sep is not specified or is None, any whitespace string
is a separator.
"""
return s.rsplit(sep, maxsplit)
# Join fields with optional separator
def join(words, sep = ' '):
"""join(list [,sep]) -> string
Return a string composed of the words in list, with
intervening occurrences of sep. The default separator is a
single space.
(joinfields and join are synonymous)
"""
return sep.join(words)
joinfields = join
# Find substring, raise exception if not found
def index(s, *args):
"""index(s, sub [,start [,end]]) -> int
Like find but raises ValueError when the substring is not found.
"""
return s.index(*args)
# Find last substring, raise exception if not found
def rindex(s, *args):
"""rindex(s, sub [,start [,end]]) -> int
Like rfind but raises ValueError when the substring is not found.
"""
return s.rindex(*args)
# Count non-overlapping occurrences of substring
def count(s, *args):
"""count(s, sub[, start[,end]]) -> int
Return the number of occurrences of substring sub in string
s[start:end]. Optional arguments start and end are
interpreted as in slice notation.
"""
return s.count(*args)
# Find substring, return -1 if not found
def find(s, *args):
"""find(s, sub [,start [,end]]) -> in
Return the lowest index in s where substring sub is found,
such that sub is contained within s[start,end]. Optional
arguments start and end are interpreted as in slice notation.
Return -1 on failure.
"""
return s.find(*args)
# Find last substring, return -1 if not found
def rfind(s, *args):
"""rfind(s, sub [,start [,end]]) -> int
Return the highest index in s where substring sub is found,
such that sub is contained within s[start,end]. Optional
arguments start and end are interpreted as in slice notation.
Return -1 on failure.
"""
return s.rfind(*args)
# for a bit of speed
_float = float
_int = int
_long = long
# Convert string to float
def atof(s):
"""atof(s) -> float
Return the floating point number represented by the string s.
"""
return _float(s)
# Convert string to integer
def atoi(s , base=10):
"""atoi(s [,base]) -> int
Return the integer represented by the string s in the given
base, which defaults to 10. The string s must consist of one
or more digits, possibly preceded by a sign. If base is 0, it
is chosen from the leading characters of s, 0 for octal, 0x or
0X for hexadecimal. If base is 16, a preceding 0x or 0X is
accepted.
"""
return _int(s, base)
# Convert string to long integer
def atol(s, base=10):
"""atol(s [,base]) -> long
Return the long integer represented by the string s in the
given base, which defaults to 10. The string s must consist
of one or more digits, possibly preceded by a sign. If base
is 0, it is chosen from the leading characters of s, 0 for
octal, 0x or 0X for hexadecimal. If base is 16, a preceding
0x or 0X is accepted. A trailing L or l is not accepted,
unless base is 0.
"""
return _long(s, base)
# Left-justify a string
def ljust(s, width, *args):
"""ljust(s, width[, fillchar]) -> string
Return a left-justified version of s, in a field of the
specified width, padded with spaces as needed. The string is
never truncated. If specified the fillchar is used instead of spaces.
"""
return s.ljust(width, *args)
# Right-justify a string
def rjust(s, width, *args):
"""rjust(s, width[, fillchar]) -> string
Return a right-justified version of s, in a field of the
specified width, padded with spaces as needed. The string is
never truncated. If specified the fillchar is used instead of spaces.
"""
return s.rjust(width, *args)
# Center a string
def center(s, width, *args):
"""center(s, width[, fillchar]) -> string
Return a center version of s, in a field of the specified
width. padded with spaces as needed. The string is never
truncated. If specified the fillchar is used instead of spaces.
"""
return s.center(width, *args)
# Zero-fill a number, e.g., (12, 3) --> '012' and (-3, 3) --> '-03'
# Decadent feature: the argument may be a string or a number
# (Use of this is deprecated; it should be a string as with ljust c.s.)
def zfill(x, width):
"""zfill(x, width) -> string
Pad a numeric string x with zeros on the left, to fill a field
of the specified width. The string x is never truncated.
"""
if not isinstance(x, basestring):
x = repr(x)
return x.zfill(width)
# Expand tabs in a string.
# Doesn't take non-printing chars into account, but does understand \n.
def expandtabs(s, tabsize=8):
"""expandtabs(s [,tabsize]) -> string
Return a copy of the string s with all tab characters replaced
by the appropriate number of spaces, depending on the current
column, and the tabsize (default 8).
"""
return s.expandtabs(tabsize)
# Character translation through look-up table.
def translate(s, table, deletions=""):
"""translate(s,table [,deletions]) -> string
Return a copy of the string s, where all characters occurring
in the optional argument deletions are removed, and the
remaining characters have been mapped through the given
translation table, which must be a string of length 256. The
deletions argument is not allowed for Unicode strings.
"""
if deletions:
return s.translate(table, deletions)
else:
# Add s[:0] so that if s is Unicode and table is an 8-bit string,
# table is converted to Unicode. This means that table *cannot*
# be a dictionary -- for that feature, use u.translate() directly.
return s.translate(table + s[:0])
# Capitalize a string, e.g. "aBc dEf" -> "Abc def".
def capitalize(s):
"""capitalize(s) -> string
Return a copy of the string s with only its first character
capitalized.
"""
return s.capitalize()
# Substring replacement (global)
def replace(s, old, new, maxsplit=-1):
"""replace (str, old, new[, maxsplit]) -> string
Return a copy of string str with all occurrences of substring
old replaced by new. If the optional argument maxsplit is
given, only the first maxsplit occurrences are replaced.
"""
return s.replace(old, new, maxsplit)
# Try importing optional built-in module "strop" -- if it exists,
# it redefines some string operations that are 100-1000 times faster.
# It also defines values for whitespace, lowercase and uppercase
# that match <ctype.h>'s definitions.
try:
from strop import maketrans, lowercase, uppercase, whitespace
letters = lowercase + uppercase
except ImportError:
pass # Use the original versions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,758 @@
"""
A small templating language
This implements a small templating language for use internally in
Paste and Paste Script. This language implements if/elif/else,
for/continue/break, expressions, and blocks of Python code. The
syntax is::
{{any expression (function calls etc)}}
{{any expression | filter}}
{{for x in y}}...{{endfor}}
{{if x}}x{{elif y}}y{{else}}z{{endif}}
{{py:x=1}}
{{py:
def foo(bar):
return 'baz'
}}
{{default var = default_value}}
{{# comment}}
You use this with the ``Template`` class or the ``sub`` shortcut.
The ``Template`` class takes the template string and the name of
the template (for errors) and a default namespace. Then (like
``string.Template``) you can call the ``tmpl.substitute(**kw)``
method to make a substitution (or ``tmpl.substitute(a_dict)``).
``sub(content, **kw)`` substitutes the template immediately. You
can use ``__name='tmpl.html'`` to set the name of the template.
If there are syntax errors ``TemplateError`` will be raised.
"""
import re
import sys
import cgi
import urllib
from paste.util.looper import looper
__all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate',
'sub_html', 'html', 'bunch']
token_re = re.compile(r'\{\{|\}\}')
in_re = re.compile(r'\s+in\s+')
var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I)
class TemplateError(Exception):
"""Exception raised while parsing a template
"""
def __init__(self, message, position, name=None):
self.message = message
self.position = position
self.name = name
def __str__(self):
msg = '%s at line %s column %s' % (
self.message, self.position[0], self.position[1])
if self.name:
msg += ' in %s' % self.name
return msg
class _TemplateContinue(Exception):
pass
class _TemplateBreak(Exception):
pass
class Template(object):
default_namespace = {
'start_braces': '{{',
'end_braces': '}}',
'looper': looper,
}
default_encoding = 'utf8'
def __init__(self, content, name=None, namespace=None):
self.content = content
self._unicode = isinstance(content, unicode)
self.name = name
self._parsed = parse(content, name=name)
if namespace is None:
namespace = {}
self.namespace = namespace
def from_filename(cls, filename, namespace=None, encoding=None):
f = open(filename, 'rb')
c = f.read()
f.close()
if encoding:
c = c.decode(encoding)
return cls(content=c, name=filename, namespace=namespace)
from_filename = classmethod(from_filename)
def __repr__(self):
return '<%s %s name=%r>' % (
self.__class__.__name__,
hex(id(self))[2:], self.name)
def substitute(self, *args, **kw):
if args:
if kw:
raise TypeError(
"You can only give positional *or* keyword arguments")
if len(args) > 1:
raise TypeError(
"You can only give on positional argument")
kw = args[0]
ns = self.default_namespace.copy()
ns.update(self.namespace)
ns.update(kw)
result = self._interpret(ns)
return result
def _interpret(self, ns):
__traceback_hide__ = True
parts = []
self._interpret_codes(self._parsed, ns, out=parts)
return ''.join(parts)
def _interpret_codes(self, codes, ns, out):
__traceback_hide__ = True
for item in codes:
if isinstance(item, basestring):
out.append(item)
else:
self._interpret_code(item, ns, out)
def _interpret_code(self, code, ns, out):
__traceback_hide__ = True
name, pos = code[0], code[1]
if name == 'py':
self._exec(code[2], ns, pos)
elif name == 'continue':
raise _TemplateContinue()
elif name == 'break':
raise _TemplateBreak()
elif name == 'for':
vars, expr, content = code[2], code[3], code[4]
expr = self._eval(expr, ns, pos)
self._interpret_for(vars, expr, content, ns, out)
elif name == 'cond':
parts = code[2:]
self._interpret_if(parts, ns, out)
elif name == 'expr':
parts = code[2].split('|')
base = self._eval(parts[0], ns, pos)
for part in parts[1:]:
func = self._eval(part, ns, pos)
base = func(base)
out.append(self._repr(base, pos))
elif name == 'default':
var, expr = code[2], code[3]
if var not in ns:
result = self._eval(expr, ns, pos)
ns[var] = result
elif name == 'comment':
return
else:
assert 0, "Unknown code: %r" % name
def _interpret_for(self, vars, expr, content, ns, out):
__traceback_hide__ = True
for item in expr:
if len(vars) == 1:
ns[vars[0]] = item
else:
if len(vars) != len(item):
raise ValueError(
'Need %i items to unpack (got %i items)'
% (len(vars), len(item)))
for name, value in zip(vars, item):
ns[name] = value
try:
self._interpret_codes(content, ns, out)
except _TemplateContinue:
continue
except _TemplateBreak:
break
def _interpret_if(self, parts, ns, out):
__traceback_hide__ = True
# @@: if/else/else gets through
for part in parts:
assert not isinstance(part, basestring)
name, pos = part[0], part[1]
if name == 'else':
result = True
else:
result = self._eval(part[2], ns, pos)
if result:
self._interpret_codes(part[3], ns, out)
break
def _eval(self, code, ns, pos):
__traceback_hide__ = True
try:
value = eval(code, ns)
return value
except:
exc_info = sys.exc_info()
e = exc_info[1]
if getattr(e, 'args'):
arg0 = e.args[0]
else:
arg0 = str(e)
e.args = (self._add_line_info(arg0, pos),)
raise exc_info[0], e, exc_info[2]
def _exec(self, code, ns, pos):
__traceback_hide__ = True
try:
exec code in ns
except:
exc_info = sys.exc_info()
e = exc_info[1]
e.args = (self._add_line_info(e.args[0], pos),)
raise exc_info[0], e, exc_info[2]
def _repr(self, value, pos):
__traceback_hide__ = True
try:
if value is None:
return ''
if self._unicode:
try:
value = unicode(value)
except UnicodeDecodeError:
value = str(value)
else:
value = str(value)
except:
exc_info = sys.exc_info()
e = exc_info[1]
e.args = (self._add_line_info(e.args[0], pos),)
raise exc_info[0], e, exc_info[2]
else:
if self._unicode and isinstance(value, str):
if not self.decode_encoding:
raise UnicodeDecodeError(
'Cannot decode str value %r into unicode '
'(no default_encoding provided)' % value)
value = value.decode(self.default_encoding)
elif not self._unicode and isinstance(value, unicode):
if not self.decode_encoding:
raise UnicodeEncodeError(
'Cannot encode unicode value %r into str '
'(no default_encoding provided)' % value)
value = value.encode(self.default_encoding)
return value
def _add_line_info(self, msg, pos):
msg = "%s at line %s column %s" % (
msg, pos[0], pos[1])
if self.name:
msg += " in file %s" % self.name
return msg
def sub(content, **kw):
name = kw.get('__name')
tmpl = Template(content, name=name)
return tmpl.substitute(kw)
return result
def paste_script_template_renderer(content, vars, filename=None):
tmpl = Template(content, name=filename)
return tmpl.substitute(vars)
class bunch(dict):
def __init__(self, **kw):
for name, value in kw.items():
setattr(self, name, value)
def __setattr__(self, name, value):
self[name] = value
def __getattr__(self, name):
try:
return self[name]
except KeyError:
raise AttributeError(name)
def __getitem__(self, key):
if 'default' in self:
try:
return dict.__getitem__(self, key)
except KeyError:
return dict.__getitem__(self, 'default')
else:
return dict.__getitem__(self, key)
def __repr__(self):
items = [
(k, v) for k, v in self.items()]
items.sort()
return '<%s %s>' % (
self.__class__.__name__,
' '.join(['%s=%r' % (k, v) for k, v in items]))
############################################################
## HTML Templating
############################################################
class html(object):
def __init__(self, value):
self.value = value
def __str__(self):
return self.value
def __repr__(self):
return '<%s %r>' % (
self.__class__.__name__, self.value)
def html_quote(value):
if value is None:
return ''
if not isinstance(value, basestring):
if hasattr(value, '__unicode__'):
value = unicode(value)
else:
value = str(value)
value = cgi.escape(value, 1)
if isinstance(value, unicode):
value = value.encode('ascii', 'xmlcharrefreplace')
return value
def url(v):
if not isinstance(v, basestring):
if hasattr(v, '__unicode__'):
v = unicode(v)
else:
v = str(v)
if isinstance(v, unicode):
v = v.encode('utf8')
return urllib.quote(v)
def attr(**kw):
kw = kw.items()
kw.sort()
parts = []
for name, value in kw:
if value is None:
continue
if name.endswith('_'):
name = name[:-1]
parts.append('%s="%s"' % (html_quote(name), html_quote(value)))
return html(' '.join(parts))
class HTMLTemplate(Template):
default_namespace = Template.default_namespace.copy()
default_namespace.update(dict(
html=html,
attr=attr,
url=url,
))
def _repr(self, value, pos):
plain = Template._repr(self, value, pos)
if isinstance(value, html):
return plain
else:
return html_quote(plain)
def sub_html(content, **kw):
name = kw.get('__name')
tmpl = HTMLTemplate(content, name=name)
return tmpl.substitute(kw)
return result
############################################################
## Lexing and Parsing
############################################################
def lex(s, name=None, trim_whitespace=True):
"""
Lex a string into chunks:
>>> lex('hey')
['hey']
>>> lex('hey {{you}}')
['hey ', ('you', (1, 7))]
>>> lex('hey {{')
Traceback (most recent call last):
...
TemplateError: No }} to finish last expression at line 1 column 7
>>> lex('hey }}')
Traceback (most recent call last):
...
TemplateError: }} outside expression at line 1 column 7
>>> lex('hey {{ {{')
Traceback (most recent call last):
...
TemplateError: {{ inside expression at line 1 column 10
"""
in_expr = False
chunks = []
last = 0
last_pos = (1, 1)
for match in token_re.finditer(s):
expr = match.group(0)
pos = find_position(s, match.end())
if expr == '{{' and in_expr:
raise TemplateError('{{ inside expression', position=pos,
name=name)
elif expr == '}}' and not in_expr:
raise TemplateError('}} outside expression', position=pos,
name=name)
if expr == '{{':
part = s[last:match.start()]
if part:
chunks.append(part)
in_expr = True
else:
chunks.append((s[last:match.start()], last_pos))
in_expr = False
last = match.end()
last_pos = pos
if in_expr:
raise TemplateError('No }} to finish last expression',
name=name, position=last_pos)
part = s[last:]
if part:
chunks.append(part)
if trim_whitespace:
chunks = trim_lex(chunks)
return chunks
statement_re = re.compile(r'^(?:if |elif |else |for |py:)')
single_statements = ['endif', 'endfor', 'continue', 'break']
trail_whitespace_re = re.compile(r'\n[\t ]*$')
lead_whitespace_re = re.compile(r'^[\t ]*\n')
def trim_lex(tokens):
r"""
Takes a lexed set of tokens, and removes whitespace when there is
a directive on a line by itself:
>>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False)
>>> tokens
[('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny']
>>> trim_lex(tokens)
[('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y']
"""
for i in range(len(tokens)):
current = tokens[i]
if isinstance(tokens[i], basestring):
# we don't trim this
continue
item = current[0]
if not statement_re.search(item) and item not in single_statements:
continue
if not i:
prev = ''
else:
prev = tokens[i-1]
if i+1 >= len(tokens):
next = ''
else:
next = tokens[i+1]
if (not isinstance(next, basestring)
or not isinstance(prev, basestring)):
continue
if ((not prev or trail_whitespace_re.search(prev))
and (not next or lead_whitespace_re.search(next))):
if prev:
m = trail_whitespace_re.search(prev)
# +1 to leave the leading \n on:
prev = prev[:m.start()+1]
tokens[i-1] = prev
if next:
m = lead_whitespace_re.search(next)
next = next[m.end():]
tokens[i+1] = next
return tokens
def find_position(string, index):
"""Given a string and index, return (line, column)"""
leading = string[:index].splitlines()
return (len(leading), len(leading[-1])+1)
def parse(s, name=None):
r"""
Parses a string into a kind of AST
>>> parse('{{x}}')
[('expr', (1, 3), 'x')]
>>> parse('foo')
['foo']
>>> parse('{{if x}}test{{endif}}')
[('cond', (1, 3), ('if', (1, 3), 'x', ['test']))]
>>> parse('series->{{for x in y}}x={{x}}{{endfor}}')
['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])]
>>> parse('{{for x, y in z:}}{{continue}}{{endfor}}')
[('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])]
>>> parse('{{py:x=1}}')
[('py', (1, 3), 'x=1')]
>>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}')
[('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))]
Some exceptions::
>>> parse('{{continue}}')
Traceback (most recent call last):
...
TemplateError: continue outside of for loop at line 1 column 3
>>> parse('{{if x}}foo')
Traceback (most recent call last):
...
TemplateError: No {{endif}} at line 1 column 3
>>> parse('{{else}}')
Traceback (most recent call last):
...
TemplateError: else outside of an if block at line 1 column 3
>>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}')
Traceback (most recent call last):
...
TemplateError: Unexpected endif at line 1 column 25
>>> parse('{{if}}{{endif}}')
Traceback (most recent call last):
...
TemplateError: if with no expression at line 1 column 3
>>> parse('{{for x y}}{{endfor}}')
Traceback (most recent call last):
...
TemplateError: Bad for (no "in") in 'x y' at line 1 column 3
>>> parse('{{py:x=1\ny=2}}')
Traceback (most recent call last):
...
TemplateError: Multi-line py blocks must start with a newline at line 1 column 3
"""
tokens = lex(s, name=name)
result = []
while tokens:
next, tokens = parse_expr(tokens, name)
result.append(next)
return result
def parse_expr(tokens, name, context=()):
if isinstance(tokens[0], basestring):
return tokens[0], tokens[1:]
expr, pos = tokens[0]
expr = expr.strip()
if expr.startswith('py:'):
expr = expr[3:].lstrip(' \t')
if expr.startswith('\n'):
expr = expr[1:]
else:
if '\n' in expr:
raise TemplateError(
'Multi-line py blocks must start with a newline',
position=pos, name=name)
return ('py', pos, expr), tokens[1:]
elif expr in ('continue', 'break'):
if 'for' not in context:
raise TemplateError(
'continue outside of for loop',
position=pos, name=name)
return (expr, pos), tokens[1:]
elif expr.startswith('if '):
return parse_cond(tokens, name, context)
elif (expr.startswith('elif ')
or expr == 'else'):
raise TemplateError(
'%s outside of an if block' % expr.split()[0],
position=pos, name=name)
elif expr in ('if', 'elif', 'for'):
raise TemplateError(
'%s with no expression' % expr,
position=pos, name=name)
elif expr in ('endif', 'endfor'):
raise TemplateError(
'Unexpected %s' % expr,
position=pos, name=name)
elif expr.startswith('for '):
return parse_for(tokens, name, context)
elif expr.startswith('default '):
return parse_default(tokens, name, context)
elif expr.startswith('#'):
return ('comment', pos, tokens[0][0]), tokens[1:]
return ('expr', pos, tokens[0][0]), tokens[1:]
def parse_cond(tokens, name, context):
start = tokens[0][1]
pieces = []
context = context + ('if',)
while 1:
if not tokens:
raise TemplateError(
'Missing {{endif}}',
position=start, name=name)
if (isinstance(tokens[0], tuple)
and tokens[0][0] == 'endif'):
return ('cond', start) + tuple(pieces), tokens[1:]
next, tokens = parse_one_cond(tokens, name, context)
pieces.append(next)
def parse_one_cond(tokens, name, context):
(first, pos), tokens = tokens[0], tokens[1:]
content = []
if first.endswith(':'):
first = first[:-1]
if first.startswith('if '):
part = ('if', pos, first[3:].lstrip(), content)
elif first.startswith('elif '):
part = ('elif', pos, first[5:].lstrip(), content)
elif first == 'else':
part = ('else', pos, None, content)
else:
assert 0, "Unexpected token %r at %s" % (first, pos)
while 1:
if not tokens:
raise TemplateError(
'No {{endif}}',
position=pos, name=name)
if (isinstance(tokens[0], tuple)
and (tokens[0][0] == 'endif'
or tokens[0][0].startswith('elif ')
or tokens[0][0] == 'else')):
return part, tokens
next, tokens = parse_expr(tokens, name, context)
content.append(next)
def parse_for(tokens, name, context):
first, pos = tokens[0]
tokens = tokens[1:]
context = ('for',) + context
content = []
assert first.startswith('for ')
if first.endswith(':'):
first = first[:-1]
first = first[3:].strip()
match = in_re.search(first)
if not match:
raise TemplateError(
'Bad for (no "in") in %r' % first,
position=pos, name=name)
vars = first[:match.start()]
if '(' in vars:
raise TemplateError(
'You cannot have () in the variable section of a for loop (%r)'
% vars, position=pos, name=name)
vars = tuple([
v.strip() for v in first[:match.start()].split(',')
if v.strip()])
expr = first[match.end():]
while 1:
if not tokens:
raise TemplateError(
'No {{endfor}}',
position=pos, name=name)
if (isinstance(tokens[0], tuple)
and tokens[0][0] == 'endfor'):
return ('for', pos, vars, expr, content), tokens[1:]
next, tokens = parse_expr(tokens, name, context)
content.append(next)
def parse_default(tokens, name, context):
first, pos = tokens[0]
assert first.startswith('default ')
first = first.split(None, 1)[1]
parts = first.split('=', 1)
if len(parts) == 1:
raise TemplateError(
"Expression must be {{default var=value}}; no = found in %r" % first,
position=pos, name=name)
var = parts[0].strip()
if ',' in var:
raise TemplateError(
"{{default x, y = ...}} is not supported",
position=pos, name=name)
if not var_re.search(var):
raise TemplateError(
"Not a valid variable name for {{default}}: %r"
% var, position=pos, name=name)
expr = parts[1].strip()
return ('default', pos, var, expr), tokens[1:]
_fill_command_usage = """\
%prog [OPTIONS] TEMPLATE arg=value
Use py:arg=value to set a Python value; otherwise all values are
strings.
"""
def fill_command(args=None):
import sys, optparse, pkg_resources, os
if args is None:
args = sys.argv[1:]
dist = pkg_resources.get_distribution('Paste')
parser = optparse.OptionParser(
version=str(dist),
usage=_fill_command_usage)
parser.add_option(
'-o', '--output',
dest='output',
metavar="FILENAME",
help="File to write output to (default stdout)")
parser.add_option(
'--html',
dest='use_html',
action='store_true',
help="Use HTML style filling (including automatic HTML quoting)")
parser.add_option(
'--env',
dest='use_env',
action='store_true',
help="Put the environment in as top-level variables")
options, args = parser.parse_args(args)
if len(args) < 1:
print 'You must give a template filename'
print dir(parser)
assert 0
template_name = args[0]
args = args[1:]
vars = {}
if options.use_env:
vars.update(os.environ)
for value in args:
if '=' not in value:
print 'Bad argument: %r' % value
sys.exit(2)
name, value = value.split('=', 1)
if name.startswith('py:'):
name = name[:3]
value = eval(value)
vars[name] = value
if template_name == '-':
template_content = sys.stdin.read()
template_name = '<stdin>'
else:
f = open(template_name, 'rb')
template_content = f.read()
f.close()
if options.use_html:
TemplateClass = HTMLTemplate
else:
TemplateClass = Template
template = TemplateClass(template_content, name=template_name)
result = template.substitute(vars)
if options.output:
f = open(options.output, 'wb')
f.write(result)
f.close()
else:
sys.stdout.write(result)
if __name__ == '__main__':
from paste.util.template import fill_command
fill_command()

View File

@@ -0,0 +1,250 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
"""
threadedprint.py
================
:author: Ian Bicking
:date: 12 Jul 2004
Multi-threaded printing; allows the output produced via print to be
separated according to the thread.
To use this, you must install the catcher, like::
threadedprint.install()
The installation optionally takes one of three parameters:
default
The default destination for print statements (e.g., ``sys.stdout``).
factory
A function that will produce the stream for a thread, given the
thread's name.
paramwriter
Instead of writing to a file-like stream, this function will be
called like ``paramwriter(thread_name, text)`` for every write.
The thread name is the value returned by
``threading.currentThread().getName()``, a string (typically something
like Thread-N).
You can also submit file-like objects for specific threads, which will
override any of these parameters. To do this, call ``register(stream,
[threadName])``. ``threadName`` is optional, and if not provided the
stream will be registered for the current thread.
If no specific stream is registered for a thread, and no default has
been provided, then an error will occur when anything is written to
``sys.stdout`` (or printed).
Note: the stream's ``write`` method will be called in the thread the
text came from, so you should consider thread safety, especially if
multiple threads share the same writer.
Note: if you want access to the original standard out, use
``sys.__stdout__``.
You may also uninstall this, via::
threadedprint.uninstall()
TODO
----
* Something with ``sys.stderr``.
* Some default handlers. Maybe something that hooks into `logging`.
* Possibly cache the results of ``factory`` calls. This would be a
semantic change.
"""
import threading
import sys
from paste.util import filemixin
class PrintCatcher(filemixin.FileMixin):
def __init__(self, default=None, factory=None, paramwriter=None,
leave_stdout=False):
assert len(filter(lambda x: x is not None,
[default, factory, paramwriter])) <= 1, (
"You can only provide one of default, factory, or paramwriter")
if leave_stdout:
assert not default, (
"You cannot pass in both default (%r) and "
"leave_stdout=True" % default)
default = sys.stdout
if default:
self._defaultfunc = self._writedefault
elif factory:
self._defaultfunc = self._writefactory
elif paramwriter:
self._defaultfunc = self._writeparam
else:
self._defaultfunc = self._writeerror
self._default = default
self._factory = factory
self._paramwriter = paramwriter
self._catchers = {}
def write(self, v, currentThread=threading.currentThread):
name = currentThread().getName()
catchers = self._catchers
if not catchers.has_key(name):
self._defaultfunc(name, v)
else:
catcher = catchers[name]
catcher.write(v)
def seek(self, *args):
# Weird, but Google App Engine is seeking on stdout
name = threading.currentThread().getName()
catchers = self._catchers
if not name in catchers:
self._default.seek(*args)
else:
catchers[name].seek(*args)
def read(self, *args):
name = threading.currentThread().getName()
catchers = self._catchers
if not name in catchers:
self._default.read(*args)
else:
catchers[name].read(*args)
def _writedefault(self, name, v):
self._default.write(v)
def _writefactory(self, name, v):
self._factory(name).write(v)
def _writeparam(self, name, v):
self._paramwriter(name, v)
def _writeerror(self, name, v):
assert False, (
"There is no PrintCatcher output stream for the thread %r"
% name)
def register(self, catcher, name=None,
currentThread=threading.currentThread):
if name is None:
name = currentThread().getName()
self._catchers[name] = catcher
def deregister(self, name=None,
currentThread=threading.currentThread):
if name is None:
name = currentThread().getName()
assert self._catchers.has_key(name), (
"There is no PrintCatcher catcher for the thread %r" % name)
del self._catchers[name]
_printcatcher = None
_oldstdout = None
def install(**kw):
global _printcatcher, _oldstdout, register, deregister
if (not _printcatcher or sys.stdout is not _printcatcher):
_oldstdout = sys.stdout
_printcatcher = sys.stdout = PrintCatcher(**kw)
register = _printcatcher.register
deregister = _printcatcher.deregister
def uninstall():
global _printcatcher, _oldstdout, register, deregister
if _printcatcher:
sys.stdout = _oldstdout
_printcatcher = _oldstdout = None
register = not_installed_error
deregister = not_installed_error
def not_installed_error(*args, **kw):
assert False, (
"threadedprint has not yet been installed (call "
"threadedprint.install())")
register = deregister = not_installed_error
class StdinCatcher(filemixin.FileMixin):
def __init__(self, default=None, factory=None, paramwriter=None):
assert len(filter(lambda x: x is not None,
[default, factory, paramwriter])) <= 1, (
"You can only provide one of default, factory, or paramwriter")
if default:
self._defaultfunc = self._readdefault
elif factory:
self._defaultfunc = self._readfactory
elif paramwriter:
self._defaultfunc = self._readparam
else:
self._defaultfunc = self._readerror
self._default = default
self._factory = factory
self._paramwriter = paramwriter
self._catchers = {}
def read(self, size=None, currentThread=threading.currentThread):
name = currentThread().getName()
catchers = self._catchers
if not catchers.has_key(name):
return self._defaultfunc(name, size)
else:
catcher = catchers[name]
return catcher.read(size)
def _readdefault(self, name, size):
self._default.read(size)
def _readfactory(self, name, size):
self._factory(name).read(size)
def _readparam(self, name, size):
self._paramreader(name, size)
def _readerror(self, name, size):
assert False, (
"There is no StdinCatcher output stream for the thread %r"
% name)
def register(self, catcher, name=None,
currentThread=threading.currentThread):
if name is None:
name = currentThread().getName()
self._catchers[name] = catcher
def deregister(self, catcher, name=None,
currentThread=threading.currentThread):
if name is None:
name = currentThread().getName()
assert self._catchers.has_key(name), (
"There is no StdinCatcher catcher for the thread %r" % name)
del self._catchers[name]
_stdincatcher = None
_oldstdin = None
def install_stdin(**kw):
global _stdincatcher, _oldstdin, register_stdin, deregister_stdin
if not _stdincatcher:
_oldstdin = sys.stdin
_stdincatcher = sys.stdin = StdinCatcher(**kw)
register_stdin = _stdincatcher.register
deregister_stdin = _stdincatcher.deregister
def uninstall():
global _stdincatcher, _oldstin, register_stdin, deregister_stdin
if _stdincatcher:
sys.stdin = _oldstdin
_stdincatcher = _oldstdin = None
register_stdin = deregister_stdin = not_installed_error_stdin
def not_installed_error_stdin(*args, **kw):
assert False, (
"threadedprint has not yet been installed for stdin (call "
"threadedprint.install_stdin())")

View File

@@ -0,0 +1,43 @@
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
"""
Implementation of thread-local storage, for Python versions that don't
have thread local storage natively.
"""
try:
import threading
except ImportError:
# No threads, so "thread local" means process-global
class local(object):
pass
else:
try:
local = threading.local
except AttributeError:
# Added in 2.4, but now we'll have to define it ourselves
import thread
class local(object):
def __init__(self):
self.__dict__['__objs'] = {}
def __getattr__(self, attr, g=thread.get_ident):
try:
return self.__dict__['__objs'][g()][attr]
except KeyError:
raise AttributeError(
"No variable %s defined for the thread %s"
% (attr, g()))
def __setattr__(self, attr, value, g=thread.get_ident):
self.__dict__['__objs'].setdefault(g(), {})[attr] = value
def __delattr__(self, attr, g=thread.get_ident):
try:
del self.__dict__['__objs'][g()][attr]
except KeyError:
raise AttributeError(
"No variable %s defined for thread %s"
% (attr, g()))