Some Title

# HG changeset patch # User cmlenz # Date 1272056906 0 # Node ID 09cc3627654c1bd2a567b153d780cef499a8f67a # Parent de82830f8816d57ec397ce43b13e138fe8c635e9 Sync `experimental/inline` branch with [source:trunk@1126]. diff --git a/COPYING b/COPYING --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2006-2008 Edgewall Software +Copyright (C) 2006-2010 Edgewall Software All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,15 +1,18 @@ Version 0.6 http://svn.edgewall.org/repos/genshi/tags/0.6.0/ -(???, from branches/stable/0.6.x) +(Apr 22 2010, from branches/stable/0.6.x) * Support for Python 2.3 has been dropped. - * Added caching in the serilization stage for improved performance. - - -Version 0.5.2 -http://svn.edgewall.org/repos/genshi/tags/0.5.2/ -(???, from branches/stable/0.5.x) - + * Rewrite of the XPath evaluation engine for better performance and improved + correctness. This is the result of integrating work done by Marcin Kurczych + during GSoC 2008. + * Updated the Python AST processing for template code evaluation to use the + `_ast` module instead of the deprecated `compiler` package, including an + adapter layer for Python 2.4. This, too, is the result of integrating work + done by Marcin Kurczych during GSoC 2008. + * Added caching in the serialization stage for improved performance in some + cases. + * Various improvements to the HTML sanitization filter. * Fix problem with I18n filter that would get confused by expressions in attribute values when inside an `i18n:msg` block (ticket #250). * Fix problem with the transformation filter dropping events after the @@ -20,6 +23,15 @@ * Import statements inside function definitions in template code blocks no longer result in an UndefinedError when the imported name is accessed (ticket #276). + * Fixed handling of relative URLs with fragment identifiers containing colons + in the `HTMLSanitizer` (ticket #274). + * Added an option to the `HTMLFiller` to also populate password fields. + * Match template processing no longer produces unwanted duplicate output in + some cases (ticket #254). + * Templates instantiated without a loader now get an implicit loader based on + their file path, or the current directory as a fallback (ticket #320). + * Added documentation for the `TemplateLoader`. + * Enhanced documentation for internationalization. Version 0.5.1 diff --git a/doc/filters.txt b/doc/filters.txt --- a/doc/filters.txt +++ b/doc/filters.txt @@ -47,7 +47,7 @@ ...

... """) >>> filler = HTMLFormFiller(data=dict(username='john', remember=True)) - >>> print template.generate() | filler + >>> print(template.generate() | filler)

User name: @@ -104,7 +104,7 @@ ... ... """) >>> sanitize = HTMLSanitizer() - >>> print html | sanitize + >>> print(html | sanitize)

Innocent looking text.

@@ -130,7 +130,7 @@ ...
... """) >>> sanitize = HTMLSanitizer(safe_attrs=HTMLSanitizer.SAFE_ATTRS | set(['style'])) - >>> print html | sanitize + >>> print(html | sanitize)

@@ -164,10 +164,10 @@ ... ... ''') - >>> print html | Transformer('body/em').map(unicode.upper, TEXT) \ - ... .unwrap().wrap(tag.u).end() \ - ... .select('body/u') \ - ... .prepend('underlined ') + >>> print(html | Transformer('body/em').map(unicode.upper, TEXT) + ... .unwrap().wrap(tag.u).end() + ... .select('body/u') + ... .prepend('underlined ')) Some Title @@ -216,7 +216,7 @@ >>> xform = Transformer('body//em').map(unicode.upper, TEXT) \ >>> xform = xform.apply(RenameTransformation('u')) - >>> print html | xform + >>> print(html | xform) Some Title diff --git a/doc/i18n.txt b/doc/i18n.txt --- a/doc/i18n.txt +++ b/doc/i18n.txt @@ -4,15 +4,15 @@ Internationalization and Localization ===================================== -Genshi provides basic supporting infrastructure for internationalizing -and localizing templates. That includes functionality for extracting localizable -strings from templates, as well as a template filter that can apply translations -to templates as they get rendered. +Genshi provides comprehensive supporting infrastructure for internationalizing +and localizing templates. That includes functionality for extracting +localizable strings from templates, as well as a template filter and special +directives that can apply translations to templates as they get rendered. This support is based on `gettext`_ message catalogs and the `gettext Python -module`_. The extraction process can be used from the API level, or through the -front-ends implemented by the `Babel`_ project, for which Genshi provides a -plugin. +module`_. The extraction process can be used from the API level, or through +the front-ends implemented by the `Babel`_ project, for which Genshi provides +a plugin. .. _`gettext`: http://www.gnu.org/software/gettext/ .. _`gettext python module`: http://docs.python.org/lib/module-gettext.html @@ -28,18 +28,19 @@ ====== The simplest way to internationalize and translate templates would be to wrap -all localizable strings in a ``gettext()`` function call (which is often aliased -to ``_()`` for brevity). In that case, no extra template filter is required. +all localizable strings in a ``gettext()`` function call (which is often +aliased to ``_()`` for brevity). In that case, no extra template filter is +required. .. code-block:: genshi

${_("Hello, world!")}

-However, this approach results in significant “character noise” in templates, +However, this approach results in significant “character noise” in templates, making them harder to read and preview. The ``genshi.filters.Translator`` filter allows you to get rid of the -explicit `gettext`_ function calls, so you can continue to just write: +explicit `gettext`_ function calls, so you can (often) just continue to write: .. code-block:: genshi @@ -48,23 +49,28 @@ This text will still be extracted and translated as if you had wrapped it in a ``_()`` call. -.. note:: For parameterized or pluralizable messages, you need to continue using - the appropriate ``gettext`` functions. +.. note:: For parameterized or pluralizable messages, you need to use the + special `template directives`_ described below, or use the + corresponding ``gettext`` function in embedded Python expressions. -You can control which tags should be ignored by this process; for example, it +You can control which tags should be ignored by this process; for example, it doesn't really make sense to translate the content of the HTML ```` element. Both ``') - >>> print tmpl.generate() + >>> print(tmpl.generate()) On the other hand, Genshi will always replace two dollar signs in text with a @@ -237,7 +240,7 @@ .. code-block:: pycon >>> tmpl = MarkupTemplate('') - >>> print tmpl.generate() + >>> print(tmpl.generate()) @@ -338,7 +341,7 @@ >>> from genshi.template import MarkupTemplate >>> tmpl = MarkupTemplate('

${defined("doh")}

') - >>> print tmpl.generate().render('xhtml') + >>> print(tmpl.generate().render('xhtml'))

False

.. note:: Lenient error handling was the default in Genshi prior to version 0.5. @@ -364,7 +367,7 @@ >>> from genshi.template import MarkupTemplate >>> tmpl = MarkupTemplate('

${doh}

', lookup='lenient') - >>> print tmpl.generate().render('xhtml') + >>> print(tmpl.generate().render('xhtml'))

You *will* however get an exception if you try to call an undefined variable, or @@ -374,7 +377,7 @@ >>> from genshi.template import MarkupTemplate >>> tmpl = MarkupTemplate('

${doh.oops}

', lookup='lenient') - >>> print tmpl.generate().render('xhtml') + >>> print(tmpl.generate().render('xhtml')) Traceback (most recent call last): ... UndefinedError: "doh" not defined @@ -387,7 +390,7 @@ >>> from genshi.template import MarkupTemplate >>> tmpl = MarkupTemplate('

${type(doh) is not Undefined}

', ... lookup='lenient') - >>> print tmpl.generate().render('xhtml') + >>> print(tmpl.generate().render('xhtml'))

False

Alternatively, the built-in functions defined_ or value_of_ can be used in this diff --git a/doc/upgrade.txt b/doc/upgrade.txt --- a/doc/upgrade.txt +++ b/doc/upgrade.txt @@ -18,6 +18,11 @@ Support for Python 2.3 has been dropped in this release. Python 2.4 is now the minimum version of Python required to run Genshi. +The XPath engine has been completely overhauled for this version. Some +patterns that previously matched incorrectly will no longer match, and +the other way around. In such cases, the XPath expressions need to be +fixed in your application and templates. + ------------------------------------ Upgrading from Genshi 0.4.x to 0.5.x diff --git a/doc/xpath.txt b/doc/xpath.txt --- a/doc/xpath.txt +++ b/doc/xpath.txt @@ -86,8 +86,8 @@ ... ... ''') - >>> print doc.select('items/item[@status="closed" and ' - ... '(@resolution="invalid" or not(@resolution))]/summary/text()') + >>> print(doc.select('items/item[@status="closed" and ' + ... '(@resolution="invalid" or not(@resolution))]/summary/text()')) BarBaz diff --git a/genshi/__init__.py b/genshi/__init__.py --- a/genshi/__init__.py +++ b/genshi/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -20,14 +20,7 @@ """ __docformat__ = 'restructuredtext en' -try: - from pkg_resources import get_distribution, ResolutionError - try: - __version__ = get_distribution('Genshi').version - except ResolutionError: - __version__ = None # unknown -except ImportError: - __version__ = None # unknown +__version__ = '0.7' from genshi.core import * from genshi.input import ParseError, XML, HTML diff --git a/genshi/builder.py b/genshi/builder.py --- a/genshi/builder.py +++ b/genshi/builder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -32,7 +32,7 @@ >>> doc(tag.br) ->>> print doc +>>> print(doc)

Some text and a link.

If an attribute name collides with a Python keyword, simply append an underscore @@ -40,7 +40,7 @@ >>> doc(class_='intro') ->>> print doc +>>> print(doc)

Some text and a link.

As shown above, an `Element` can easily be directly rendered to XML text by @@ -51,7 +51,7 @@ >>> stream = doc.generate() >>> stream #doctest: +ELLIPSIS ->>> print stream +>>> print(stream)

Some text and a link.

@@ -64,7 +64,7 @@ >>> fragment = tag('Hello, ', tag.em('world'), '!') >>> fragment ->>> print fragment +>>> print(fragment) Hello, world! """ @@ -93,14 +93,15 @@ :see: `append` """ - map(self.append, args) + for arg in args: + self.append(arg) return self def __iter__(self): return self._generate() def __repr__(self): - return '<%s>' % self.__class__.__name__ + return '<%s>' % type(self).__name__ def __str__(self): return str(self.generate()) @@ -125,7 +126,8 @@ self.children.extend(node.children) elif node is not None: try: - map(self.append, iter(node)) + for child in node: + self.append(child) except TypeError: self.children.append(node) @@ -166,18 +168,18 @@ Construct XML elements by passing the tag name to the constructor: - >>> print Element('strong') + >>> print(Element('strong')) Attributes can be specified using keyword arguments. The values of the arguments will be converted to strings and any special XML characters escaped: - >>> print Element('textarea', rows=10, cols=60) + >>> print(Element('textarea', rows=10, cols=60)) - >>> print Element('span', title='1 < 2') + >>> print(Element('span', title='1 < 2')) <span title="1 < 2"/> - >>> print Element('span', title='"baz"') + >>> print(Element('span', title='"baz"')) <span title=""baz""/> The " character is escaped using a numerical entity. @@ -186,50 +188,50 @@ If an attribute value evaluates to `None`, that attribute is not included in the output: - >>> print Element('a', name=None) + >>> print(Element('a', name=None)) <a/> Attribute names that conflict with Python keywords can be specified by appending an underscore: - >>> print Element('div', class_='warning') + >>> print(Element('div', class_='warning')) <div class="warning"/> Nested elements can be added to an element using item access notation. The call notation can also be used for this and for adding attributes using keyword arguments, as one would do in the constructor. - >>> print Element('ul')(Element('li'), Element('li')) + >>> print(Element('ul')(Element('li'), Element('li'))) <ul><li/><li/></ul> - >>> print Element('a')('Label') + >>> print(Element('a')('Label')) <a>Label</a> - >>> print Element('a')('Label', href="target") + >>> print(Element('a')('Label', href="target")) <a href="target">Label</a> Text nodes can be nested in an element by adding strings instead of elements. Any special characters in the strings are escaped automatically: - >>> print Element('em')('Hello world') + >>> print(Element('em')('Hello world')) <em>Hello world</em> - >>> print Element('em')(42) + >>> print(Element('em')(42)) <em>42</em> - >>> print Element('em')('1 < 2') + >>> print(Element('em')('1 < 2')) <em>1 < 2</em> This technique also allows mixed content: - >>> print Element('p')('Hello ', Element('b')('world')) + >>> print(Element('p')('Hello ', Element('b')('world'))) <p>Hello <b>world</b></p> Quotes are not escaped inside text nodes: - >>> print Element('p')('"Hello"') + >>> print(Element('p')('"Hello"')) <p>"Hello"</p> Elements can also be combined with other elements or strings using the addition operator, which results in a `Fragment` object that contains the operands: - >>> print Element('br') + 'some text' + Element('br') + >>> print(Element('br') + 'some text' + Element('br')) <br/>some text<br/> Elements with a namespace can be generated using the `Namespace` and/or @@ -237,7 +239,7 @@ >>> from genshi.core import Namespace >>> xhtml = Namespace('http://www.w3.org/1999/xhtml') - >>> print Element(xhtml.html, lang='en') + >>> print(Element(xhtml.html, lang='en')) <html xmlns="http://www.w3.org/1999/xhtml" lang="en"/> """ __slots__ = ['tag', 'attrib'] @@ -260,7 +262,7 @@ return self def __repr__(self): - return '<%s "%s">' % (self.__class__.__name__, self.tag) + return '<%s "%s">' % (type(self).__name__, self.tag) def _generate(self): yield START, (self.tag, self.attrib), (None, -1, -1) @@ -283,28 +285,28 @@ attribute of the factory object: >>> factory = ElementFactory() - >>> print factory.foo + >>> print(factory.foo) <foo/> - >>> print factory.foo(id=2) + >>> print(factory.foo(id=2)) <foo id="2"/> Markup fragments (lists of nodes without a parent element) can be created by calling the factory: - >>> print factory('Hello, ', factory.em('world'), '!') + >>> print(factory('Hello, ', factory.em('world'), '!')) Hello, <em>world</em>! A factory can also be bound to a specific namespace: >>> factory = ElementFactory('http://www.w3.org/1999/xhtml') - >>> print factory.html(lang="en") + >>> print(factory.html(lang="en")) <html xmlns="http://www.w3.org/1999/xhtml" lang="en"/> The namespace for a specific element can be altered on an existing factory by specifying the new namespace using item access: >>> factory = ElementFactory() - >>> print factory.html(factory['http://www.w3.org/2000/svg'].g(id=3)) + >>> print(factory.html(factory['http://www.w3.org/2000/svg'].g(id=3))) <html><g xmlns="http://www.w3.org/2000/svg" id="3"/></html> Usually, the `ElementFactory` class is not be used directly. Rather, the diff --git a/genshi/core.py b/genshi/core.py --- a/genshi/core.py +++ b/genshi/core.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -14,13 +14,13 @@ """Core classes for markup processing.""" try: + reduce # builtin in Python < 3 +except NameError: from functools import reduce -except ImportError: - pass # builtin in Python <= 2.5 from itertools import chain import operator -from genshi.util import plaintext, stripentities, striptags +from genshi.util import plaintext, stripentities, striptags, stringrepr __all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Attrs', 'Namespace', 'QName'] @@ -93,7 +93,7 @@ >>> from genshi.input import HTML >>> html = HTML('''<p onclick="alert('Whoa')">Hello, world!</p>''') - >>> print html + >>> print(html) <p onclick="alert('Whoa')">Hello, world!</p> A filter such as the HTML sanitizer can be applied to that stream using @@ -101,7 +101,7 @@ >>> from genshi.filters import HTMLSanitizer >>> sanitizer = HTMLSanitizer() - >>> print html | sanitizer + >>> print(html | sanitizer) <p>Hello, world!</p> Filters can be any function that accepts and produces a stream (where @@ -112,14 +112,14 @@ ... if kind is TEXT: ... data = data.upper() ... yield kind, data, pos - >>> print html | sanitizer | uppercase + >>> print(html | sanitizer | uppercase) <p>HELLO, WORLD!</p> Serializers can also be used with this notation: >>> from genshi.output import TextSerializer >>> output = TextSerializer() - >>> print html | sanitizer | uppercase | output + >>> print(html | sanitizer | uppercase | output) HELLO, WORLD! Commonly, serializers should be used at the end of the "pipeline"; @@ -188,9 +188,9 @@ >>> from genshi import HTML >>> stream = HTML('<doc><elem>foo</elem><elem>bar</elem></doc>') - >>> print stream.select('elem') + >>> print(stream.select('elem')) <elem>foo</elem><elem>bar</elem> - >>> print stream.select('elem/text()') + >>> print(stream.select('elem/text()')) foobar Note that the outermost element of the stream becomes the *context @@ -198,13 +198,13 @@ not match anything in the example above, because it only tests against child elements of the outermost element: - >>> print stream.select('doc') + >>> print(stream.select('doc')) <BLANKLINE> You can use the "." expression to match the context node itself (although that usually makes little sense): - >>> print stream.select('.') + >>> print(stream.select('.')) <doc><elem>foo</elem><elem>bar</elem></doc> :param path: a string containing the XPath expression @@ -265,6 +265,7 @@ PI = Stream.PI COMMENT = Stream.COMMENT + def _ensure(stream): """Ensure that every item on the stream is actually a markup event.""" stream = iter(stream) @@ -353,6 +354,20 @@ if attr == name: return True + def __getitem__(self, i): + """Return an item or slice of the attributes list. + + >>> attrs = Attrs([('href', '#'), ('title', 'Foo')]) + >>> attrs[1] + ('title', 'Foo') + >>> attrs[1:] + Attrs([('title', 'Foo')]) + """ + items = tuple.__getitem__(self, i) + if type(i) is slice: + return Attrs(items) + return items + def __getslice__(self, i, j): """Return a slice of the attributes list. @@ -412,12 +427,12 @@ attributes joined together. >>> Attrs([('href', '#'), ('title', 'Foo')]).totuple() - ('TEXT', u'#Foo', (None, -1, -1)) + ('TEXT', '#Foo', (None, -1, -1)) :return: a `TEXT` event :rtype: `tuple` """ - return TEXT, u''.join([x[1] for x in self]), (None, -1, -1) + return TEXT, ''.join([x[1] for x in self]), (None, -1, -1) class Markup(unicode): @@ -427,10 +442,10 @@ __slots__ = [] def __add__(self, other): - return Markup(unicode(self) + unicode(escape(other))) + return Markup(unicode.__add__(self, escape(other))) def __radd__(self, other): - return Markup(unicode(escape(other)) + unicode(self)) + return Markup(unicode.__add__(escape(other), self)) def __mod__(self, args): if isinstance(args, dict): @@ -442,13 +457,11 @@ return Markup(unicode.__mod__(self, args)) def __mul__(self, num): - return Markup(unicode(self) * num) - - def __rmul__(self, num): - return Markup(num * unicode(self)) + return Markup(unicode.__mul__(self, num)) + __rmul__ = __mul__ def __repr__(self): - return '<%s %r>' % (self.__class__.__name__, unicode(self)) + return "<%s %s>" % (type(self).__name__, unicode.__repr__(self)) def join(self, seq, escape_quotes=True): """Return a `Markup` object which is the concatenation of the strings @@ -465,7 +478,7 @@ :rtype: `Markup` :see: `escape` """ - return Markup(unicode(self).join([escape(item, quotes=escape_quotes) + return Markup(unicode.join(self, [escape(item, quotes=escape_quotes) for item in seq])) @classmethod @@ -496,9 +509,9 @@ if hasattr(text, '__html__'): return Markup(text.__html__()) - text = unicode(text).replace('&', '&') \ - .replace('<', '<') \ - .replace('>', '>') + text = text.replace('&', '&') \ + .replace('<', '<') \ + .replace('>', '>') if quotes: text = text.replace('"', '"') return cls(text) @@ -514,7 +527,7 @@ :see: `genshi.core.unescape` """ if not self: - return u'' + return '' return unicode(self).replace('"', '"') \ .replace('>', '>') \ .replace('<', '<') \ @@ -549,8 +562,10 @@ except ImportError: pass # just use the Python implementation + escape = Markup.escape + def unescape(text): """Reverse-escapes &, <, >, and \" and returns a `unicode` object. @@ -583,7 +598,7 @@ >>> html = Namespace('http://www.w3.org/1999/xhtml') >>> html - <Namespace "http://www.w3.org/1999/xhtml"> + Namespace('http://www.w3.org/1999/xhtml') >>> html.uri u'http://www.w3.org/1999/xhtml' @@ -591,7 +606,7 @@ that namespace: >>> html.body - QName(u'http://www.w3.org/1999/xhtml}body') + QName('http://www.w3.org/1999/xhtml}body') >>> html.body.localname u'body' >>> html.body.namespace @@ -601,7 +616,7 @@ attribute names that are not valid Python identifiers: >>> html['body'] - QName(u'http://www.w3.org/1999/xhtml}body') + QName('http://www.w3.org/1999/xhtml}body') A `Namespace` object can also be used to test whether a specific `QName` belongs to that namespace using the ``in`` operator: @@ -641,14 +656,14 @@ return self.uri == other def __getitem__(self, name): - return QName(self.uri + u'}' + name) + return QName(self.uri + '}' + name) __getattr__ = __getitem__ def __hash__(self): return hash(self.uri) def __repr__(self): - return '<Namespace "%s">' % self.uri + return '%s(%s)' % (type(self).__name__, stringrepr(self.uri)) def __str__(self): return self.uri.encode('utf-8') @@ -671,14 +686,14 @@ >>> qname = QName('foo') >>> qname - QName(u'foo') + QName('foo') >>> qname.localname u'foo' >>> qname.namespace >>> qname = QName('http://www.w3.org/1999/xhtml}body') >>> qname - QName(u'http://www.w3.org/1999/xhtml}body') + QName('http://www.w3.org/1999/xhtml}body') >>> qname.localname u'body' >>> qname.namespace @@ -696,9 +711,9 @@ if type(qname) is cls: return qname - parts = qname.lstrip(u'{').split(u'}', 1) + parts = qname.lstrip('{').split('}', 1) if len(parts) > 1: - self = unicode.__new__(cls, u'{%s' % qname) + self = unicode.__new__(cls, '{%s' % qname) self.namespace, self.localname = map(unicode, parts) else: self = unicode.__new__(cls, qname) @@ -709,4 +724,4 @@ return (self.lstrip('{'),) def __repr__(self): - return 'QName(%s)' % unicode.__repr__(self.lstrip('{')) + return '%s(%s)' % (type(self).__name__, stringrepr(self.lstrip('{'))) diff --git a/genshi/filters/__init__.py b/genshi/filters/__init__.py --- a/genshi/filters/__init__.py +++ b/genshi/filters/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which diff --git a/genshi/filters/html.py b/genshi/filters/html.py --- a/genshi/filters/html.py +++ b/genshi/filters/html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -13,6 +13,10 @@ """Implementation of a number of stream filters.""" +try: + any +except NameError: + from genshi.util import any import re from genshi.core import Attrs, QName, stripentities @@ -30,7 +34,7 @@ ... <p><input type="text" name="foo" /></p> ... </form>''') >>> filler = HTMLFormFiller(data={'foo': 'bar'}) - >>> print html | filler + >>> print(html | filler) <form> <p><input type="text" name="foo" value="bar"/></p> </form> @@ -39,7 +43,7 @@ # (if not in a multiple-select) # TODO: only apply to elements in the XHTML namespace (or no namespace)? - def __init__(self, name=None, id=None, data=None): + def __init__(self, name=None, id=None, data=None, passwords=False): """Create the filter. :param name: The name of the form that should be populated. If this @@ -51,12 +55,17 @@ :param data: The dictionary of form values, where the keys are the names of the form fields, and the values are the values to fill in. + :param passwords: Whether password input fields should be populated. + This is off by default for security reasons (for + example, a password may end up in the browser cache) + :note: Changed in 0.5.2: added the `passwords` option """ self.name = name self.id = id if data is None: data = {} self.data = data + self.passwords = passwords def __call__(self, stream): """Apply the filter to the given stream. @@ -83,7 +92,7 @@ elif in_form: if tagname == 'input': - type = attrs.get('type') + type = attrs.get('type', '').lower() if type in ('checkbox', 'radio'): name = attrs.get('name') if name and name in self.data: @@ -95,7 +104,7 @@ checked = declval in [unicode(v) for v in value] else: - checked = bool(filter(None, value)) + checked = any(value) else: if declval: checked = declval == unicode(value) @@ -105,14 +114,17 @@ attrs |= [(QName('checked'), 'checked')] elif 'checked' in attrs: attrs -= 'checked' - elif type in (None, 'hidden', 'text'): + elif type in ('', 'hidden', 'text') \ + or type == 'password' and self.passwords: name = attrs.get('name') if name and name in self.data: value = self.data[name] if isinstance(value, (list, tuple)): value = value[0] if value is not None: - attrs |= [(QName('value'), unicode(value))] + attrs |= [ + (QName('value'), unicode(value)) + ] elif tagname == 'select': name = attrs.get('name') if name in self.data: @@ -187,7 +199,7 @@ >>> from genshi import HTML >>> html = HTML('<div><script>alert(document.cookie)</script></div>') - >>> print html | HTMLSanitizer() + >>> print(html | HTMLSanitizer()) <div/> The default set of safe tags and attributes can be modified when the filter @@ -196,14 +208,14 @@ >>> html = HTML('<div style="background: #000"></div>') >>> sanitizer = HTMLSanitizer(safe_attrs=HTMLSanitizer.SAFE_ATTRS | set(['style'])) - >>> print html | sanitizer + >>> print(html | sanitizer) <div style="background: #000"/> Note that even in this case, the filter *does* attempt to remove dangerous constructs from style attributes: >>> html = HTML('<div style="background: url(javascript:void); color: #000"></div>') - >>> print html | sanitizer + >>> print(html | sanitizer) <div style="color: #000"/> This handles HTML entities, unicode escapes in CSS and Javascript text, as @@ -211,6 +223,10 @@ default because it is very hard for such sanitizing to be completely safe, especially considering how much error recovery current web browsers perform. + It also does some basic filtering of CSS properties that may be used for + typical phishing attacks. For more sophisticated filtering, this class + provides a couple of hooks that can be overridden in sub-classes. + :warn: Note that this special processing of CSS is currently only applied to style attributes, **not** style elements. """ @@ -274,7 +290,7 @@ if waiting_for: continue tag, attrs = data - if tag not in self.safe_tags: + if not self.is_safe_elem(tag, attrs): waiting_for = tag continue @@ -309,6 +325,43 @@ if not waiting_for: yield kind, data, pos + def is_safe_css(self, propname, value): + """Determine whether the given css property declaration is to be + considered safe for inclusion in the output. + + :param propname: the CSS property name + :param value: the value of the property + :return: whether the property value should be considered safe + :rtype: bool + :since: version 0.6 + """ + if propname == 'position': + return False + if propname.startswith('margin') and '-' in value: + # Negative margins can be used for phishing + return False + return True + + def is_safe_elem(self, tag, attrs): + """Determine whether the given element should be considered safe for + inclusion in the output. + + :param tag: the tag name of the element + :type tag: QName + :param attrs: the element attributes + :type attrs: Attrs + :return: whether the element should be considered safe + :rtype: bool + :since: version 0.6 + """ + if tag not in self.safe_tags: + return False + if tag.localname == 'input': + input_type = attrs.get('type', '').lower() + if input_type == 'password': + return False + return True + def is_safe_uri(self, uri): """Determine whether the given URI is to be considered safe for inclusion in the output. @@ -327,6 +380,8 @@ :rtype: `bool` :since: version 0.4.3 """ + if '#' in uri: + uri = uri.split('#', 1)[0] # Strip out the fragment identifier if ':' not in uri: return True # This is a relative URI chars = [char for char in uri.split(':', 1)[0] if char.isalnum()] @@ -363,14 +418,20 @@ """ decls = [] text = self._strip_css_comments(self._replace_unicode_escapes(text)) - for decl in filter(None, text.split(';')): + for decl in text.split(';'): decl = decl.strip() if not decl: continue + try: + propname, value = decl.split(':', 1) + except ValueError: + continue + if not self.is_safe_css(propname.strip().lower(), value.strip()): + continue is_evil = False - if 'expression' in decl: + if 'expression' in value: is_evil = True - for match in re.finditer(r'url\s*\(([^)]+)', decl): + for match in re.finditer(r'url\s*\(([^)]+)', value): if not self.is_safe_uri(match.group(1)): is_evil = True break diff --git a/genshi/filters/i18n.py b/genshi/filters/i18n.py --- a/genshi/filters/i18n.py +++ b/genshi/filters/i18n.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -11,71 +11,544 @@ # individuals. For the exact contribution history, see the revision # history and logs, available at http://genshi.edgewall.org/log/. -"""Utilities for internationalization and localization of templates. +"""Directives and utilities for internationalization and localization of +templates. :since: version 0.4 +:note: Directives support added since version 0.6 """ +try: + any +except NameError: + from genshi.util import any from gettext import NullTranslations +import os import re from types import FunctionType -from genshi.core import Attrs, Namespace, QName, START, END, TEXT, START_NS, \ - END_NS, XML_NAMESPACE, _ensure +from genshi.core import Attrs, Namespace, QName, START, END, TEXT, \ + XML_NAMESPACE, _ensure, StreamEventKind from genshi.template.eval import _ast from genshi.template.base import DirectiveFactory, EXPR, SUB, _apply_directives -from genshi.template.directives import Directive +from genshi.template.directives import Directive, StripDirective from genshi.template.markup import MarkupTemplate, EXEC __all__ = ['Translator', 'extract'] __docformat__ = 'restructuredtext en' + I18N_NAMESPACE = Namespace('http://genshi.edgewall.org/i18n') +MSGBUF = StreamEventKind('MSGBUF') +SUB_START = StreamEventKind('SUB_START') +SUB_END = StreamEventKind('SUB_END') + +GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext', 'dgettext', 'dngettext', + 'ugettext', 'ungettext') + -class CommentDirective(Directive): +class I18NDirective(Directive): + """Simple interface for i18n directives to support messages extraction.""" - __slots__ = [] + def __call__(self, stream, directives, ctxt, **vars): + return _apply_directives(stream, directives, ctxt, vars) + + +class ExtractableI18NDirective(I18NDirective): + """Simple interface for directives to support messages extraction.""" + + def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS, + search_text=True, comment_stack=None): + raise NotImplementedError + + +class CommentDirective(I18NDirective): + """Implementation of the ``i18n:comment`` template directive which adds + translation comments. + + >>> tmpl = MarkupTemplate('''<html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <p i18n:comment="As in Foo Bar">Foo</p> + ... </html>''') + >>> translator = Translator() + >>> translator.setup(tmpl) + >>> list(translator.extract(tmpl.stream)) + [(2, None, u'Foo', [u'As in Foo Bar'])] + """ + __slots__ = ['comment'] + + def __init__(self, value, template=None, namespaces=None, lineno=-1, + offset=-1): + Directive.__init__(self, None, template, namespaces, lineno, offset) + self.comment = value + + +class MsgDirective(ExtractableI18NDirective): + r"""Implementation of the ``i18n:msg`` directive which marks inner content + as translatable. Consider the following examples: + + >>> tmpl = MarkupTemplate('''<html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <div i18n:msg=""> + ... <p>Foo</p> + ... <p>Bar</p> + ... </div> + ... <p i18n:msg="">Foo <em>bar</em>!</p> + ... </html>''') + + >>> translator = Translator() + >>> translator.setup(tmpl) + >>> list(translator.extract(tmpl.stream)) + [(2, None, u'[1:Foo]\n [2:Bar]', []), (6, None, u'Foo [1:bar]!', [])] + >>> print(tmpl.generate().render()) + <html> + <div><p>Foo</p> + <p>Bar</p></div> + <p>Foo <em>bar</em>!</p> + </html> + + >>> tmpl = MarkupTemplate('''<html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <div i18n:msg="fname, lname"> + ... <p>First Name: ${fname}</p> + ... <p>Last Name: ${lname}</p> + ... </div> + ... <p i18n:msg="">Foo <em>bar</em>!</p> + ... </html>''') + >>> translator.setup(tmpl) + >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE + [(2, None, u'[1:First Name: %(fname)s]\n [2:Last Name: %(lname)s]', []), + (6, None, u'Foo [1:bar]!', [])] + + >>> tmpl = MarkupTemplate('''<html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <div i18n:msg="fname, lname"> + ... <p>First Name: ${fname}</p> + ... <p>Last Name: ${lname}</p> + ... </div> + ... <p i18n:msg="">Foo <em>bar</em>!</p> + ... </html>''') + >>> translator.setup(tmpl) + >>> print(tmpl.generate(fname='John', lname='Doe').render()) + <html> + <div><p>First Name: John</p> + <p>Last Name: Doe</p></div> + <p>Foo <em>bar</em>!</p> + </html> + + Starting and ending white-space is stripped of to make it simpler for + translators. Stripping it is not that important since it's on the html + source, the rendered output will remain the same. + """ + __slots__ = ['params', 'lineno'] + + def __init__(self, value, template=None, namespaces=None, lineno=-1, + offset=-1): + Directive.__init__(self, None, template, namespaces, lineno, offset) + self.params = [param.strip() for param in value.split(',') if param] + self.lineno = lineno @classmethod def attach(cls, template, stream, value, namespaces, pos): - return None, stream + if type(value) is dict: + value = value.get('params', '').strip() + return super(MsgDirective, cls).attach(template, stream, value.strip(), + namespaces, pos) + + def __call__(self, stream, directives, ctxt, **vars): + gettext = ctxt.get('_i18n.gettext') + if ctxt.get('_i18n.domain'): + dgettext = ctxt.get('_i18n.dgettext') + assert hasattr(dgettext, '__call__'), \ + 'No domain gettext function passed' + gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg) + + def _generate(): + msgbuf = MessageBuffer(self) + previous = stream.next() + if previous[0] is START: + yield previous + else: + msgbuf.append(*previous) + previous = stream.next() + for kind, data, pos in stream: + msgbuf.append(*previous) + previous = kind, data, pos + if previous[0] is not END: + msgbuf.append(*previous) + previous = None + for event in msgbuf.translate(gettext(msgbuf.format())): + yield event + if previous: + yield previous + + return _apply_directives(_generate(), directives, ctxt, vars) + + def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS, + search_text=True, comment_stack=None): + msgbuf = MessageBuffer(self) + strip = False + + stream = iter(stream) + previous = stream.next() + if previous[0] is START: + for message in translator._extract_attrs(previous, + gettext_functions, + search_text=search_text): + yield message + previous = stream.next() + strip = True + for event in stream: + if event[0] is START: + for message in translator._extract_attrs(event, + gettext_functions, + search_text=search_text): + yield message + msgbuf.append(*previous) + previous = event + if not strip: + msgbuf.append(*previous) + + yield self.lineno, None, msgbuf.format(), comment_stack[-1:] -class MsgDirective(Directive): - +class ChooseBranchDirective(I18NDirective): __slots__ = ['params'] - def __init__(self, value, template, hints=None, namespaces=None, - lineno=-1, offset=-1): - Directive.__init__(self, None, template, namespaces, lineno, offset) - self.params = [name.strip() for name in value.split(',')] + def __call__(self, stream, directives, ctxt, **vars): + self.params = ctxt.get('_i18n.choose.params', [])[:] + msgbuf = MessageBuffer(self) + stream = _apply_directives(stream, directives, ctxt, vars) - def __call__(self, stream, directives, ctxt, **vars): - msgbuf = MessageBuffer(self.params) + previous = stream.next() + if previous[0] is START: + yield previous + else: + msgbuf.append(*previous) - stream = iter(stream) - yield stream.next() # the outer start tag - previous = stream.next() + try: + previous = stream.next() + except StopIteration: + # For example <i18n:singular> or <i18n:plural> directives + yield MSGBUF, (), -1 # the place holder for msgbuf output + ctxt['_i18n.choose.%s' % self.tagname] = msgbuf + return + for event in stream: msgbuf.append(*previous) previous = event + yield MSGBUF, (), -1 # the place holder for msgbuf output - gettext = ctxt.get('_i18n.gettext') - for event in msgbuf.translate(gettext(msgbuf.format())): + if previous[0] is END: + yield previous # the outer end tag + else: + msgbuf.append(*previous) + ctxt['_i18n.choose.%s' % self.tagname] = msgbuf + + def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS, + search_text=True, comment_stack=None, msgbuf=None): + stream = iter(stream) + previous = stream.next() + + if previous[0] is START: + # skip the enclosing element + for message in translator._extract_attrs(previous, + gettext_functions, + search_text=search_text): + yield message + previous = stream.next() + + for event in stream: + if previous[0] is START: + for message in translator._extract_attrs(previous, + gettext_functions, + search_text=search_text): + yield message + msgbuf.append(*previous) + previous = event + + if previous[0] is not END: + msgbuf.append(*previous) + + +class SingularDirective(ChooseBranchDirective): + """Implementation of the ``i18n:singular`` directive to be used with the + ``i18n:choose`` directive.""" + + +class PluralDirective(ChooseBranchDirective): + """Implementation of the ``i18n:plural`` directive to be used with the + ``i18n:choose`` directive.""" + + +class ChooseDirective(ExtractableI18NDirective): + """Implementation of the ``i18n:choose`` directive which provides plural + internationalisation of strings. + + This directive requires at least one parameter, the one which evaluates to + an integer which will allow to choose the plural/singular form. If you also + have expressions inside the singular and plural version of the string you + also need to pass a name for those parameters. Consider the following + examples: + + >>> tmpl = MarkupTemplate('''\ + <html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <div i18n:choose="num; num"> + ... <p i18n:singular="">There is $num coin</p> + ... <p i18n:plural="">There are $num coins</p> + ... </div> + ... </html>''') + >>> translator = Translator() + >>> translator.setup(tmpl) + >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE + [(2, 'ngettext', (u'There is %(num)s coin', + u'There are %(num)s coins'), [])] + + >>> tmpl = MarkupTemplate('''\ + <html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <div i18n:choose="num; num"> + ... <p i18n:singular="">There is $num coin</p> + ... <p i18n:plural="">There are $num coins</p> + ... </div> + ... </html>''') + >>> translator.setup(tmpl) + >>> print(tmpl.generate(num=1).render()) + <html> + <div> + <p>There is 1 coin</p> + </div> + </html> + >>> print(tmpl.generate(num=2).render()) + <html> + <div> + <p>There are 2 coins</p> + </div> + </html> + + When used as a element and not as an attribute: + + >>> tmpl = MarkupTemplate('''\ + <html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <i18n:choose numeral="num" params="num"> + ... <p i18n:singular="">There is $num coin</p> + ... <p i18n:plural="">There are $num coins</p> + ... </i18n:choose> + ... </html>''') + >>> translator.setup(tmpl) + >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE + [(2, 'ngettext', (u'There is %(num)s coin', + u'There are %(num)s coins'), [])] + """ + __slots__ = ['numeral', 'params', 'lineno'] + + def __init__(self, value, template=None, namespaces=None, lineno=-1, + offset=-1): + Directive.__init__(self, None, template, namespaces, lineno, offset) + params = [v.strip() for v in value.split(';')] + self.numeral = self._parse_expr(params.pop(0), template, lineno, offset) + self.params = params and [name.strip() for name in + params[0].split(',') if name] or [] + self.lineno = lineno + + @classmethod + def attach(cls, template, stream, value, namespaces, pos): + if type(value) is dict: + numeral = value.get('numeral', '').strip() + assert numeral is not '', "at least pass the numeral param" + params = [v.strip() for v in value.get('params', '').split(',')] + value = '%s; ' % numeral + ', '.join(params) + return super(ChooseDirective, cls).attach(template, stream, value, + namespaces, pos) + + def __call__(self, stream, directives, ctxt, **vars): + ctxt.push({'_i18n.choose.params': self.params, + '_i18n.choose.singular': None, + '_i18n.choose.plural': None}) + + ngettext = ctxt.get('_i18n.ngettext') + assert hasattr(ngettext, '__call__'), 'No ngettext function available' + dngettext = ctxt.get('_i18n.dngettext') + if not dngettext: + dngettext = lambda d, s, p, n: ngettext(s, p, n) + + new_stream = [] + singular_stream = None + singular_msgbuf = None + plural_stream = None + plural_msgbuf = None + + numeral = self.numeral.evaluate(ctxt) + is_plural = self._is_plural(numeral, ngettext) + + for event in stream: + if event[0] is SUB and any(isinstance(d, ChooseBranchDirective) + for d in event[1][0]): + subdirectives, substream = event[1] + + if isinstance(subdirectives[0], SingularDirective): + singular_stream = list(_apply_directives(substream, + subdirectives, + ctxt, vars)) + new_stream.append((MSGBUF, None, (None, -1, -1))) + + elif isinstance(subdirectives[0], PluralDirective): + if is_plural: + plural_stream = list(_apply_directives(substream, + subdirectives, + ctxt, vars)) + + else: + new_stream.append(event) + + if ctxt.get('_i18n.domain'): + ngettext = lambda s, p, n: dngettext(ctxt.get('_i18n.domain'), + s, p, n) + + singular_msgbuf = ctxt.get('_i18n.choose.singular') + if is_plural: + plural_msgbuf = ctxt.get('_i18n.choose.plural') + msgbuf, choice = plural_msgbuf, plural_stream + else: + msgbuf, choice = singular_msgbuf, singular_stream + plural_msgbuf = MessageBuffer(self) + + for kind, data, pos in new_stream: + if kind is MSGBUF: + for event in choice: + if event[0] is MSGBUF: + translation = ngettext(singular_msgbuf.format(), + plural_msgbuf.format(), + numeral) + for subevent in msgbuf.translate(translation): + yield subevent + else: + yield event + else: + yield kind, data, pos + + ctxt.pop() + + def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS, + search_text=True, comment_stack=None): + strip = False + stream = iter(stream) + previous = stream.next() + + if previous[0] is START: + # skip the enclosing element + for message in translator._extract_attrs(previous, + gettext_functions, + search_text=search_text): + yield message + previous = stream.next() + strip = True + + singular_msgbuf = MessageBuffer(self) + plural_msgbuf = MessageBuffer(self) + + for event in stream: + if previous[0] is SUB: + directives, substream = previous[1] + for directive in directives: + if isinstance(directive, SingularDirective): + for message in directive.extract(translator, + substream, gettext_functions, search_text, + comment_stack, msgbuf=singular_msgbuf): + yield message + elif isinstance(directive, PluralDirective): + for message in directive.extract(translator, + substream, gettext_functions, search_text, + comment_stack, msgbuf=plural_msgbuf): + yield message + elif not isinstance(directive, StripDirective): + singular_msgbuf.append(*previous) + plural_msgbuf.append(*previous) + else: + if previous[0] is START: + for message in translator._extract_attrs(previous, + gettext_functions, + search_text): + yield message + singular_msgbuf.append(*previous) + plural_msgbuf.append(*previous) + previous = event + + if not strip: + singular_msgbuf.append(*previous) + plural_msgbuf.append(*previous) + + yield self.lineno, 'ngettext', \ + (singular_msgbuf.format(), plural_msgbuf.format()), \ + comment_stack[-1:] + + def _is_plural(self, numeral, ngettext): + # XXX: should we test which form was chosen like this!?!?!? + # There should be no match in any catalogue for these singular and + # plural test strings + singular = u'O\x85\xbe\xa9\xa8az\xc3?\xe6\xa1\x02n\x84\x93' + plural = u'\xcc\xfb+\xd3Pn\x9d\tT\xec\x1d\xda\x1a\x88\x00' + return ngettext(singular, plural, numeral) == plural + + +class DomainDirective(I18NDirective): + """Implementation of the ``i18n:domain`` directive which allows choosing + another i18n domain(catalog) to translate from. + + >>> from genshi.filters.tests.i18n import DummyTranslations + >>> tmpl = MarkupTemplate('''\ + <html xmlns:i18n="http://genshi.edgewall.org/i18n"> + ... <p i18n:msg="">Bar</p> + ... <div i18n:domain="foo"> + ... <p i18n:msg="">FooBar</p> + ... <p>Bar</p> + ... <p i18n:domain="bar" i18n:msg="">Bar</p> + ... <p i18n:domain="">Bar</p> + ... </div> + ... <p>Bar</p> + ... </html>''') + + >>> translations = DummyTranslations({'Bar': 'Voh'}) + >>> translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'foo_Bar'}) + >>> translations.add_domain('bar', {'Bar': 'bar_Bar'}) + >>> translator = Translator(translations) + >>> translator.setup(tmpl) + + >>> print(tmpl.generate().render()) + <html> + <p>Voh</p> + <div> + <p>BarFoo</p> + <p>foo_Bar</p> + <p>bar_Bar</p> + <p>Voh</p> + </div> + <p>Voh</p> + </html> + """ + __slots__ = ['domain'] + + def __init__(self, value, template=None, namespaces=None, lineno=-1, + offset=-1): + Directive.__init__(self, None, template, namespaces, lineno, offset) + self.domain = value and value.strip() or '__DEFAULT__' + + @classmethod + def attach(cls, template, stream, value, namespaces, pos): + if type(value) is dict: + value = value.get('name') + return super(DomainDirective, cls).attach(template, stream, value, + namespaces, pos) + + def __call__(self, stream, directives, ctxt, **vars): + ctxt.push({'_i18n.domain': self.domain}) + for event in _apply_directives(stream, directives, ctxt, vars): yield event - - yield previous # the outer end tag + ctxt.pop() class Translator(DirectiveFactory): """Can extract and translate localizable strings from markup streams and templates. - For example, assume the followng template: + For example, assume the following template: - >>> from genshi.template import MarkupTemplate - >>> >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> @@ -94,7 +567,6 @@ ... 'Example': 'Beispiel', ... 'Hello, %(name)s': 'Hallo, %(name)s' ... }[string] - >>> >>> translator = Translator(pseudo_gettext) Next, the translator needs to be prepended to any already defined filters @@ -105,7 +577,7 @@ When generating the template output, our hard-coded translations should be applied as expected: - >>> print tmpl.generate(username='Hans', _=pseudo_gettext) + >>> print(tmpl.generate(username='Hans', _=pseudo_gettext)) <html> <head> <title>Beispiel</title> @@ -115,23 +587,28 @@ <p>Hallo, Hans</p> </body> </html> - + Note that elements defining ``xml:lang`` attributes that do not contain variable expressions are ignored by this filter. That can be used to exclude specific parts of a template from being extracted and translated. """ directives = [ + ('domain', DomainDirective), ('comment', CommentDirective), - ('msg', MsgDirective) + ('msg', MsgDirective), + ('choose', ChooseDirective), + ('singular', SingularDirective), + ('plural', PluralDirective) ] IGNORE_TAGS = frozenset([ QName('script'), QName('http://www.w3.org/1999/xhtml}script'), QName('style'), QName('http://www.w3.org/1999/xhtml}style') ]) - INCLUDE_ATTRS = frozenset(['abbr', 'alt', 'label', 'prompt', 'standby', - 'summary', 'title']) + INCLUDE_ATTRS = frozenset([ + 'abbr', 'alt', 'label', 'prompt', 'standby', 'summary', 'title' + ]) NAMESPACE = I18N_NAMESPACE def __init__(self, translate=NullTranslations(), ignore_tags=IGNORE_TAGS, @@ -145,7 +622,7 @@ :param extract_text: whether the content of text nodes should be extracted, or only text in explicit ``gettext`` function calls - + :note: Changed in 0.6: the `translate` parameter can now be either a ``gettext``-style function, or an object compatible with the ``NullTransalations`` or ``GNUTranslations`` interface @@ -155,7 +632,8 @@ self.include_attrs = include_attrs self.extract_text = extract_text - def __call__(self, stream, ctxt=None, search_text=True): + def __call__(self, stream, ctxt=None, translate_text=True, + translate_attrs=True): """Translate any localizable strings in the given stream. This function shouldn't be called directly. Instead, an instance of @@ -166,25 +644,41 @@ :param stream: the markup event stream :param ctxt: the template context (not used) - :param search_text: whether text nodes should be translated (used - internally) + :param translate_text: whether text nodes should be translated (used + internally) + :param translate_attrs: whether attribute values should be translated + (used internally) :return: the localized stream """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs skip = 0 xml_lang = XML_NAMESPACE['lang'] + if not self.extract_text: + translate_text = False + translate_attrs = False if type(self.translate) is FunctionType: gettext = self.translate + if ctxt: + ctxt['_i18n.gettext'] = gettext else: gettext = self.translate.ugettext - if ctxt: - ctxt['_i18n.gettext'] = gettext + ngettext = self.translate.ungettext + try: + dgettext = self.translate.dugettext + dngettext = self.translate.dungettext + except AttributeError: + dgettext = lambda _, y: gettext(y) + dngettext = lambda _, s, p, n: ngettext(s, p, n) + if ctxt: + ctxt['_i18n.gettext'] = gettext + ctxt['_i18n.ngettext'] = ngettext + ctxt['_i18n.dgettext'] = dgettext + ctxt['_i18n.dngettext'] = dngettext - extract_text = self.extract_text - if not extract_text: - search_text = False + if ctxt and ctxt.get('_i18n.domain'): + gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg) for kind, data, pos in stream: @@ -208,14 +702,15 @@ new_attrs = [] changed = False + for name, value in attrs: newval = value - if extract_text and isinstance(value, basestring): - if name in include_attrs: + if isinstance(value, basestring): + if translate_attrs and name in include_attrs: newval = gettext(value) else: - newval = list(self(_ensure(value), ctxt, - search_text=False) + newval = list( + self(_ensure(value), ctxt, translate_text=False) ) if newval != value: value = newval @@ -226,7 +721,7 @@ yield kind, (tag, attrs), pos - elif search_text and kind is TEXT: + elif translate_text and kind is TEXT: text = data.strip() if text: data = data.replace(text, unicode(gettext(text))) @@ -234,22 +729,36 @@ elif kind is SUB: directives, substream = data - # If this is an i18n:msg directive, no need to translate text + current_domain = None + for idx, directive in enumerate(directives): + # Organize directives to make everything work + # FIXME: There's got to be a better way to do this! + if isinstance(directive, DomainDirective): + # Grab current domain and update context + current_domain = directive.domain + ctxt.push({'_i18n.domain': current_domain}) + # Put domain directive as the first one in order to + # update context before any other directives evaluation + directives.insert(0, directives.pop(idx)) + + # If this is an i18n directive, no need to translate text # nodes here - is_msg = filter(None, [isinstance(d, MsgDirective) - for d in directives]) + is_i18n_directive = any([ + isinstance(d, ExtractableI18NDirective) + for d in directives + ]) substream = list(self(substream, ctxt, - search_text=not is_msg)) + translate_text=not is_i18n_directive, + translate_attrs=translate_attrs)) yield kind, (directives, substream), pos + if current_domain: + ctxt.pop() else: yield kind, data, pos - GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext', 'dgettext', 'dngettext', - 'ugettext', 'ungettext') - def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, - search_text=True, msgbuf=None): + search_text=True, comment_stack=None): """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, @@ -264,8 +773,6 @@ * ``comments`` is a list of comments related to the message, extracted from ``i18n:comment`` attributes found in the markup - >>> from genshi.template import MarkupTemplate - >>> >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> @@ -276,9 +783,8 @@ ... <p>${ngettext("You have %d item", "You have %d items", num)}</p> ... </body> ... </html>''', filename='example.html') - >>> >>> for line, func, msg, comments in Translator().extract(tmpl.stream): - ... print "%d, %r, %r" % (line, func, msg) + ... print('%d, %r, %r' % (line, func, msg)) 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' @@ -295,18 +801,18 @@ :note: Changed in 0.4.1: For a function with multiple string arguments (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. - :note: Changed in 0.6: The returned tuples now include a 4th element, - which is a list of comments for the translator + :note: Changed in 0.6: The returned tuples now include a fourth + element, which is a list of comments for the translator. """ if not self.extract_text: search_text = False + if comment_stack is None: + comment_stack = [] skip = 0 - i18n_comment = I18N_NAMESPACE['comment'] - i18n_msg = I18N_NAMESPACE['msg'] + xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: - if skip: if kind is START: skip += 1 @@ -315,64 +821,103 @@ if kind is START and not skip: tag, attrs = data - if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 continue - for name, value in attrs: - if search_text and isinstance(value, basestring): - if name in self.include_attrs: - text = value.strip() - if text: - yield pos[1], None, text, [] - else: - for lineno, funcname, text, comments in self.extract( - _ensure(value), gettext_functions, - search_text=False): - yield lineno, funcname, text, comments - - if msgbuf: - msgbuf.append(kind, data, pos) - else: - msg_params = attrs.get(i18n_msg) - if msg_params is not None: - if type(msg_params) is list: # event tuple - msg_params = msg_params[0][1] - msgbuf = MessageBuffer( - msg_params, attrs.get(i18n_comment), pos[1] - ) + for message in self._extract_attrs((kind, data, pos), + gettext_functions, + search_text=search_text): + yield message elif not skip and search_text and kind is TEXT: - if not msgbuf: - text = data.strip() - if text and filter(None, [ch.isalpha() for ch in text]): - yield pos[1], None, text, [] - else: - msgbuf.append(kind, data, pos) - - elif not skip and msgbuf and kind is END: - msgbuf.append(kind, data, pos) - if not msgbuf.depth: - yield msgbuf.lineno, None, msgbuf.format(), \ - filter(None, [msgbuf.comment]) - msgbuf = None + text = data.strip() + if text and [ch for ch in text if ch.isalpha()]: + yield pos[1], None, text, comment_stack[-1:] elif kind is EXPR or kind is EXEC: - if msgbuf: - msgbuf.append(kind, data, pos) for funcname, strings in extract_from_code(data, gettext_functions): + # XXX: Do we need to grab i18n:comment from comment_stack ??? yield pos[1], funcname, strings, [] elif kind is SUB: - subkind, substream = data - messages = self.extract(substream, gettext_functions, - search_text=search_text and not skip, - msgbuf=msgbuf) - for lineno, funcname, text, comments in messages: - yield lineno, funcname, text, comments + directives, substream = data + in_comment = False + + for idx, directive in enumerate(directives): + # Do a first loop to see if there's a comment directive + # If there is update context and pop it from directives + if isinstance(directive, CommentDirective): + in_comment = True + comment_stack.append(directive.comment) + if len(directives) == 1: + # in case we're in the presence of something like: + # <p i18n:comment="foo">Foo</p> + for message in self.extract( + substream, gettext_functions, + search_text=search_text and not skip, + comment_stack=comment_stack): + yield message + directives.pop(idx) + elif not isinstance(directive, I18NDirective): + # Remove all other non i18n directives from the process + directives.pop(idx) + + if not directives and not in_comment: + # Extract content if there's no directives because + # strip was pop'ed and not because comment was pop'ed. + # Extraction in this case has been taken care of. + for message in self.extract( + substream, gettext_functions, + search_text=search_text and not skip): + yield message + + for directive in directives: + if isinstance(directive, ExtractableI18NDirective): + for message in directive.extract(self, + substream, gettext_functions, + search_text=search_text and not skip, + comment_stack=comment_stack): + yield message + else: + for message in self.extract( + substream, gettext_functions, + search_text=search_text and not skip, + comment_stack=comment_stack): + yield message + + if in_comment: + comment_stack.pop() + + def get_directive_index(self, dir_cls): + total = len(self._dir_order) + if dir_cls in self._dir_order: + return self._dir_order.index(dir_cls) - total + return total + + def setup(self, template): + """Convenience function to register the `Translator` filter and the + related directives with the given template. + + :param template: a `Template` instance + """ + template.filters.insert(0, self) + if hasattr(template, 'add_directives'): + template.add_directives(Translator.NAMESPACE, self) + + def _extract_attrs(self, event, gettext_functions, search_text): + for name, value in event[1][1]: + if search_text and isinstance(value, basestring): + if name in self.include_attrs: + text = value.strip() + if text: + yield event[2][1], None, text, [] + else: + for message in self.extract(_ensure(value), gettext_functions, + search_text=False): + yield message class MessageBuffer(object): @@ -381,25 +926,23 @@ :since: version 0.5 """ - def __init__(self, params=u'', comment=None, lineno=-1): + def __init__(self, directive=None): """Initialize the message buffer. - :param params: comma-separated list of parameter names - :type params: `basestring` - :param lineno: the line number on which the first stream event - belonging to the message was found + :param directive: the directive owning the buffer + :type directive: I18NDirective """ - if isinstance(params, basestring): - params = [name.strip() for name in params.split(',')] - self.params = params - self.comment = comment - self.lineno = lineno + # params list needs to be copied so that directives can be evaluated + # more than once + self.orig_params = self.params = directive.params[:] + self.directive = directive self.string = [] self.events = {} self.values = {} self.depth = 1 self.order = 1 self.stack = [0] + self.subdirectives = {} def append(self, kind, data, pos): """Append a stream event to the buffer. @@ -408,33 +951,62 @@ :param data: the event data :param pos: the position of the event in the source """ - if kind is TEXT: + if kind is SUB: + # The order needs to be +1 because a new START kind event will + # happen and we we need to wrap those events into our custom kind(s) + order = self.stack[-1] + 1 + subdirectives, substream = data + # Store the directives that should be applied after translation + self.subdirectives.setdefault(order, []).extend(subdirectives) + self.events.setdefault(order, []).append((SUB_START, None, pos)) + for skind, sdata, spos in substream: + self.append(skind, sdata, spos) + self.events.setdefault(order, []).append((SUB_END, None, pos)) + elif kind is TEXT: + if '[' in data or ']' in data: + # Quote [ and ] if it ain't us adding it, ie, if the user is + # using those chars in his templates, escape them + data = data.replace('[', '\[').replace(']', '\]') self.string.append(data) - self.events.setdefault(self.stack[-1], []).append(None) + self.events.setdefault(self.stack[-1], []).append((kind, data, pos)) elif kind is EXPR: - param = self.params.pop(0) + if self.params: + param = self.params.pop(0) + else: + params = ', '.join(['"%s"' % p for p in self.orig_params if p]) + if params: + params = "(%s)" % params + raise IndexError("%d parameters%s given to 'i18n:%s' but " + "%d or more expressions used in '%s', line %s" + % (len(self.orig_params), params, + self.directive.tagname, + len(self.orig_params) + 1, + os.path.basename(pos[0] or + 'In-memory Template'), + pos[1])) self.string.append('%%(%s)s' % param) - self.events.setdefault(self.stack[-1], []).append(None) + self.events.setdefault(self.stack[-1], []).append((kind, data, pos)) self.values[param] = (kind, data, pos) else: - if kind is START: - self.string.append(u'[%d:' % self.order) - self.events.setdefault(self.order, []).append((kind, data, pos)) + if kind is START: + self.string.append('[%d:' % self.order) self.stack.append(self.order) + self.events.setdefault(self.stack[-1], + []).append((kind, data, pos)) self.depth += 1 self.order += 1 elif kind is END: self.depth -= 1 if self.depth: self.events[self.stack[-1]].append((kind, data, pos)) - self.string.append(u']') + self.string.append(']') self.stack.pop() def format(self): """Return a message identifier representing the content in the buffer. """ - return u''.join(self.string).strip() + return ''.join(self.string).strip() def translate(self, string, regex=re.compile(r'%\((\w+)\)s')): """Interpolate the given message translation with the events in the @@ -442,41 +1014,108 @@ :param string: the translated message string """ + substream = None + + def yield_parts(string): + for idx, part in enumerate(regex.split(string)): + if idx % 2: + yield self.values[part] + elif part: + yield (TEXT, + part.replace('\[', '[').replace('\]', ']'), + (None, -1, -1) + ) + parts = parse_msg(string) + parts_counter = {} for order, string in parts: - events = self.events[order] - while events: - event = events.pop(0) - if event: - yield event + parts_counter.setdefault(order, []).append(None) + + while parts: + order, string = parts.pop(0) + if len(parts_counter[order]) == 1: + events = self.events[order] + else: + events = [self.events[order].pop(0)] + parts_counter[order].pop() + + for event in events: + if event[0] is SUB_START: + substream = [] + elif event[0] is SUB_END: + # Yield a substream which might have directives to be + # applied to it (after translation events) + yield SUB, (self.subdirectives[order], substream), event[2] + substream = None + elif event[0] is TEXT: + if string: + for part in yield_parts(string): + if substream is not None: + substream.append(part) + else: + yield part + # String handled, reset it + string = None + elif event[0] is START: + if substream is not None: + substream.append(event) + else: + yield event + if string: + for part in yield_parts(string): + if substream is not None: + substream.append(part) + else: + yield part + # String handled, reset it + string = None + elif event[0] is END: + if string: + for part in yield_parts(string): + if substream is not None: + substream.append(part) + else: + yield part + # String handled, reset it + string = None + if substream is not None: + substream.append(event) + else: + yield event + elif event[0] is EXPR: + # These are handled on the strings itself + continue else: - if not string: - break - for idx, part in enumerate(regex.split(string)): - if idx % 2: - yield self.values[part] - elif part: - yield TEXT, part, (None, -1, -1) - if not self.events[order] or not self.events[order][0]: - break + if string: + for part in yield_parts(string): + if substream is not None: + substream.append(part) + else: + yield part + # String handled, reset it + string = None + if substream is not None: + substream.append(event) + else: + yield event -def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|\]')): +def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|(?<!\\)\]')): """Parse a translated message using Genshi mixed content message formatting. - + >>> parse_msg("See [1:Help].") [(0, 'See '), (1, 'Help'), (0, '.')] - + >>> parse_msg("See [1:our [2:Help] page] for details.") [(0, 'See '), (1, 'our '), (2, 'Help'), (1, ' page'), (0, ' for details.')] - + >>> parse_msg("[2:Details] finden Sie in [1:Hilfe].") [(2, 'Details'), (0, ' finden Sie in '), (1, 'Hilfe'), (0, '.')] - + >>> parse_msg("[1:] Bilder pro Seite anzeigen.") [(1, ''), (0, ' Bilder pro Seite anzeigen.')] - + :param string: the translated message string :return: a list of ``(order, string)`` tuples :rtype: `list` @@ -510,14 +1149,13 @@ """Extract strings from Python bytecode. >>> from genshi.template.eval import Expression - >>> expr = Expression('_("Hello")') - >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS)) + >>> list(extract_from_code(expr, GETTEXT_FUNCTIONS)) [('_', u'Hello')] - + >>> expr = Expression('ngettext("You have %(num)s item", ' ... '"You have %(num)s items", num)') - >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS)) + >>> list(extract_from_code(expr, GETTEXT_FUNCTIONS)) [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None))] :param code: the `Code` object @@ -591,6 +1229,10 @@ tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None), encoding=encoding) + tmpl.loader = None + translator = Translator(None, ignore_tags, include_attrs, extract_text) + if hasattr(tmpl, 'add_directives'): + tmpl.add_directives(Translator.NAMESPACE, translator) for message in translator.extract(tmpl.stream, gettext_functions=keywords): yield message diff --git a/genshi/filters/tests/html.py b/genshi/filters/tests/html.py --- a/genshi/filters/tests/html.py +++ b/genshi/filters/tests/html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -26,7 +26,7 @@ </p></form>""") | HTMLFormFiller() self.assertEquals("""<form><p> <input type="text" name="foo"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_input_text_single_value(self): html = HTML("""<form><p> @@ -34,7 +34,7 @@ </p></form>""") | HTMLFormFiller(data={'foo': 'bar'}) self.assertEquals("""<form><p> <input type="text" name="foo" value="bar"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_input_text_multi_value(self): html = HTML("""<form><p> @@ -42,7 +42,7 @@ </p></form>""") | HTMLFormFiller(data={'foo': ['bar']}) self.assertEquals("""<form><p> <input type="text" name="foo" value="bar"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_input_hidden_no_value(self): html = HTML("""<form><p> @@ -50,7 +50,7 @@ </p></form>""") | HTMLFormFiller() self.assertEquals("""<form><p> <input type="hidden" name="foo"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_input_hidden_single_value(self): html = HTML("""<form><p> @@ -58,7 +58,7 @@ </p></form>""") | HTMLFormFiller(data={'foo': 'bar'}) self.assertEquals("""<form><p> <input type="hidden" name="foo" value="bar"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_input_hidden_multi_value(self): html = HTML("""<form><p> @@ -66,7 +66,7 @@ </p></form>""") | HTMLFormFiller(data={'foo': ['bar']}) self.assertEquals("""<form><p> <input type="hidden" name="foo" value="bar"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_textarea_no_value(self): html = HTML("""<form><p> @@ -74,7 +74,7 @@ </p></form>""") | HTMLFormFiller() self.assertEquals("""<form><p> <textarea name="foo"/> - </p></form>""", unicode(html)) + </p></form>""", html.render()) def test_fill_textarea_single_value(self): html = HTML("""<form><p> @@ -82,7 +82,7 @@ </p></form>""") | HTMLFormFiller(data={'foo': 'bar'}) self.assertEquals("""<form><p> <textarea name="foo">bar -
""", unicode(html)) +
""", html.render()) def test_fill_textarea_multi_value(self): html = HTML("""
@@ -90,7 +90,7 @@
""") | HTMLFormFiller(data={'foo': ['bar']}) self.assertEquals("""
bar -
""", unicode(html)) +
""", html.render()) def test_fill_input_checkbox_no_value(self): html = HTML("""
@@ -98,7 +98,7 @@
""") | HTMLFormFiller() self.assertEquals("""
-
""", unicode(html)) +
""", html.render()) def test_fill_input_checkbox_single_value_auto(self): html = HTML("""
@@ -106,10 +106,10 @@
""") self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': ''}))) +
""", (html | HTMLFormFiller(data={'foo': ''})).render()) self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': 'on'}))) +
""", (html | HTMLFormFiller(data={'foo': 'on'})).render()) def test_fill_input_checkbox_single_value_defined(self): html = HTML("""
@@ -117,10 +117,10 @@
""") self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': '1'}))) +
""", (html | HTMLFormFiller(data={'foo': '1'})).render()) self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': '2'}))) +
""", (html | HTMLFormFiller(data={'foo': '2'})).render()) def test_fill_input_checkbox_multi_value_auto(self): html = HTML("""
@@ -128,10 +128,10 @@
""") self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': []}))) +
""", (html | HTMLFormFiller(data={'foo': []})).render()) self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': ['on']}))) +
""", (html | HTMLFormFiller(data={'foo': ['on']})).render()) def test_fill_input_checkbox_multi_value_defined(self): html = HTML("""
@@ -139,10 +139,10 @@
""") self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': ['1']}))) +
""", (html | HTMLFormFiller(data={'foo': ['1']})).render()) self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': ['2']}))) +
""", (html | HTMLFormFiller(data={'foo': ['2']})).render()) def test_fill_input_radio_no_value(self): html = HTML("""
@@ -150,7 +150,7 @@
""") | HTMLFormFiller() self.assertEquals("""
-
""", unicode(html)) +
""", html.render()) def test_fill_input_radio_single_value(self): html = HTML("""
@@ -158,10 +158,10 @@
""") self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': '1'}))) +
""", (html | HTMLFormFiller(data={'foo': '1'})).render()) self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': '2'}))) +
""", (html | HTMLFormFiller(data={'foo': '2'})).render()) def test_fill_input_radio_multi_value(self): html = HTML("""
@@ -169,10 +169,10 @@
""") self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': ['1']}))) +
""", (html | HTMLFormFiller(data={'foo': ['1']})).render()) self.assertEquals("""
-
""", unicode(html | HTMLFormFiller(data={'foo': ['2']}))) +
""", (html | HTMLFormFiller(data={'foo': ['2']})).render()) def test_fill_select_no_value_auto(self): html = HTML("""
@@ -188,7 +188,7 @@ -
""", unicode(html)) +
""", html.render()) def test_fill_select_no_value_defined(self): html = HTML("""
@@ -204,7 +204,7 @@ -
""", unicode(html)) +
""", html.render()) def test_fill_select_single_value_auto(self): html = HTML("""
@@ -220,7 +220,7 @@ -
""", unicode(html)) +
""", html.render()) def test_fill_select_single_value_defined(self): html = HTML("""
@@ -236,7 +236,7 @@ -
""", unicode(html)) +
""", html.render()) def test_fill_select_multi_value_auto(self): html = HTML("""
@@ -252,7 +252,7 @@ -
""", unicode(html)) +
""", html.render()) def test_fill_select_multi_value_defined(self): html = HTML("""
@@ -268,7 +268,7 @@ -
""", unicode(html)) +
""", html.render()) def test_fill_option_segmented_text(self): html = MarkupTemplate("""
@@ -280,7 +280,7 @@ -
""", unicode(html)) + """, html.render()) def test_fill_option_segmented_text_no_value(self): html = MarkupTemplate("""
@@ -292,10 +292,10 @@ -
""", unicode(html)) + """, html.render()) def test_fill_option_unicode_value(self): - html = HTML(u"""
+ html = HTML(""" @@ -304,123 +304,163 @@ -
""", unicode(html)) + """, html.render(encoding=None)) + + def test_fill_input_password_disabled(self): + html = HTML("""
+ +
""") | HTMLFormFiller(data={'pass': 'bar'}) + self.assertEquals("""
+ +
""", html.render()) + + def test_fill_input_password_enabled(self): + html = HTML("""
+ +
""") | HTMLFormFiller(data={'pass': '1234'}, passwords=True) + self.assertEquals("""
+ +
""", html.render()) class HTMLSanitizerTestCase(unittest.TestCase): def test_sanitize_unchanged(self): html = HTML('fo
o') - self.assertEquals(u'fo
o', - unicode(html | HTMLSanitizer())) + self.assertEquals('fo
o', + (html | HTMLSanitizer()).render()) + html = HTML('foo') + self.assertEquals('foo', + (html | HTMLSanitizer()).render()) def test_sanitize_escape_text(self): html = HTML('fo&') - self.assertEquals(u'fo&', - unicode(html | HTMLSanitizer())) + self.assertEquals('fo&', + (html | HTMLSanitizer()).render()) html = HTML('<foo>') - self.assertEquals(u'<foo>', - unicode(html | HTMLSanitizer())) + self.assertEquals('<foo>', + (html | HTMLSanitizer()).render()) def test_sanitize_entityref_text(self): html = HTML('foö') self.assertEquals(u'foö', - unicode(html | HTMLSanitizer())) + (html | HTMLSanitizer()).render(encoding=None)) def test_sanitize_escape_attr(self): html = HTML('
') - self.assertEquals(u'
', - unicode(html | HTMLSanitizer())) + self.assertEquals('
', + (html | HTMLSanitizer()).render()) def test_sanitize_close_empty_tag(self): html = HTML('fo
o') - self.assertEquals(u'fo
o', - unicode(html | HTMLSanitizer())) + self.assertEquals('fo
o', + (html | HTMLSanitizer()).render()) def test_sanitize_invalid_entity(self): html = HTML('&junk;') - self.assertEquals('&junk;', unicode(html | HTMLSanitizer())) + self.assertEquals('&junk;', (html | HTMLSanitizer()).render()) def test_sanitize_remove_script_elem(self): html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) self.assertRaises(ParseError, HTML, 'alert("foo")') self.assertRaises(ParseError, HTML, '') def test_sanitize_remove_onclick_attr(self): html = HTML('
') - self.assertEquals(u'
', unicode(html | HTMLSanitizer())) + self.assertEquals('
', (html | HTMLSanitizer()).render()) + + def test_sanitize_remove_input_password(self): + html = HTML('
') + self.assertEquals('
', (html | HTMLSanitizer()).render()) def test_sanitize_remove_comments(self): html = HTML('''
''') - self.assertEquals(u'
', unicode(html | HTMLSanitizer())) + self.assertEquals('
', (html | HTMLSanitizer()).render()) def test_sanitize_remove_style_scripts(self): sanitizer = HTMLSanitizer(safe_attrs=HTMLSanitizer.SAFE_ATTRS | set(['style'])) # Inline style with url() using javascript: scheme html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) # Inline style with url() using javascript: scheme, using control char html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) # Inline style with url() using javascript: scheme, in quotes html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) # IE expressions in CSS not allowed html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) html = HTML('
') - self.assertEquals(u'
', - unicode(html | sanitizer)) + self.assertEquals('
', + (html | sanitizer).render()) # Inline style with url() using javascript: scheme, using unicode # escapes html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) html = HTML('
') - self.assertEquals(u'
', unicode(html | sanitizer)) + self.assertEquals('
', (html | sanitizer).render()) + + def test_sanitize_remove_style_phishing(self): + sanitizer = HTMLSanitizer(safe_attrs=HTMLSanitizer.SAFE_ATTRS | set(['style'])) + # The position property is not allowed + html = HTML('
') + self.assertEquals('
', (html | sanitizer).render()) + # Normal margins get passed through + html = HTML('
') + self.assertEquals('
', + (html | sanitizer).render()) + # But not negative margins + html = HTML('
') + self.assertEquals('
', (html | sanitizer).render()) + html = HTML('
') + self.assertEquals('
', (html | sanitizer).render()) + html = HTML('
') + self.assertEquals('
', (html | sanitizer).render()) def test_sanitize_remove_src_javascript(self): html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) # Case-insensitive protocol matching html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) # Grave accents (not parsed) self.assertRaises(ParseError, HTML, '') # Protocol encoded using UTF-8 numeric entities html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) # Protocol encoded using UTF-8 numeric entities without a semicolon # (which is allowed because the max number of digits is used) html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) # Protocol encoded using UTF-8 numeric hex entities without a semicolon # (which is allowed because the max number of digits is used) html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) # Embedded tab character in protocol html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) # Embedded tab character in protocol, but encoded this time html = HTML('') - self.assertEquals(u'', unicode(html | HTMLSanitizer())) + self.assertEquals('', (html | HTMLSanitizer()).render()) def suite(): @@ -430,5 +470,6 @@ suite.addTest(unittest.makeSuite(HTMLSanitizerTestCase, 'test')) return suite + if __name__ == '__main__': unittest.main(defaultTest='suite') diff --git a/genshi/filters/tests/i18n.py b/genshi/filters/tests/i18n.py --- a/genshi/filters/tests/i18n.py +++ b/genshi/filters/tests/i18n.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007-2008 Edgewall Software +# Copyright (C) 2007-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -18,16 +18,26 @@ import unittest from genshi.core import Attrs -from genshi.template import MarkupTemplate +from genshi.template import MarkupTemplate, Context from genshi.filters.i18n import Translator, extract from genshi.input import HTML class DummyTranslations(NullTranslations): + _domains = {} - def __init__(self, catalog): + def __init__(self, catalog=()): NullTranslations.__init__(self) - self._catalog = catalog + self._catalog = catalog or {} + self.plural = lambda n: n != 1 + + def add_domain(self, domain, catalog): + translation = DummyTranslations(catalog) + translation.add_fallback(self) + self._domains[domain] = translation + + def _domain_call(self, func, domain, *args, **kwargs): + return getattr(self._domains.get(domain, self), func)(*args, **kwargs) def ugettext(self, message): missing = object() @@ -38,6 +48,23 @@ return unicode(message) return tmsg + def dugettext(self, domain, message): + return self._domain_call('ugettext', domain, message) + + def ungettext(self, msgid1, msgid2, n): + try: + return self._catalog[(msgid1, self.plural(n))] + except KeyError: + if self._fallback: + return self._fallback.ngettext(msgid1, msgid2, n) + if n == 1: + return msgid1 + else: + return msgid2 + + def dungettext(self, domain, singular, plural, numeral): + return self._domain_call('ungettext', domain, singular, plural, numeral) + class TranslatorTestCase(unittest.TestCase): @@ -61,7 +88,7 @@ translator = Translator(extract_text=False) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((3, 'ngettext', (u'Singular', u'Plural', None), []), + self.assertEqual((3, 'ngettext', ('Singular', 'Plural', None), []), messages[0]) def test_extract_plural_form(self): @@ -71,7 +98,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, 'ngettext', (u'Singular', u'Plural', None), []), + self.assertEqual((2, 'ngettext', ('Singular', 'Plural', None), []), messages[0]) def test_extract_funky_plural_form(self): @@ -99,7 +126,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'Foo', []), messages[0]) + self.assertEqual((2, None, 'Foo', []), messages[0]) def test_extract_attribute_expr(self): tmpl = MarkupTemplate(""" @@ -108,7 +135,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, '_', u'Save', []), messages[0]) + self.assertEqual((2, '_', 'Save', []), messages[0]) def test_extract_non_included_attribute_interpolated(self): tmpl = MarkupTemplate(""" @@ -117,7 +144,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'Foo', []), messages[0]) + self.assertEqual((2, None, 'Foo', []), messages[0]) def test_extract_text_from_sub(self): tmpl = MarkupTemplate(""" @@ -126,7 +153,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'Foo', []), messages[0]) + self.assertEqual((2, None, 'Foo', []), messages[0]) def test_ignore_tag_with_fixed_xml_lang(self): tmpl = MarkupTemplate(""" @@ -143,7 +170,7 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, None, u'(c) 2007 Edgewall Software', []), + self.assertEqual((2, None, '(c) 2007 Edgewall Software', []), messages[0]) def test_ignore_attribute_with_expression(self): @@ -154,6 +181,20 @@ messages = list(translator.extract(tmpl.stream)) self.assertEqual(0, len(messages)) + def test_translate_with_translations_object(self): + tmpl = MarkupTemplate(""" +
Foo
+ """) + translator = Translator(DummyTranslations({'Foo': 'Voh'})) + translator.setup(tmpl) + self.assertEqual(""" +
Voh
+ """, tmpl.generate().render()) + + +class MsgDirectiveTestCase(unittest.TestCase): + def test_extract_i18n_msg(self): tmpl = MarkupTemplate(""" @@ -162,6 +203,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Please see [1:Help] for details.', messages[0][2]) @@ -175,12 +217,159 @@ """) gettext = lambda s: u"Für Details siehe bitte [1:Hilfe]." translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Für Details siehe bitte Hilfe.
""", tmpl.generate().render()) + def test_extract_i18n_msg_nonewline(self): + tmpl = MarkupTemplate(""" +
Please see Help
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual('Please see [1:Help]', messages[0][2]) + + def test_translate_i18n_msg_nonewline(self): + tmpl = MarkupTemplate(""" +
Please see Help
+ """) + gettext = lambda s: u"Für Details siehe bitte [1:Hilfe]" + translator = Translator(gettext) + translator.setup(tmpl) + self.assertEqual(""" +
Für Details siehe bitte Hilfe
+ """, tmpl.generate().render()) + + def test_extract_i18n_msg_elt_nonewline(self): + tmpl = MarkupTemplate(""" + Please see Help + """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual('Please see [1:Help]', messages[0][2]) + + def test_translate_i18n_msg_elt_nonewline(self): + tmpl = MarkupTemplate(""" + Please see Help + """) + gettext = lambda s: u"Für Details siehe bitte [1:Hilfe]" + translator = Translator(gettext) + translator.setup(tmpl) + self.assertEqual(""" + Für Details siehe bitte Hilfe + """, tmpl.generate().render()) + + def test_extract_i18n_msg_with_attributes(self): + tmpl = MarkupTemplate(""" +
+ Please see Help +
+ """) + translator = Translator() + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(3, len(messages)) + self.assertEqual('A helpful paragraph', messages[0][2]) + self.assertEqual(3, messages[0][0]) + self.assertEqual('Click for help', messages[1][2]) + self.assertEqual(4, messages[1][0]) + self.assertEqual('Please see [1:Help]', messages[2][2]) + self.assertEqual(3, messages[2][0]) + + def test_translate_i18n_msg_with_attributes(self): + tmpl = MarkupTemplate(""" +
+ Please see Help +
+ """) + translator = Translator(lambda msgid: { + 'A helpful paragraph': 'Ein hilfreicher Absatz', + 'Click for help': u'Klicken für Hilfe', + 'Please see [1:Help]': u'Siehe bitte [1:Hilfe]' + }[msgid]) + translator.setup(tmpl) + self.assertEqual(u""" +
Siehe bitte Hilfe
+ """, tmpl.generate().render(encoding=None)) + + def test_extract_i18n_msg_with_dynamic_attributes(self): + tmpl = MarkupTemplate(""" +
+ Please see Help +
+ """) + translator = Translator() + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(3, len(messages)) + self.assertEqual('A helpful paragraph', messages[0][2]) + self.assertEqual(3, messages[0][0]) + self.assertEqual('Click for help', messages[1][2]) + self.assertEqual(4, messages[1][0]) + self.assertEqual('Please see [1:Help]', messages[2][2]) + self.assertEqual(3, messages[2][0]) + + def test_translate_i18n_msg_with_dynamic_attributes(self): + tmpl = MarkupTemplate(""" +
+ Please see Help +
+ """) + translator = Translator(lambda msgid: { + 'A helpful paragraph': 'Ein hilfreicher Absatz', + 'Click for help': u'Klicken für Hilfe', + 'Please see [1:Help]': u'Siehe bitte [1:Hilfe]' + }[msgid]) + translator.setup(tmpl) + self.assertEqual(u""" +
Siehe bitte Hilfe
+ """, tmpl.generate(_=translator.translate).render(encoding=None)) + + def test_extract_i18n_msg_as_element_with_attributes(self): + tmpl = MarkupTemplate(""" + + Please see Help + + """) + translator = Translator() + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(2, len(messages)) + self.assertEqual('Click for help', messages[0][2]) + self.assertEqual(4, messages[0][0]) + self.assertEqual('Please see [1:Help]', messages[1][2]) + self.assertEqual(3, messages[1][0]) + + def test_translate_i18n_msg_as_element_with_attributes(self): + tmpl = MarkupTemplate(""" + + Please see Help + + """) + translator = Translator(lambda msgid: { + 'Click for help': u'Klicken für Hilfe', + 'Please see [1:Help]': u'Siehe bitte [1:Hilfe]' + }[msgid]) + translator.setup(tmpl) + self.assertEqual(u""" + Siehe bitte Hilfe + """, tmpl.generate().render(encoding=None)) + def test_extract_i18n_msg_nested(self): tmpl = MarkupTemplate(""" @@ -189,6 +378,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Please see [1:[2:Help] page] for details.', @@ -203,12 +393,39 @@ """) gettext = lambda s: u"Für Details siehe bitte [1:[2:Hilfeseite]]." translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Für Details siehe bitte Hilfeseite.
""", tmpl.generate().render()) + def test_extract_i18n_msg_label_with_nested_input(self): + tmpl = MarkupTemplate(""" +
+ days back +
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual('[1:[2:] days back]', + messages[0][2]) + + def test_translate_i18n_msg_label_with_nested_input(self): + tmpl = MarkupTemplate(""" +
+ foo bar +
+ """) + gettext = lambda s: "[1:[2:] foo bar]" + translator = Translator(gettext) + translator.setup(tmpl) + self.assertEqual(""" +
foo bar
+ """, tmpl.generate().render()) + def test_extract_i18n_msg_empty(self): tmpl = MarkupTemplate(""" @@ -217,6 +434,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Show me [1:] entries per page.', messages[0][2]) @@ -230,8 +448,7 @@ """) gettext = lambda s: u"[1:] Einträge pro Seite anzeigen." translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Einträge pro Seite anzeigen.
""", tmpl.generate().render()) @@ -244,6 +461,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Please see [1:Help] for [2:details].', messages[0][2]) @@ -257,8 +475,7 @@ """) gettext = lambda s: u"Für [2:Details] siehe bitte [1:Hilfe]." translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Für Details siehe bitte Hilfe.
""", tmpl.generate().render()) @@ -271,6 +488,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Show me [1:] entries per page, starting at page [2:].', @@ -285,8 +503,7 @@ """) gettext = lambda s: u"[1:] Einträge pro Seite, beginnend auf Seite [2:]." translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Eintr\xc3\xa4ge pro Seite, beginnend auf Seite .
""", tmpl.generate().render()) @@ -299,6 +516,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Hello, %(name)s!', messages[0][2]) @@ -312,8 +530,7 @@ """) gettext = lambda s: u"Hallo, %(name)s!" translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Hallo, Jim!
""", tmpl.generate(user=dict(name='Jim')).render()) @@ -327,8 +544,7 @@ """) gettext = lambda s: u"%(name)s, sei gegrüßt!" translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Jim, sei gegrüßt!
""", tmpl.generate(user=dict(name='Jim')).render()) @@ -342,8 +558,7 @@ """) gettext = lambda s: u"Sei gegrüßt, [1:Alter]!" translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Sei gegrüßt, Alter!
""", tmpl.generate(anchor='42').render()) @@ -356,6 +571,7 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Posted by %(name)s at %(time)s', messages[0][2]) @@ -369,8 +585,7 @@ """) gettext = lambda s: u"%(name)s schrieb dies um %(time)s" translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) entry = { 'author': 'Jim', 'time': datetime(2008, 4, 1, 14, 30) @@ -387,33 +602,44 @@
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) self.assertEqual('Show me [1:] entries per page.', messages[0][2]) - # FIXME: this currently fails :-/ -# def test_translate_i18n_msg_with_directive(self): -# tmpl = MarkupTemplate(""" -#
-# Show me entries per page. -#
-# """) -# gettext = lambda s: u"[1:] Einträge pro Seite anzeigen." -# tmpl.filters.insert(0, Translator(gettext)) -# self.assertEqual(""" -#
Einträge pro Seite anzeigen.
-# """, tmpl.generate().render()) + def test_translate_i18n_msg_with_directive(self): + tmpl = MarkupTemplate(""" +
+ Show me entries per page. +
+ """) + gettext = lambda s: u"[1:] Einträge pro Seite anzeigen." + translator = Translator(gettext) + translator.setup(tmpl) + self.assertEqual(""" +
Einträge pro Seite anzeigen.
+ """, tmpl.generate().render()) def test_extract_i18n_msg_with_comment(self): tmpl = MarkupTemplate(""" +
Foo
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, None, 'Foo', ['As in foo bar']), messages[0]) + tmpl = MarkupTemplate("""
Foo
""") translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((3, None, u'Foo', ['As in foo bar']), messages[0]) + self.assertEqual((3, None, 'Foo', ['As in foo bar']), messages[0]) def test_translate_i18n_msg_with_comment(self): tmpl = MarkupTemplate("""""") gettext = lambda s: u"Voh" translator = Translator(gettext) - tmpl.filters.insert(0, translator) - tmpl.add_directives(Translator.NAMESPACE, translator) + translator.setup(tmpl) self.assertEqual("""
Voh
""", tmpl.generate().render()) @@ -436,8 +661,8 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(2, len(messages)) - self.assertEqual((3, None, u'Foo bar', []), messages[0]) - self.assertEqual((3, None, u'Foo', []), messages[1]) + self.assertEqual((3, None, 'Foo bar', []), messages[0]) + self.assertEqual((3, None, 'Foo', []), messages[1]) def test_translate_i18n_msg_with_attr(self): tmpl = MarkupTemplate("""""") gettext = lambda s: u"Voh" translator = Translator(DummyTranslations({ - 'Foo': u'Voh', + 'Foo': 'Voh', 'Foo bar': u'Voh bär' })) tmpl.filters.insert(0, translator) @@ -455,18 +680,1127 @@
Voh
""", tmpl.generate().render()) - def test_translate_with_translations_object(self): + def test_translate_i18n_msg_and_py_strip_directives(self): tmpl = MarkupTemplate(""" -
Foo
+
Foo
+
Foo
""") translator = Translator(DummyTranslations({'Foo': 'Voh'})) - tmpl.filters.insert(0, translator) + translator.setup(tmpl) + self.assertEqual(""" + Voh + Voh + """, tmpl.generate().render()) + + def test_i18n_msg_ticket_300_extract(self): + tmpl = MarkupTemplate(""" + + Changed ${ '10/12/2008' } ago by ${ 'me, the author' } + + """) + translator = Translator() tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual( + (3, None, 'Changed %(date)s ago by %(author)s', []), messages[0] + ) + + def test_i18n_msg_ticket_300_translate(self): + tmpl = MarkupTemplate(""" + + Changed ${ date } ago by ${ author } + + """) + translations = DummyTranslations({ + 'Changed %(date)s ago by %(author)s': u'Modificado à %(date)s por %(author)s' + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" + Modificado à um dia por Pedro + """, tmpl.generate(date='um dia', author="Pedro").render()) + + + def test_i18n_msg_ticket_251_extract(self): + tmpl = MarkupTemplate(""" +
Translation[ 0 ]: One coin
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual( + (3, None, u'[1:[2:Translation\\[\xa00\xa0\\]]: [3:One coin]]', []), messages[0] + ) + + def test_i18n_msg_ticket_251_translate(self): + tmpl = MarkupTemplate(""" +
Translation[ 0 ]: One coin
+ """) + translations = DummyTranslations({ + u'[1:[2:Translation\\[\xa00\xa0\\]]: [3:One coin]]': + u'[1:[2:Trandução\\[\xa00\xa0\\]]: [3:Uma moeda]]' + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
Trandução[ 0 ]: Uma moeda
+ """, tmpl.generate().render()) + + def test_extract_i18n_msg_with_other_directives_nested(self): + tmpl = MarkupTemplate(""" +
Before you do that, though, please first try + searching + for similar issues, as it is quite likely that this problem + has been reported before. For questions about installation + and configuration of Trac, please try the + mailing list + instead of filing a ticket. +
+ """) + translator = Translator() + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual( + 'Before you do that, though, please first try\n ' + '[1:[2:searching]\n for similar issues], as it is ' + 'quite likely that this problem\n has been reported ' + 'before. For questions about installation\n and ' + 'configuration of Trac, please try the\n ' + '[3:mailing list]\n instead of filing a ticket.', + messages[0][2] + ) + + def test_translate_i18n_msg_with_other_directives_nested(self): + tmpl = MarkupTemplate(""" +
Before you do that, though, please first try + searching + for similar issues, as it is quite likely that this problem + has been reported before. For questions about installation + and configuration of Trac, please try the + mailing list + instead of filing a ticket. +
+ """) + translations = DummyTranslations({ + 'Before you do that, though, please first try\n ' + '[1:[2:searching]\n for similar issues], as it is ' + 'quite likely that this problem\n has been reported ' + 'before. For questions about installation\n and ' + 'configuration of Trac, please try the\n ' + '[3:mailing list]\n instead of filing a ticket.': + u'Antes de o fazer, porém,\n ' + u'[1:por favor tente [2:procurar]\n por problemas semelhantes], uma vez que ' + u'é muito provável que este problema\n já tenha sido reportado ' + u'anteriormente. Para questões relativas à instalação\n e ' + u'configuração do Trac, por favor tente a\n ' + u'[3:mailing list]\n em vez de criar um assunto.' + }) + translator = Translator(translations) + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + ctx = Context() + ctx.push({'trac': {'homepage': 'http://trac.edgewall.org/'}}) + self.assertEqual(""" +
Antes de o fazer, porém, + por favor tente procurar + por problemas semelhantes, uma vez que é muito provável que este problema + já tenha sido reportado anteriormente. Para questões relativas à instalação + e configuração do Trac, por favor tente a + mailing list + em vez de criar um assunto.
+ """, tmpl.generate(ctx).render()) + + def test_i18n_msg_with_other_nested_directives_with_reordered_content(self): + # See: http://genshi.edgewall.org/ticket/300#comment:10 + tmpl = MarkupTemplate(""" +
+ Note: This repository is defined in + trac.ini + and cannot be edited on this page. +
+ """) + translations = DummyTranslations({ + '[1:Note:] This repository is defined in\n ' + '[2:[3:trac.ini]]\n and cannot be edited on this page.': + u'[1:Nota:] Este repositório está definido em \n ' + u'[2:[3:trac.ini]]\n e não pode ser editado nesta página.', + }) + translator = Translator(translations) + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual( + '[1:Note:] This repository is defined in\n ' + '[2:[3:trac.ini]]\n and cannot be edited on this page.', + messages[0][2] + ) + self.assertEqual(""" +
Nota: Este repositório está definido em + trac.ini + e não pode ser editado nesta página.
+ """, tmpl.generate(editable=False).render()) + + def test_extract_i18n_msg_with_py_strip(self): + tmpl = MarkupTemplate(""" +
+ Please see Help for details. +
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, None, 'Please see [1:Help] for details.', []), + messages[0]) + + def test_extract_i18n_msg_with_py_strip_and_comment(self): + tmpl = MarkupTemplate(""" +
+ Please see Help for details. +
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, None, 'Please see [1:Help] for details.', + ['Foo']), messages[0]) + + def test_translate_i18n_msg_and_comment_with_py_strip_directives(self): + tmpl = MarkupTemplate(""" +
Foo
+
Foo
+ """) + translator = Translator(DummyTranslations({'Foo': 'Voh'})) + translator.setup(tmpl) + self.assertEqual(""" + Voh + Voh + """, tmpl.generate().render()) + + +class ChooseDirectiveTestCase(unittest.TestCase): + + def test_translate_i18n_choose_as_attribute(self): + tmpl = MarkupTemplate(""" +
+
FooBar
+
FooBars
+
+
+
FooBar
+
FooBars
+
+ """) + translations = DummyTranslations() + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
FooBar
+
+
+
FooBars
+
+ """, tmpl.generate(one=1, two=2).render()) + + def test_translate_i18n_choose_as_directive(self): + tmpl = MarkupTemplate(""" + +
FooBar
+
FooBars
+ + +
FooBar
+
FooBars
+ + """) + translations = DummyTranslations() + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
FooBars
+
FooBar
+ """, tmpl.generate(one=1, two=2).render()) + + def test_translate_i18n_choose_as_directive_singular_and_plural_with_strip(self): + tmpl = MarkupTemplate(""" + +
FooBar Singular with Strip
+
FooBars Plural without Strip
+ + +
FooBar singular without strip
+
FooBars plural with strip
+ + +
FooBar singular without strip
+
FooBars plural with strip
+ + +
FooBar singular with strip
+
FooBars plural without strip
+ + """) + translations = DummyTranslations() + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
FooBars Plural without Strip
+ FooBars plural with strip +
FooBar singular without strip
+ FooBar singular with strip + """, tmpl.generate(one=1, two=2).render()) + + def test_translate_i18n_choose_plural_singular_as_directive(self): + # Ticket 371 + tmpl = MarkupTemplate(""" + + FooBar + FooBars + + + FooBar + FooBars + + """) + translations = DummyTranslations({ + ('FooBar', 0): 'FuBar', + ('FooBars', 1): 'FuBars', + 'FooBar': 'FuBar', + 'FooBars': 'FuBars', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" + FuBars + FuBar + """, tmpl.generate(one=1, two=2).render()) + + def test_translate_i18n_choose_as_attribute_with_params(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translations = DummyTranslations({ + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
Vohs John Doe
+
+ """, tmpl.generate(two=2, fname='John', lname='Doe').render()) + + def test_translate_i18n_choose_as_attribute_with_params_and_domain_as_param(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translations = DummyTranslations() + translations.add_domain('foo', { + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
Vohs John Doe
+
+ """, tmpl.generate(two=2, fname='John', lname='Doe').render()) + + def test_translate_i18n_choose_as_directive_with_params(self): + tmpl = MarkupTemplate(""" + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + """) + translations = DummyTranslations({ + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
Vohs John Doe
+
Voh John Doe
+ """, tmpl.generate(one=1, two=2, + fname='John', lname='Doe').render()) + + def test_translate_i18n_choose_as_directive_with_params_and_domain_as_directive(self): + tmpl = MarkupTemplate(""" + + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + """) + translations = DummyTranslations() + translations.add_domain('foo', { + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
Vohs John Doe
+
Foo John Doe
+ """, tmpl.generate(one=1, two=2, + fname='John', lname='Doe').render()) + + def test_extract_i18n_choose_as_attribute(self): + tmpl = MarkupTemplate(""" +
+
FooBar
+
FooBars
+
+
+
FooBar
+
FooBars
+
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(2, len(messages)) + self.assertEqual((3, 'ngettext', ('FooBar', 'FooBars'), []), messages[0]) + self.assertEqual((7, 'ngettext', ('FooBar', 'FooBars'), []), messages[1]) + + def test_extract_i18n_choose_as_directive(self): + tmpl = MarkupTemplate(""" + +
FooBar
+
FooBars
+ + +
FooBar
+
FooBars
+ + """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(2, len(messages)) + self.assertEqual((3, 'ngettext', ('FooBar', 'FooBars'), []), messages[0]) + self.assertEqual((7, 'ngettext', ('FooBar', 'FooBars'), []), messages[1]) + + def test_extract_i18n_choose_as_attribute_with_params(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), []), + messages[0]) + + def test_extract_i18n_choose_as_attribute_with_params_and_domain_as_param(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((4, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), []), + messages[0]) + + def test_extract_i18n_choose_as_directive_with_params(self): + tmpl = MarkupTemplate(""" + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(2, len(messages)) + self.assertEqual((3, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), []), + messages[0]) + self.assertEqual((7, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), []), + messages[1]) + + def test_extract_i18n_choose_as_directive_with_params_and_domain_as_directive(self): + tmpl = MarkupTemplate(""" + + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(2, len(messages)) + self.assertEqual((4, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), []), + messages[0]) + self.assertEqual((9, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), []), + messages[1]) + + def test_extract_i18n_choose_as_attribute_with_params_and_comment(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), + ['As in Foo Bar']), + messages[0]) + + def test_extract_i18n_choose_as_directive_with_params_and_comment(self): + tmpl = MarkupTemplate(""" + +
Foo ${fname} ${lname}
+
Foos ${fname} ${lname}
+ + """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, 'ngettext', ('Foo %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s'), + ['As in Foo Bar']), + messages[0]) + + def test_extract_i18n_choose_with_attributes(self): + tmpl = MarkupTemplate(""" +
+ + There is ${num} thing. + + + There are ${num} things. + +
+ """) + translator = Translator() + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(4, len(messages)) + self.assertEqual((3, None, 'Things', []), messages[0]) + self.assertEqual((5, None, 'View thing', []), messages[1]) + self.assertEqual((8, None, 'View things', []), messages[2]) + self.assertEqual( + (3, 'ngettext', ('There is [1:%(num)s thing].', + 'There are [1:%(num)s things].'), []), + messages[3]) + + def test_translate_i18n_choose_with_attributes(self): + tmpl = MarkupTemplate(""" +
+ + There is ${num} thing. + + + There are ${num} things. + +
+ """) + translations = DummyTranslations({ + 'Things': 'Sachen', + 'View thing': 'Sache betrachten', + 'View things': 'Sachen betrachten', + ('There is [1:%(num)s thing].', 0): 'Da ist [1:%(num)s Sache].', + ('There is [1:%(num)s thing].', 1): 'Da sind [1:%(num)s Sachen].' + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(u""" +
+ Da ist 1 Sache. +
+ """, tmpl.generate(link="/things", num=1).render(encoding=None)) + self.assertEqual(u""" +
+ Da sind 3 Sachen. +
+ """, tmpl.generate(link="/things", num=3).render(encoding=None)) + + def test_extract_i18n_choose_as_element_with_attributes(self): + tmpl = MarkupTemplate(""" + +
+ There is ${num} thing. +
+
+ There are ${num} things. +
+ + """) + translator = Translator() + translator.setup(tmpl) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(5, len(messages)) + self.assertEqual((4, None, 'Things', []), messages[0]) + self.assertEqual((5, None, 'View thing', []), messages[1]) + self.assertEqual((7, None, 'Things', []), messages[2]) + self.assertEqual((8, None, 'View things', []), messages[3]) + self.assertEqual( + (3, 'ngettext', ('There is [1:%(num)s thing].', + 'There are [1:%(num)s things].'), []), + messages[4]) + + def test_translate_i18n_choose_as_element_with_attributes(self): + tmpl = MarkupTemplate(""" + +
+ There is ${num} thing. +
+
+ There are ${num} things. +
+ + """) + translations = DummyTranslations({ + 'Things': 'Sachen', + 'View thing': 'Sache betrachten', + 'View things': 'Sachen betrachten', + ('There is [1:%(num)s thing].', 0): 'Da ist [1:%(num)s Sache].', + ('There is [1:%(num)s thing].', 1): 'Da sind [1:%(num)s Sachen].' + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(u""" +
Da ist 1 Sache.
+ """, tmpl.generate(link="/things", num=1).render(encoding=None)) + self.assertEqual(u""" +
Da sind 3 Sachen.
+ """, tmpl.generate(link="/things", num=3).render(encoding=None)) + + def test_translate_i18n_choose_and_py_strip(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translations = DummyTranslations({ + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
Vohs John Doe
+
+ """, tmpl.generate(two=2, fname='John', lname='Doe').render()) + + def test_translate_i18n_choose_and_domain_and_py_strip(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translations = DummyTranslations() + translations.add_domain('foo', { + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
Vohs John Doe
+
+ """, tmpl.generate(two=2, fname='John', lname='Doe').render()) + + def test_translate_i18n_choose_and_singular_with_py_strip(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translations = DummyTranslations({ + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
Vohs John Doe
+
+
+ Voh John Doe +
+ """, tmpl.generate( + one=1, two=2, fname='John',lname='Doe').render()) + + def test_translate_i18n_choose_and_plural_with_py_strip(self): + tmpl = MarkupTemplate(""" +
+
Foo $fname $lname
+
Foos $fname $lname
+
+ """) + translations = DummyTranslations({ + ('Foo %(fname)s %(lname)s', 0): 'Voh %(fname)s %(lname)s', + ('Foo %(fname)s %(lname)s', 1): 'Vohs %(fname)s %(lname)s', + 'Foo %(fname)s %(lname)s': 'Voh %(fname)s %(lname)s', + 'Foos %(fname)s %(lname)s': 'Vohs %(fname)s %(lname)s', + }) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+ Voh John Doe +
+ """, tmpl.generate(two=1, fname='John', lname='Doe').render()) + + def test_extract_i18n_choose_as_attribute_and_py_strip(self): + tmpl = MarkupTemplate(""" +
+
FooBar
+
FooBars
+
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, 'ngettext', ('FooBar', 'FooBars'), []), messages[0]) + + +class DomainDirectiveTestCase(unittest.TestCase): + + def test_translate_i18n_domain_with_msg_directives(self): + #"""translate with i18n:domain and nested i18n:msg directives """ + + tmpl = MarkupTemplate(""" +
+
FooBar
+
Bar
+
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'PT_Foo'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
+
BarFoo
+
PT_Foo
+
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_with_inline_directives(self): + #"""translate with inlined i18n:domain and i18n:msg directives""" + tmpl = MarkupTemplate(""" +
FooBar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
BarFoo
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_without_msg_directives(self): + #"""translate domain call without i18n:msg directives still uses current domain""" + + tmpl = MarkupTemplate(""" +
Bar
+
+
FooBar
+
Bar
+
Bar
+
+
Bar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'PT_Foo'}) + translator = Translator(translations) + translator.setup(tmpl) self.assertEqual("""
Voh
+
+
BarFoo
+
PT_Foo
+
PT_Foo
+
+
Voh
""", tmpl.generate().render()) + def test_translate_i18n_domain_as_directive_not_attribute(self): + #"""translate with domain as directive""" + + tmpl = MarkupTemplate(""" + +
FooBar
+
Bar
+
Bar
+ +
Bar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'PT_Foo'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
BarFoo
+
PT_Foo
+
PT_Foo
+
Voh
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_nested_directives(self): + #"""translate with nested i18n:domain directives""" + + tmpl = MarkupTemplate(""" +
Bar
+
+
FooBar
+
Bar
+
Bar
+
+
Bar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'foo_Bar'}) + translations.add_domain('bar', {'Bar': 'bar_Bar'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
Voh
+
+
BarFoo
+
bar_Bar
+
foo_Bar
+
+
Voh
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_with_empty_nested_domain_directive(self): + #"""translate with empty nested i18n:domain directive does not use dngettext""" + + tmpl = MarkupTemplate(""" +
Bar
+
+
FooBar
+
Bar
+
Bar
+
+
Bar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'foo_Bar'}) + translations.add_domain('bar', {'Bar': 'bar_Bar'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
Voh
+
+
BarFoo
+
Voh
+
foo_Bar
+
+
Voh
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_with_inline_directive_on_START_NS(self): + tmpl = MarkupTemplate(""" +
FooBar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
BarFoo
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_with_inline_directive_on_START_NS_with_py_strip(self): + tmpl = MarkupTemplate(""" +
FooBar
+ """) + translations = DummyTranslations({'Bar': 'Voh'}) + translations.add_domain('foo', {'FooBar': 'BarFoo'}) + translator = Translator(translations) + translator.setup(tmpl) + self.assertEqual(""" +
BarFoo
+ """, tmpl.generate().render()) + + def test_translate_i18n_domain_with_nested_includes(self): + import os, shutil, tempfile + from genshi.template.loader import TemplateLoader + dirname = tempfile.mkdtemp(suffix='genshi_test') + try: + for idx in range(7): + file1 = open(os.path.join(dirname, 'tmpl%d.html' % idx), 'w') + try: + file1.write(""" +
Included tmpl$idx
+
Bar $idx
+
Bar
+
Bar $idx
+
Bar $idx
+ + + + """) + finally: + file1.close() + + file2 = open(os.path.join(dirname, 'tmpl10.html'), 'w') + try: + file2.write(""" + + """) + finally: + file2.close() + + def callback(template): + translations = DummyTranslations({'Bar %(idx)s': 'Voh %(idx)s'}) + translations.add_domain('foo', {'Bar %(idx)s': 'foo_Bar %(idx)s'}) + translations.add_domain('bar', {'Bar': 'bar_Bar'}) + translator = Translator(translations) + translator.setup(template) + loader = TemplateLoader([dirname], callback=callback) + tmpl = loader.load('tmpl10.html') + + self.assertEqual(""" +
Included tmpl0
+
foo_Bar 0
+
bar_Bar
+
Voh 0
+
Voh 0
+
Included tmpl1
+
foo_Bar 1
+
bar_Bar
+
Voh 1
+
Voh 1
+
Included tmpl2
+
foo_Bar 2
+
bar_Bar
+
Voh 2
+
Voh 2
+
Included tmpl3
+
foo_Bar 3
+
bar_Bar
+
Voh 3
+
Voh 3
+
Included tmpl4
+
foo_Bar 4
+
bar_Bar
+
Voh 4
+
Voh 4
+
Included tmpl5
+
foo_Bar 5
+
bar_Bar
+
Voh 5
+
Voh 5
+
Included tmpl6
+
foo_Bar 6
+
bar_Bar
+
Voh 6
+
Voh 6
+ """, tmpl.generate(idx=-1).render()) + finally: + shutil.rmtree(dirname) + + def test_translate_i18n_domain_with_nested_includes_with_translatable_attrs(self): + import os, shutil, tempfile + from genshi.template.loader import TemplateLoader + dirname = tempfile.mkdtemp(suffix='genshi_test') + try: + for idx in range(4): + file1 = open(os.path.join(dirname, 'tmpl%d.html' % idx), 'w') + try: + file1.write(""" +
Included tmpl$idx
+
Bar $idx
+
Bar
+
Bar $idx
+
Bar $idx
+
Bar $idx
+ + + + """) + finally: + file1.close() + + file2 = open(os.path.join(dirname, 'tmpl10.html'), 'w') + try: + file2.write(""" + + """) + finally: + file2.close() + + translations = DummyTranslations({'Bar %(idx)s': 'Voh %(idx)s', + 'Bar': 'Voh'}) + translations.add_domain('foo', {'Bar %(idx)s': 'foo_Bar %(idx)s'}) + translations.add_domain('bar', {'Bar': 'bar_Bar'}) + translator = Translator(translations) + + def callback(template): + translator.setup(template) + loader = TemplateLoader([dirname], callback=callback) + tmpl = loader.load('tmpl10.html') + + self.assertEqual(""" +
Included tmpl0
+
foo_Bar 0
+
bar_Bar
+
Voh 0
+
Voh 0
+
Voh 0
+
Included tmpl1
+
foo_Bar 1
+
bar_Bar
+
Voh 1
+
Voh 1
+
Voh 1
+
Included tmpl2
+
foo_Bar 2
+
bar_Bar
+
Voh 2
+
Voh 2
+
Voh 2
+
Included tmpl3
+
foo_Bar 3
+
bar_Bar
+
Voh 3
+
Voh 3
+
Voh 3
+ """, tmpl.generate(idx=-1, + dg=translations.dugettext).render()) + finally: + shutil.rmtree(dirname) + class ExtractTestCase(unittest.TestCase): @@ -483,10 +1817,10 @@ """) results = list(extract(buf, ['_', 'ngettext'], [], {})) self.assertEqual([ - (3, None, u'Example', []), - (6, None, u'Example', []), - (7, '_', u'Hello, %(name)s', []), - (8, 'ngettext', (u'You have %d item', u'You have %d items', None), + (3, None, 'Example', []), + (6, None, 'Example', []), + (7, '_', 'Hello, %(name)s', []), + (8, 'ngettext', ('You have %d item', 'You have %d items', None), []), ], results) @@ -499,26 +1833,26 @@ 'extract_text': 'no' })) self.assertEqual([ - (3, 'ngettext', (u'Singular', u'Plural', None), []), + (3, 'ngettext', ('Singular', 'Plural', None), []), ], results) def test_text_template_extraction(self): buf = StringIO("""${_("Dear %(name)s") % {'name': name}}, - + ${ngettext("Your item:", "Your items", len(items))} #for item in items * $item #end - + All the best, Foobar""") results = list(extract(buf, ['_', 'ngettext'], [], { 'template_class': 'genshi.template:TextTemplate' })) self.assertEqual([ - (1, '_', u'Dear %(name)s', []), - (3, 'ngettext', (u'Your item:', u'Your items', None), []), - (7, None, u'All the best,\n Foobar', []) + (1, '_', 'Dear %(name)s', []), + (3, 'ngettext', ('Your item:', 'Your items', None), []), + (7, None, 'All the best,\n Foobar', []) ], results) def test_extraction_with_keyword_arg(self): @@ -527,7 +1861,7 @@ """) results = list(extract(buf, ['gettext'], [], {})) self.assertEqual([ - (2, 'gettext', (u'Foobar'), []), + (2, 'gettext', ('Foobar'), []), ], results) def test_extraction_with_nonstring_arg(self): @@ -536,7 +1870,7 @@ """) results = list(extract(buf, ['dgettext'], [], {})) self.assertEqual([ - (2, 'dgettext', (None, u'Foobar'), []), + (2, 'dgettext', (None, 'Foobar'), []), ], results) def test_extraction_inside_ignored_tags(self): @@ -550,7 +1884,7 @@ """) results = list(extract(buf, ['_'], [], {})) self.assertEqual([ - (5, '_', u'Please wait...', []), + (5, '_', 'Please wait...', []), ], results) def test_extraction_inside_ignored_tags_with_directives(self): @@ -563,11 +1897,68 @@ """) self.assertEqual([], list(extract(buf, ['_'], [], {}))) + def test_extract_py_def_directive_with_py_strip(self): + # Failed extraction from Trac + tmpl = MarkupTemplate(""" + + View differences + +
+ Show + lines around each change +
+
+ Ignore: +
+ + Blank lines +
+
+ + Case changes +
+
+ + White space changes +
+
+
+ +
+ """) + translator = Translator() + tmpl.add_directives(Translator.NAMESPACE, translator) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(10, len(messages)) + self.assertEqual([ + (3, None, 'View differences', []), + (6, None, 'inline', []), + (8, None, 'side by side', []), + (10, None, 'Show', []), + (13, None, 'lines around each change', []), + (16, None, 'Ignore:', []), + (20, None, 'Blank lines', []), + (25, None, 'Case changes',[]), + (30, None, 'White space changes', []), + (34, '_', 'Update', [])], messages) + def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(Translator.__module__)) suite.addTest(unittest.makeSuite(TranslatorTestCase, 'test')) + suite.addTest(unittest.makeSuite(MsgDirectiveTestCase, 'test')) + suite.addTest(unittest.makeSuite(ChooseDirectiveTestCase, 'test')) + suite.addTest(unittest.makeSuite(DomainDirectiveTestCase, 'test')) suite.addTest(unittest.makeSuite(ExtractTestCase, 'test')) return suite diff --git a/genshi/filters/tests/transform.py b/genshi/filters/tests/transform.py --- a/genshi/filters/tests/transform.py +++ b/genshi/filters/tests/transform.py @@ -544,29 +544,29 @@ def test_map_element(self): self.assertEqual( self._map('foo'), - [(QName(u'foo'), Attrs([(QName(u'name'), u'foo'), - (QName(u'size'), u'100')])), + [(QName('foo'), Attrs([(QName('name'), u'foo'), + (QName('size'), u'100')])), u'FOO', - QName(u'foo')] - ) + QName('foo')] + ) def test_map_with_text_kind(self): self.assertEqual( self._map('.', TEXT), [u'ROOT', u'FOO', u'BAR'] - ) + ) def test_map_with_root_and_end_kind(self): self.assertEqual( self._map('.', END), - [QName(u'foo'), QName(u'bar'), QName(u'root')] - ) + [QName('foo'), QName('bar'), QName('root')] + ) def test_map_with_attribute(self): self.assertEqual( self._map('foo/@name'), - [(QName(u'foo@*'), Attrs([('name', u'foo')]))] - ) + [(QName('foo@*'), Attrs([('name', u'foo')]))] + ) class SubstituteTest(unittest.TestCase): diff --git a/genshi/filters/transform.py b/genshi/filters/transform.py --- a/genshi/filters/transform.py +++ b/genshi/filters/transform.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -32,8 +32,8 @@ ... Some body text. ... ... ''') ->>> print html | Transformer('body/em').map(unicode.upper, TEXT) \\ -... .unwrap().wrap(tag.u) +>>> print(html | Transformer('body/em').map(unicode.upper, TEXT) +... .unwrap().wrap(tag.u)) Some Title @@ -142,7 +142,7 @@ Here's an example of removing some markup (the title, in this case) selected by an expression: - >>> print html | Transformer('head/title').remove() + >>> print(html | Transformer('head/title').remove()) Some body text. Inserted content can be passed in the form of a string, or a markup event @@ -150,7 +150,7 @@ `builder` module: >>> from genshi.builder import tag - >>> print html | Transformer('body').prepend(tag.h1('Document Title')) + >>> print(html | Transformer('body').prepend(tag.h1('Document Title'))) Some Title
Document Title
Some body text. @@ -160,8 +160,8 @@ copied text into the body as ``
`` enclosed text: >>> buffer = StreamBuffer() - >>> print html | Transformer('head/title/text()').copy(buffer) \\ - ... .end().select('body').prepend(tag.h1(buffer)) + >>> print(html | Transformer('head/title/text()').copy(buffer) + ... .end().select('body').prepend(tag.h1(buffer))) Some Title
Some Title
Some body text. @@ -170,7 +170,7 @@ transforms: >>> emphasis = Transformer('body//em').attr('class', 'emphasis') - >>> print html | emphasis + >>> print(html | emphasis) Some TitleSome body text. """ @@ -216,7 +216,7 @@ ... else: ... yield mark, (kind, data, pos) >>> short_stream = HTML('Some test text') - >>> print short_stream | Transformer('.//em/text()').apply(upper) + >>> print(short_stream | Transformer('.//em/text()').apply(upper)) Some TEST text """ transformer = Transformer() @@ -234,14 +234,14 @@ selection. >>> html = HTML('Some test text') - >>> print html | Transformer().select('.//em').trace() - (None, ('START', (QName(u'body'), Attrs()), (None, 1, 0))) + >>> print(html | Transformer().select('.//em').trace()) + (None, ('START', (QName('body'), Attrs()), (None, 1, 0))) (None, ('TEXT', u'Some ', (None, 1, 6))) - ('ENTER', ('START', (QName(u'em'), Attrs()), (None, 1, 11))) + ('ENTER', ('START', (QName('em'), Attrs()), (None, 1, 11))) ('INSIDE', ('TEXT', u'test', (None, 1, 15))) - ('EXIT', ('END', QName(u'em'), (None, 1, 19))) + ('EXIT', ('END', QName('em'), (None, 1, 19))) (None, ('TEXT', u' text', (None, 1, 24))) - (None, ('END', QName(u'body'), (None, 1, 29))) + (None, ('END', QName('body'), (None, 1, 29))) Some test text :param path: an XPath expression (as string) or a `Path` instance @@ -258,14 +258,14 @@ are converted to OUTSIDE marks. >>> html = HTML('Some test text') - >>> print html | Transformer('//em').invert().trace() - ('OUTSIDE', ('START', (QName(u'body'), Attrs()), (None, 1, 0))) + >>> print(html | Transformer('//em').invert().trace()) + ('OUTSIDE', ('START', (QName('body'), Attrs()), (None, 1, 0))) ('OUTSIDE', ('TEXT', u'Some ', (None, 1, 6))) - (None, ('START', (QName(u'em'), Attrs()), (None, 1, 11))) + (None, ('START', (QName('em'), Attrs()), (None, 1, 11))) (None, ('TEXT', u'test', (None, 1, 15))) - (None, ('END', QName(u'em'), (None, 1, 19))) + (None, ('END', QName('em'), (None, 1, 19))) ('OUTSIDE', ('TEXT', u' text', (None, 1, 24))) - ('OUTSIDE', ('END', QName(u'body'), (None, 1, 29))) + ('OUTSIDE', ('END', QName('body'), (None, 1, 29))) Some test text :rtype: `Transformer` @@ -278,14 +278,14 @@ Example: >>> html = HTML('Some test text') - >>> print html | Transformer('//em').end().trace() - ('OUTSIDE', ('START', (QName(u'body'), Attrs()), (None, 1, 0))) + >>> print(html | Transformer('//em').end().trace()) + ('OUTSIDE', ('START', (QName('body'), Attrs()), (None, 1, 0))) ('OUTSIDE', ('TEXT', u'Some ', (None, 1, 6))) - ('OUTSIDE', ('START', (QName(u'em'), Attrs()), (None, 1, 11))) + ('OUTSIDE', ('START', (QName('em'), Attrs()), (None, 1, 11))) ('OUTSIDE', ('TEXT', u'test', (None, 1, 15))) - ('OUTSIDE', ('END', QName(u'em'), (None, 1, 19))) + ('OUTSIDE', ('END', QName('em'), (None, 1, 19))) ('OUTSIDE', ('TEXT', u' text', (None, 1, 24))) - ('OUTSIDE', ('END', QName(u'body'), (None, 1, 29))) + ('OUTSIDE', ('END', QName('body'), (None, 1, 29))) Some test text :return: the stream augmented by transformation marks @@ -302,7 +302,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em').empty() + >>> print(html | Transformer('.//em').empty()) Some TitleSome text. @@ -317,7 +317,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em').remove() + >>> print(html | Transformer('.//em').remove()) Some TitleSome text. @@ -334,7 +334,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em').unwrap() + >>> print(html | Transformer('.//em').unwrap()) Some TitleSome body text. @@ -347,7 +347,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em').wrap('strong') + >>> print(html | Transformer('.//em').wrap('strong')) Some TitleSome body text. @@ -363,7 +363,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//title/text()').replace('New Title') + >>> print(html | Transformer('.//title/text()').replace('New Title')) New TitleSome body text. @@ -381,7 +381,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em').before('emphasised ') + >>> print(html | Transformer('.//em').before('emphasised ')) Some TitleSome emphasised body text. @@ -398,7 +398,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em').after(' rock') + >>> print(html | Transformer('.//em').after(' rock')) Some TitleSome body rock text. @@ -415,7 +415,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//body').prepend('Some new body text. ') + >>> print(html | Transformer('.//body').prepend('Some new body text. ')) Some TitleSome new body text. Some body text. @@ -430,7 +430,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//body').append(' Some new body text.') + >>> print(html | Transformer('.//body').append(' Some new body text.')) Some TitleSome body text. Some new body text. @@ -451,13 +451,13 @@ >>> html = HTML('Some Title' ... 'Some body text.' ... '') - >>> print html | Transformer('body/em').attr('class', None) + >>> print(html | Transformer('body/em').attr('class', None)) Some TitleSome body text. Otherwise the attribute will be set to `value`: - >>> print html | Transformer('body/em').attr('class', 'emphasis') + >>> print(html | Transformer('body/em').attr('class', 'emphasis')) Some TitleSome body text. @@ -467,10 +467,10 @@ >>> def print_attr(name, event): ... attrs = event[1][1] - ... print attrs + ... print(attrs) ... return attrs.get(name) - >>> print html | Transformer('body/em').attr('class', print_attr) - Attrs([(QName(u'class'), u'before')]) + >>> print(html | Transformer('body/em').attr('class', print_attr)) + Attrs([(QName('class'), u'before')]) Attrs() Some TitleSome body text. @@ -494,20 +494,20 @@ >>> buffer = StreamBuffer() >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('head/title/text()').copy(buffer) \\ - ... .end().select('body').prepend(tag.h1(buffer)) + >>> print(html | Transformer('head/title/text()').copy(buffer) + ... .end().select('body').prepend(tag.h1(buffer))) Some Title
Some Title
Some body text. This example illustrates that only a single contiguous selection will be buffered: - >>> print html | Transformer('head/title/text()').copy(buffer) \\ - ... .end().select('body/em').copy(buffer).end().select('body') \\ - ... .prepend(tag.h1(buffer)) + >>> print(html | Transformer('head/title/text()').copy(buffer) + ... .end().select('body/em').copy(buffer).end().select('body') + ... .prepend(tag.h1(buffer))) Some Title
Some Title
Some body text. - >>> print buffer + >>> print(buffer) body Element attributes can also be copied for later use: @@ -518,9 +518,9 @@ >>> buffer = StreamBuffer() >>> def apply_attr(name, entry): ... return list(buffer)[0][1][1].get('class') - >>> print html | Transformer('body/em[@class]/@class').copy(buffer) \\ - ... .end().buffer().select('body/em[not(@class)]') \\ - ... .attr('class', apply_attr) + >>> print(html | Transformer('body/em[@class]/@class').copy(buffer) + ... .end().buffer().select('body/em[not(@class)]') + ... .attr('class', apply_attr)) Some TitleSome bodytext. @@ -547,8 +547,8 @@ >>> buffer = StreamBuffer() >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('.//em/text()').cut(buffer) \\ - ... .end().select('.//em').after(tag.h1(buffer)) + >>> print(html | Transformer('.//em/text()').cut(buffer) + ... .end().select('.//em').after(tag.h1(buffer))) Some TitleSome body
text. @@ -579,8 +579,8 @@ >>> doc = HTML('Some one ' ... 'text two.') >>> buffer = StreamBuffer() - >>> print doc | Transformer('body/note').cut(buffer, accumulate=True) \\ - ... .end().buffer().select('notes').prepend(buffer) + >>> print(doc | Transformer('body/note').cut(buffer, accumulate=True) + ... .end().buffer().select('notes').prepend(buffer)) onetwoSome text . @@ -596,7 +596,7 @@ >>> from genshi.filters.html import HTMLSanitizer >>> html = HTML('Some text and some more text') - >>> print html | Transformer('body/*').filter(HTMLSanitizer()) + >>> print(html | Transformer('body/*').filter(HTMLSanitizer())) Some text and some more text :param filter: The stream filter to apply. @@ -610,7 +610,7 @@ >>> html = HTML('Some Title' ... 'Some body text.') - >>> print html | Transformer('head/title').map(unicode.upper, TEXT) + >>> print(html | Transformer('head/title').map(unicode.upper, TEXT)) SOME TITLESome body text. @@ -628,13 +628,13 @@ >>> html = HTML('Some text, some more text and ' ... 'some bold text\\n' ... 'some italicised text') - >>> print html | Transformer('body/b').substitute('(?i)some', 'SOME') + >>> print(html | Transformer('body/b').substitute('(?i)some', 'SOME')) Some text, some more text and SOME bold text some italicised text >>> tags = tag.html(tag.body('Some text, some more text and\\n', ... Markup('some bold text'))) - >>> print tags.generate() | Transformer('body').substitute( - ... '(?i)some', 'SOME') + >>> print(tags.generate() | Transformer('body').substitute( + ... '(?i)some', 'SOME')) SOME text, some more text and SOME bold text @@ -650,7 +650,7 @@ >>> html = HTML('Some text, some more text and ' ... 'some bold text') - >>> print html | Transformer('body/b').rename('strong') + >>> print(html | Transformer('body/b').rename('strong')) Some text, some more text and some bold text """ return self.apply(RenameTransformation(name)) @@ -659,14 +659,14 @@ """Print events as they pass through the transform. >>> html = HTML('Some test text') - >>> print html | Transformer('em').trace() - (None, ('START', (QName(u'body'), Attrs()), (None, 1, 0))) + >>> print(html | Transformer('em').trace()) + (None, ('START', (QName('body'), Attrs()), (None, 1, 0))) (None, ('TEXT', u'Some ', (None, 1, 6))) - ('ENTER', ('START', (QName(u'em'), Attrs()), (None, 1, 11))) + ('ENTER', ('START', (QName('em'), Attrs()), (None, 1, 11))) ('INSIDE', ('TEXT', u'test', (None, 1, 15))) - ('EXIT', ('END', QName(u'em'), (None, 1, 19))) + ('EXIT', ('END', QName('em'), (None, 1, 19))) (None, ('TEXT', u' text', (None, 1, 24))) - (None, ('END', QName(u'body'), (None, 1, 29))) + (None, ('END', QName('body'), (None, 1, 29))) Some test text :param prefix: a string to prefix each event with in the output @@ -876,7 +876,7 @@ :param stream: the marked event stream to filter """ for event in stream: - print>>self.fileobj, self.prefix + str(event) + self.fileobj.write('%s%s\n' % (self.prefix, event)) yield event @@ -1025,7 +1025,7 @@ ... for event in stream: ... yield event >>> html = HTML('Some test text') - >>> print html | Transformer('.//em').apply(Top('Prefix ')) + >>> print(html | Transformer('.//em').apply(Top('Prefix '))) Prefix Some test text """ def __init__(self, content): diff --git a/genshi/input.py b/genshi/input.py --- a/genshi/input.py +++ b/genshi/input.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -16,18 +16,19 @@ """ from itertools import chain -from xml.parsers import expat +import htmlentitydefs as entities import HTMLParser as html -import htmlentitydefs from StringIO import StringIO +from xml.parsers import expat from genshi.core import Attrs, QName, Stream, stripentities -from genshi.core import START, END, XML_DECL, DOCTYPE, TEXT, START_NS, END_NS, \ - START_CDATA, END_CDATA, PI, COMMENT +from genshi.core import START, END, XML_DECL, DOCTYPE, TEXT, START_NS, \ + END_NS, START_CDATA, END_CDATA, PI, COMMENT __all__ = ['ET', 'ParseError', 'XMLParser', 'XML', 'HTMLParser', 'HTML'] __docformat__ = 'restructuredtext en' + def ET(element): """Convert a given ElementTree element to a markup stream. @@ -79,16 +80,16 @@ >>> parser = XMLParser(StringIO('Foo')) >>> for kind, data, pos in parser: - ... print kind, data - START (QName(u'root'), Attrs([(QName(u'id'), u'2')])) - START (QName(u'child'), Attrs()) + ... print('%s %s' % (kind, data)) + START (QName('root'), Attrs([(QName('id'), u'2')])) + START (QName('child'), Attrs()) TEXT Foo END child END root """ _entitydefs = ['' % (name, value) for name, value in - htmlentitydefs.name2codepoint.items()] + entities.name2codepoint.items()] _external_dtd = '\n'.join(_entitydefs) def __init__(self, source, filename=None, encoding=None): @@ -237,7 +238,7 @@ if text.startswith('&'): # deal with undefined entities try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + text = unichr(entities.name2codepoint[text[1:-1]]) self._enqueue(TEXT, text) except KeyError: filename, lineno, offset = self._getpos() @@ -256,11 +257,11 @@ iterated over multiple times: >>> xml = XML('FooBar') - >>> print xml + >>> print(xml) FooBar - >>> print xml.select('elem') + >>> print(xml.select('elem')) FooBar - >>> print xml.select('elem/text()') + >>> print(xml.select('elem/text()')) FooBar :param text: the XML source @@ -280,9 +281,9 @@ >>> parser = HTMLParser(StringIO('
Foo
')) >>> for kind, data, pos in parser: - ... print kind, data - START (QName(u'ul'), Attrs([(QName(u'compact'), u'compact')])) - START (QName(u'li'), Attrs()) + ... print('%s %s' % (kind, data)) + START (QName('ul'), Attrs([(QName('compact'), u'compact')])) + START (QName('li'), Attrs()) TEXT Foo END li END ul @@ -387,7 +388,7 @@ def handle_entityref(self, name): try: - text = unichr(htmlentitydefs.name2codepoint[name]) + text = unichr(entities.name2codepoint[name]) except KeyError: text = '&%s;' % name self._enqueue(TEXT, text) @@ -409,11 +410,11 @@ iterated over multiple times: >>> html = HTML('
Foo
') - >>> print html + >>> print(html)
Foo
- >>> print html.select('h1') + >>> print(html.select('h1'))
Foo
- >>> print html.select('h1/text()') + >>> print(html.select('h1/text()')) Foo :param text: the HTML source @@ -423,6 +424,7 @@ """ return Stream(list(HTMLParser(StringIO(text), encoding=encoding))) + def _coalesce(stream): """Coalesces adjacent TEXT events into a single event.""" textbuf = [] @@ -434,7 +436,7 @@ textpos = pos else: if textbuf: - yield TEXT, u''.join(textbuf), textpos + yield TEXT, ''.join(textbuf), textpos del textbuf[:] textpos = None if kind: diff --git a/genshi/output.py b/genshi/output.py --- a/genshi/output.py +++ b/genshi/output.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -26,6 +26,7 @@ 'XHTMLSerializer', 'HTMLSerializer', 'TextSerializer'] __docformat__ = 'restructuredtext en' + def encode(iterator, method='xml', encoding='utf-8', out=None): """Encode serializer output into a string. @@ -53,10 +54,11 @@ else: _encode = lambda string: string if out is None: - return _encode(u''.join(list(iterator))) + return _encode(''.join(list(iterator))) for chunk in iterator: out.write(_encode(chunk)) + def get_serializer(method='xml', **kwargs): """Return a serializer object for the given method. @@ -172,7 +174,7 @@ >>> from genshi.builder import tag >>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True)) - >>> print ''.join(XMLSerializer()(elem.generate())) + >>> print(''.join(XMLSerializer()(elem.generate())))

""" @@ -229,7 +231,7 @@ for attr, value in attrib: buf += [' ', attr, '="', escape(value), '"'] buf.append(kind is EMPTY and '/>' or '>') - yield _emit(kind, data, Markup(u''.join(buf))) + yield _emit(kind, data, Markup(''.join(buf))) elif kind is END: yield _emit(kind, data, Markup('' % data)) @@ -252,7 +254,7 @@ standalone = standalone and 'yes' or 'no' buf.append(' standalone="%s"' % standalone) buf.append('?>\n') - yield Markup(u''.join(buf)) + yield Markup(''.join(buf)) have_decl = True elif kind is DOCTYPE and not have_doctype: @@ -265,7 +267,7 @@ if sysid: buf.append(' "%s"') buf.append('>\n') - yield Markup(u''.join(buf)) % filter(None, data) + yield Markup(''.join(buf)) % tuple([p for p in data if p]) have_doctype = True elif kind is START_CDATA: @@ -285,7 +287,7 @@ >>> from genshi.builder import tag >>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True)) - >>> print ''.join(XHTMLSerializer()(elem.generate())) + >>> print(''.join(XHTMLSerializer()(elem.generate())))

""" @@ -345,9 +347,9 @@ for attr, value in attrib: if attr in boolean_attrs: value = attr - elif attr == u'xml:lang' and u'lang' not in attrib: + elif attr == 'xml:lang' and 'lang' not in attrib: buf += [' lang="', escape(value), '"'] - elif attr == u'xml:space': + elif attr == 'xml:space': continue buf += [' ', attr, '="', escape(value), '"'] if kind is EMPTY: @@ -357,7 +359,7 @@ buf.append('>' % tag) else: buf.append('>') - yield _emit(kind, data, Markup(u''.join(buf))) + yield _emit(kind, data, Markup(''.join(buf))) elif kind is END: yield _emit(kind, data, Markup('' % data)) @@ -381,7 +383,7 @@ if sysid: buf.append(' "%s"') buf.append('>\n') - yield Markup(u''.join(buf)) % filter(None, data) + yield Markup(''.join(buf)) % tuple([p for p in data if p]) have_doctype = True elif kind is XML_DECL and not have_decl and not drop_xml_decl: @@ -393,7 +395,7 @@ standalone = standalone and 'yes' or 'no' buf.append(' standalone="%s"' % standalone) buf.append('?>\n') - yield Markup(u''.join(buf)) + yield Markup(''.join(buf)) have_decl = True elif kind is START_CDATA: @@ -413,7 +415,7 @@ >>> from genshi.builder import tag >>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True)) - >>> print ''.join(HTMLSerializer()(elem.generate())) + >>> print(''.join(HTMLSerializer()(elem.generate())))

""" @@ -469,7 +471,8 @@ output = cache_get((kind, data)) if output is not None: yield output - if kind is START or kind is EMPTY and data[0] in noescape_elems: + if (kind is START or kind is EMPTY) \ + and data[0] in noescape_elems: noescape = True elif kind is END: noescape = False @@ -482,7 +485,7 @@ if value: buf += [' ', attr] elif ':' in attr: - if attr == 'xml:lang' and u'lang' not in attrib: + if attr == 'xml:lang' and 'lang' not in attrib: buf += [' lang="', escape(value), '"'] elif attr != 'xmlns': buf += [' ', attr, '="', escape(value), '"'] @@ -490,7 +493,7 @@ if kind is EMPTY: if tag not in empty_elems: buf.append('' % tag) - yield _emit(kind, data, Markup(u''.join(buf))) + yield _emit(kind, data, Markup(''.join(buf))) if tag in noescape_elems: noescape = True @@ -517,7 +520,7 @@ if sysid: buf.append(' "%s"') buf.append('>\n') - yield Markup(u''.join(buf)) % filter(None, data) + yield Markup(''.join(buf)) % tuple([p for p in data if p]) have_doctype = True elif kind is PI: @@ -532,23 +535,24 @@ >>> from genshi.builder import tag >>> elem = tag.div(tag.a('', href='foo'), tag.br) - >>> print elem + >>> print(elem)
<Hello!>
- >>> print ''.join(TextSerializer()(elem.generate())) + >>> print(''.join(TextSerializer()(elem.generate()))) If text events contain literal markup (instances of the `Markup` class), that markup is by default passed through unchanged: >>> elem = tag.div(Markup('Hello & Bye!
')) - >>> print elem.generate().render(TextSerializer) + >>> print(elem.generate().render(TextSerializer, encoding=None)) Hello & Bye!
You can use the ``strip_markup`` to change this behavior, so that tags and entities are stripped from the output (or in the case of entities, replaced with the equivalent character): - >>> print elem.generate().render(TextSerializer, strip_markup=True) + >>> print(elem.generate().render(TextSerializer, strip_markup=True, + ... encoding=None)) Hello & Bye! """ @@ -606,8 +610,8 @@ ... ... ''') >>> for kind, data, pos in NamespaceFlattener()(xml): - ... print kind, repr(data) - START (u'doc', Attrs([(u'xmlns', u'NS1'), (u'xmlns:two', u'NS2')])) + ... print('%s %r' % (kind, data)) + START (u'doc', Attrs([('xmlns', u'NS1'), (u'xmlns:two', u'NS2')])) TEXT u'\n ' START (u'two:item', Attrs()) END u'two:item' @@ -654,7 +658,7 @@ ns_attrs = [] _push_ns_attr = ns_attrs.append def _make_ns_attr(prefix, uri): - return u'xmlns%s' % (prefix and ':%s' % prefix or ''), uri + return 'xmlns%s' % (prefix and ':%s' % prefix or ''), uri def _gen_prefix(): val = 0 @@ -677,9 +681,9 @@ if tagns in namespaces: prefix = namespaces[tagns][-1] if prefix: - tagname = u'%s:%s' % (prefix, tagname) + tagname = '%s:%s' % (prefix, tagname) else: - _push_ns_attr((u'xmlns', tagns)) + _push_ns_attr(('xmlns', tagns)) _push_ns('', tagns) new_attrs = [] @@ -694,7 +698,7 @@ else: prefix = namespaces[attrns][-1] if prefix: - attrname = u'%s:%s' % (prefix, attrname) + attrname = '%s:%s' % (prefix, attrname) new_attrs.append((attrname, value)) yield _emit(kind, data, (tagname, Attrs(ns_attrs + new_attrs)), pos) @@ -706,7 +710,7 @@ if tagns: prefix = namespaces[tagns][-1] if prefix: - tagname = u'%s:%s' % (prefix, tagname) + tagname = '%s:%s' % (prefix, tagname) yield _emit(kind, data, tagname, pos) elif kind is START_NS: diff --git a/genshi/path.py b/genshi/path.py --- a/genshi/path.py +++ b/genshi/path.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -30,8 +30,8 @@ ... ... ... ''') ->>> print doc.select('items/item[@status="closed" and ' -... '(@resolution="invalid" or not(@resolution))]/summary/text()') +>>> print(doc.select('items/item[@status="closed" and ' +... '(@resolution="invalid" or not(@resolution))]/summary/text()')) BarBaz Because the XPath engine operates on markup streams (as opposed to tree @@ -40,9 +40,9 @@ from collections import deque try: + reduce # builtin in Python < 3 +except NameError: from functools import reduce -except ImportError: - pass # builtin in Python <= 2.5 from math import ceil, floor import operator import re @@ -263,9 +263,9 @@ def nodes_equal(node1, node2): """Tests if two node tests are equal""" - if node1.__class__ is not node2.__class__: + if type(node1) is not type(node2): return False - if node1.__class__ == LocalNameTest: + if type(node1) == LocalNameTest: return node1.name == node2.name return True @@ -278,7 +278,7 @@ return [] pi = [0] s = 0 - for i in xrange(1, len(f)): + for i in range(1, len(f)): while s > 0 and not nodes_equal(f[s], f[i]): s = pi[s-1] if nodes_equal(f[s], f[i]): @@ -537,7 +537,7 @@ self.strategies.append(strategy_class(path)) break else: - raise NotImplemented, "This path is not implemented" + raise NotImplemented('No strategy found for path') def __repr__(self): paths = [] @@ -548,7 +548,7 @@ for predicate in predicates: steps[-1] += '[%s]' % predicate paths.append('/'.join(steps)) - return '<%s "%s">' % (self.__class__.__name__, '|'.join(paths)) + return '<%s "%s">' % (type(self).__name__, '|'.join(paths)) def select(self, stream, namespaces=None, variables=None): """Returns a substream of the given stream that matches the path. @@ -558,10 +558,10 @@ >>> from genshi.input import XML >>> xml = XML('Text') - >>> print Path('.//child').select(xml) + >>> print(Path('.//child').select(xml)) Text - >>> print Path('.//child/text()').select(xml) + >>> print(Path('.//child/text()').select(xml)) Text :param stream: the stream to select from @@ -618,8 +618,8 @@ >>> namespaces, variables = {}, {} >>> for event in xml: ... if test(event, namespaces, variables): - ... print event[0], repr(event[1]) - START (QName(u'child'), Attrs([(QName(u'id'), u'2')])) + ... print('%s %r' % (event[0], event[1])) + START (QName('child'), Attrs([(QName('id'), u'2')])) :param ignore_context: if `True`, the path is interpreted like a pattern in XSLT, meaning for example that it will match @@ -667,9 +667,9 @@ def __init__(self, text, filename=None, lineno=-1): self.filename = filename self.lineno = lineno - self.tokens = filter(None, [dqstr or sqstr or number or token or name - for dqstr, sqstr, number, token, name in - self._tokenize(text)]) + self.tokens = [t for t in [dqstr or sqstr or number or token or name + for dqstr, sqstr, number, token, name in + self._tokenize(text)] if t] self.pos = 0 # Tokenizer @@ -937,7 +937,7 @@ def as_string(value): value = as_scalar(value) if value is False: - return u'' + return '' return unicode(value) def as_bool(value): @@ -1101,7 +1101,7 @@ for item in [expr(kind, data, pos, namespaces, variables) for expr in self.exprs]: strings.append(as_string(item)) - return u''.join(strings) + return ''.join(strings) def __repr__(self): return 'concat(%s)' % ', '.join([repr(expr) for expr in self.exprs]) @@ -1311,7 +1311,7 @@ index = string1.find(string2) if index >= 0: return string1[index + len(string2):] - return u'' + return '' def __repr__(self): return 'substring-after(%r, %r)' % (self.string1, self.string2) @@ -1329,7 +1329,7 @@ index = string1.find(string2) if index >= 0: return string1[:index] - return u'' + return '' def __repr__(self): return 'substring-after(%r, %r)' % (self.string1, self.string2) diff --git a/genshi/template/ast24.py b/genshi/template/ast24.py --- a/genshi/template/ast24.py +++ b/genshi/template/ast24.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2008 Edgewall Software +# Copyright (C) 2008-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -27,7 +27,7 @@ if ret._fields: for attr, value in zip(ret._fields, args): if attr in kwargs: - raise ValueError, 'Field set both in args and kwargs' + raise ValueError('Field set both in args and kwargs') setattr(ret, attr, value) for attr in kwargs: if (getattr(ret, '_fields', None) and attr in ret._fields) \ @@ -101,7 +101,7 @@ return self._new(_ast.Tuple, elts, _ast.Store()) else: raise NotImplemented - + args = [] for arg in tab: if isinstance(arg, str): @@ -111,7 +111,7 @@ else: assert False, node.__class__ - defaults = map(self.visit, node.defaults) + defaults = [self.visit(d) for d in node.defaults] return self._new(_ast.arguments, args, vararg, kwarg, defaults) @@ -378,7 +378,7 @@ def visit_Const(self, node): if node.value is None: # appears in slices return None - elif isinstance(node.value, (str, unicode,)): + elif isinstance(node.value, basestring): return self._new(_ast.Str, node.value) else: return self._new(_ast.Num, node.value) @@ -471,8 +471,7 @@ return self._new(_ast.Subscript, self.visit(node.expr), slice, ctx) def visit_Sliceobj(self, node): - a = node.nodes + [None]*(3 - len(node.nodes)) - a = map(self.visit, a) + a = [self.visit(n) for n in node.nodes + [None]*(3 - len(node.nodes))] return self._new(_ast.Slice, a[0], a[1], a[2]) def visit_Ellipsis(self, node): @@ -497,8 +496,8 @@ return False else: return True - statements = [_check_del(self.visit(n)) for n in node.nodes] - return filter(_keep, statements) + return [s for s in [_check_del(self.visit(n)) for n in node.nodes] + if _keep(s)] def parse(source, mode): diff --git a/genshi/template/astgae.py b/genshi/template/astgae.py deleted file mode 100644 --- a/genshi/template/astgae.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2008 Edgewall Software -# All rights reserved. -# -# This software is licensed as described in the file COPYING, which -# you should have received as part of this distribution. The terms -# are also available at http://genshi.edgewall.org/wiki/License. -# -# This software consists of voluntary contributions made by many -# individuals. For the exact contribution history, see the revision -# history and logs, available at http://genshi.edgewall.org/log/. - -"""Support for using the Python AST on Google App Engine.""" - -__all__ = ['restore'] -__docformat__ = 'restructuredtext en' - - -def restore(_ast): - """Gross hack to restore the required classes to the _ast module if it - appears to be missing them. Mostly lifted from Mako. - """ - _ast.PyCF_ONLY_AST = 2 << 9 - - e = compile('True', '', 'eval', _ast.PyCF_ONLY_AST) - _ast.Expression = type(e) - for cls in _ast.Expression.__mro__: - if cls.__name__ == 'AST': - _ast.AST = cls - - m = compile("""\ -foo() -bar = 'fish' -baz += bar -1 + 2 - 3 * 4 / 5 ** 6 -6 // 7 % 8 << 9 >> 10 -11 & 12 ^ 13 | 14 -15 and 16 or 17 --baz + (not +18) - ~17 -baz and 'foo' or 'bar' -(fish is baz == baz) is not baz != fish -fish > baz < fish >= baz <= fish -fish in baz not in (1, 2, 3) -baz[1, 1:2, ...] -""", '', 'exec', _ast.PyCF_ONLY_AST) - - _ast.Module = type(m) - - _ast.Expr = type(m.body[0]) - _ast.Call = type(m.body[0].value) - - _ast.Assign = type(m.body[1]) - _ast.Name = type(m.body[1].targets[0]) - _ast.Store = type(m.body[1].targets[0].ctx) - _ast.Str = type(m.body[1].value) - - _ast.AugAssign = type(m.body[2]) - _ast.Load = type(m.body[2].value.ctx) - - _ast.Sub = type(m.body[3].value.op) - _ast.Add = type(m.body[3].value.left.op) - _ast.Div = type(m.body[3].value.right.op) - _ast.Mult = type(m.body[3].value.right.left.op) - _ast.Pow = type(m.body[3].value.right.right.op) - - _ast.RShift = type(m.body[4].value.op) - _ast.LShift = type(m.body[4].value.left.op) - _ast.Mod = type(m.body[4].value.left.left.op) - _ast.FloorDiv = type(m.body[4].value.left.left.left.op) - - _ast.BitOr = type(m.body[5].value.op) - _ast.BitXor = type(m.body[5].value.left.op) - _ast.BitAnd = type(m.body[5].value.left.left.op) - - _ast.Or = type(m.body[6].value.op) - _ast.And = type(m.body[6].value.values[0].op) - - _ast.Invert = type(m.body[7].value.right.op) - _ast.Not = type(m.body[7].value.left.right.op) - _ast.UAdd = type(m.body[7].value.left.right.operand.op) - _ast.USub = type(m.body[7].value.left.left.op) - - _ast.Or = type(m.body[8].value.op) - _ast.And = type(m.body[8].value.values[0].op) - - _ast.IsNot = type(m.body[9].value.ops[0]) - _ast.NotEq = type(m.body[9].value.ops[1]) - _ast.Is = type(m.body[9].value.left.ops[0]) - _ast.Eq = type(m.body[9].value.left.ops[1]) - - _ast.Gt = type(m.body[10].value.ops[0]) - _ast.Lt = type(m.body[10].value.ops[1]) - _ast.GtE = type(m.body[10].value.ops[2]) - _ast.LtE = type(m.body[10].value.ops[3]) - - _ast.In = type(m.body[11].value.ops[0]) - _ast.NotIn = type(m.body[11].value.ops[1]) - _ast.Tuple = type(m.body[11].value.comparators[1]) - - _ast.ExtSlice = type(m.body[12].value.slice) - _ast.Index = type(m.body[12].value.slice.dims[0]) - _ast.Slice = type(m.body[12].value.slice.dims[1]) - _ast.Ellipsis = type(m.body[12].value.slice.dims[2]) diff --git a/genshi/template/astutil.py b/genshi/template/astutil.py --- a/genshi/template/astutil.py +++ b/genshi/template/astutil.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2008 Edgewall Software +# Copyright (C) 2008-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -18,9 +18,6 @@ except ImportError: from genshi.template.ast24 import _ast, parse else: - if not hasattr(_ast, 'AST'): - from genshi.template.astgae import restore - restore(_ast) def parse(source, mode): return compile(source, '', mode, _ast.PyCF_ONLY_AST) @@ -133,9 +130,14 @@ self._write('**' + node.kwarg) # FunctionDef(identifier name, arguments args, - # stmt* body, expr* decorators) + # stmt* body, expr* decorator_list) def visit_FunctionDef(self, node): - for decorator in getattr(node, 'decorators', ()): + decarators = () + if hasattr(node, 'decorator_list'): + decorators = getattr(node, 'decorator_list') + else: # different name in earlier Python versions + decorators = getattr(node, 'decorators', ()) + for decorator in decorators: self._new_line() self._write('@') self.visit(decorator) @@ -654,7 +656,7 @@ self._write(', ') self.visit(dim) else: - raise NotImplemented, 'Slice type not implemented' + raise NotImplemented('Slice type not implemented') _process_slice(node.slice) self._write(']') @@ -740,6 +742,7 @@ visit_TryExcept = _clone visit_TryFinally = _clone visit_Assert = _clone + visit_ExceptHandler = _clone visit_Import = _clone visit_ImportFrom = _clone diff --git a/genshi/template/base.py b/genshi/template/base.py --- a/genshi/template/base.py +++ b/genshi/template/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -318,20 +318,10 @@ __metaclass__ = DirectiveFactoryMeta directives = [] - """A list of `(name, cls)` tuples that define the set of directives + """A list of ``(name, cls)`` tuples that define the set of directives provided by this factory. """ - def compare_directives(self): - """Return a function that takes two directive classes and compares - them to determine their relative ordering. - """ - def _get_index(cls): - if cls in self._dir_order: - return self._dir_order.index(cls) - return 0 - return lambda a, b: cmp(_get_index(a[0]), _get_index(b[0])) - def get_directive(self, name): """Return the directive class for the given name. @@ -341,6 +331,20 @@ """ return self._dir_by_name.get(name) + def get_directive_index(self, dir_cls): + """Return a key for the given directive class that should be used to + sort it among other directives on the same `SUB` event. + + The default implementation simply returns the index of the directive in + the `directives` list. + + :param dir_cls: the directive class + :return: the sort key + """ + if dir_cls in self._dir_order: + return self._dir_order.index(dir_cls) + return len(self._dir_order) + class Template(DirectiveFactory): """Abstract template base class. @@ -392,6 +396,7 @@ self.lookup = lookup self.allow_exec = allow_exec self._init_filters() + self._init_loader() self._module = None self._prepared = False @@ -414,12 +419,24 @@ self._init_filters() def __repr__(self): - return '<%s "%s">' % (self.__class__.__name__, self.filename) + return '<%s "%s">' % (type(self).__name__, self.filename) def _init_filters(self): - self.filters = [self._flatten] - if self.loader: - self.filters.append(self._include) + self.filters = [self._flatten, self._include] + + def _init_loader(self): + if self.loader is None: + from genshi.template.loader import TemplateLoader + if self.filename: + if self.filepath != self.filename: + basedir = os.path.normpath(self.filepath)[:-len( + os.path.normpath(self.filename)) + ] + else: + basedir = os.path.dirname(self.filename) + else: + basedir = '.' + self.loader = TemplateLoader([os.path.abspath(basedir)]) @property def stream(self): @@ -453,7 +470,7 @@ if kind is SUB: directives = [] substream = data[1] - for cls, value, namespaces, pos in data[0]: + for _, cls, value, namespaces, pos in sorted(data[0]): directive, substream = cls.attach(self, substream, value, namespaces, pos) if directive: @@ -542,55 +559,65 @@ def _flatten(self, stream, ctxt, **vars): number_conv = self._number_conv - - for kind, data, pos in stream: - - if kind is START and data[1]: - # Attributes may still contain expressions in start tags at - # this point, so do some evaluation - tag, attrs = data - new_attrs = [] - for name, value in attrs: - if type(value) is list: # this is an interpolated string - values = [event[1] - for event in self._flatten(value, ctxt, **vars) - if event[0] is TEXT and event[1] is not None - ] - if not values: - continue - value = u''.join(values) - new_attrs.append((name, value)) - yield kind, (tag, Attrs(new_attrs)), pos + stack = [] + push = stack.append + pop = stack.pop + stream = iter(stream) - elif kind is EXPR: - result = _eval_expr(data, ctxt, vars) - if result is not None: - # First check for a string, otherwise the iterable test - # below succeeds, and the string will be chopped up into - # individual characters - if isinstance(result, basestring): - yield TEXT, result, pos - elif isinstance(result, (int, float, long)): - yield TEXT, number_conv(result), pos - elif hasattr(result, '__iter__'): - for event in self._flatten(_ensure(result), ctxt, - **vars): - yield event - else: - yield TEXT, unicode(result), pos + while 1: + for kind, data, pos in stream: - elif kind is EXEC: - _exec_suite(data, ctxt, vars) + if kind is START and data[1]: + # Attributes may still contain expressions in start tags at + # this point, so do some evaluation + tag, attrs = data + new_attrs = [] + for name, value in attrs: + if type(value) is list: # this is an interpolated string + values = [event[1] + for event in self._flatten(value, ctxt, **vars) + if event[0] is TEXT and event[1] is not None + ] + if not values: + continue + value = ''.join(values) + new_attrs.append((name, value)) + yield kind, (tag, Attrs(new_attrs)), pos - elif kind is SUB: - # This event is a list of directives and a list of nested - # events to which those directives should be applied - substream = _apply_directives(data[1], data[0], ctxt, vars) - for event in self._flatten(substream, ctxt, **vars): - yield event + elif kind is EXPR: + result = _eval_expr(data, ctxt, vars) + if result is not None: + # First check for a string, otherwise the iterable test + # below succeeds, and the string will be chopped up into + # individual characters + if isinstance(result, basestring): + yield TEXT, result, pos + elif isinstance(result, (int, float, long)): + yield TEXT, number_conv(result), pos + elif hasattr(result, '__iter__'): + push(stream) + stream = _ensure(result) + break + else: + yield TEXT, unicode(result), pos + + elif kind is SUB: + # This event is a list of directives and a list of nested + # events to which those directives should be applied + push(stream) + stream = _apply_directives(data[1], data[0], ctxt, vars) + break + + elif kind is EXEC: + _exec_suite(data, ctxt, vars) + + else: + yield kind, data, pos else: - yield kind, data, pos + if not stack: + break + stream = pop() def _include(self, stream, ctxt, **vars): """Internal stream filter that performs inclusion of external @@ -607,7 +634,7 @@ **vars): if subkind is TEXT: parts.append(subdata) - href = u''.join([x for x in parts if x is not None]) + href = ''.join([x for x in parts if x is not None]) try: tmpl = self.loader.load(href, relative_to=event[2][0], cls=cls or self.__class__) diff --git a/genshi/template/directives.py b/genshi/template/directives.py --- a/genshi/template/directives.py +++ b/genshi/template/directives.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -98,7 +98,7 @@ expr = '' if getattr(self, 'expr', None) is not None: expr = ' "%s"' % self.expr.source - return '<%s%s>' % (self.__class__.__name__, expr) + return '<%s%s>' % (type(self).__name__, expr) @classmethod def _parse_expr(cls, expr, template, lineno=-1, offset=-1): @@ -144,11 +144,11 @@ >>> tmpl = MarkupTemplate('''
...
Bar
...
''') - >>> print tmpl.generate(foo={'class': 'collapse'}) + >>> print(tmpl.generate(foo={'class': 'collapse'}))

Bar

- >>> print tmpl.generate(foo=[('class', 'collapse')]) + >>> print(tmpl.generate(foo=[('class', 'collapse')]))

Bar

@@ -156,7 +156,7 @@ If the value evaluates to ``None`` (or any other non-truth value), no attributes are added: - >>> print tmpl.generate(foo=None) + >>> print(tmpl.generate(foo=None))

Bar

@@ -195,7 +195,7 @@ >>> tmpl = MarkupTemplate('''
...
Hello
...
''') - >>> print tmpl.generate(bar='Bye') + >>> print(tmpl.generate(bar='Bye'))

Bye

@@ -230,7 +230,7 @@ ...
... ${echo('Hi', name='you')} ...
''') - >>> print tmpl.generate(bar='Bye') + >>> print(tmpl.generate(bar='Bye'))

Hi, you! @@ -246,7 +246,7 @@ ...
... ${helloworld()} ...
''') - >>> print tmpl.generate(bar='Bye') + >>> print(tmpl.generate(bar='Bye'))

Hello, world! @@ -321,7 +321,7 @@ return [] def __repr__(self): - return '<%s "%s">' % (self.__class__.__name__, self.name) + return '<%s "%s">' % (type(self).__name__, self.name) class ForDirective(Directive): @@ -332,7 +332,7 @@ >>> tmpl = MarkupTemplate('''
...
${item}
...
''') - >>> print tmpl.generate(items=[1, 2, 3]) + >>> print(tmpl.generate(items=[1, 2, 3]))

1
2
3

@@ -373,7 +373,7 @@ ctxt.pop() def __repr__(self): - return '<%s>' % self.__class__.__name__ + return '<%s>' % type(self).__name__ class IfDirective(Directive): @@ -384,7 +384,7 @@ >>> tmpl = MarkupTemplate('''
... ${bar} ...
''') - >>> print tmpl.generate(foo=True, bar='Hello') + >>> print(tmpl.generate(foo=True, bar='Hello'))
Hello
@@ -415,7 +415,7 @@ ... ... ...
''') - >>> print tmpl.generate() + >>> print(tmpl.generate())
Hello Dude @@ -452,7 +452,7 @@ return [] def __repr__(self): - return '<%s "%s">' % (self.__class__.__name__, self.path.source) + return '<%s "%s">' % (type(self).__name__, self.path.source) class ReplaceDirective(Directive): @@ -465,7 +465,7 @@ >>> tmpl = MarkupTemplate('''
... Hello ...
''') - >>> print tmpl.generate(bar='Bye') + >>> print(tmpl.generate(bar='Bye'))
Bye
@@ -476,7 +476,7 @@ >>> tmpl = MarkupTemplate('''
... Hello ...
''') - >>> print tmpl.generate(bar='Bye') + >>> print(tmpl.generate(bar='Bye'))
Bye
@@ -504,7 +504,7 @@ >>> tmpl = MarkupTemplate('''
...
foo
...
''') - >>> print tmpl.generate() + >>> print(tmpl.generate())
foo
@@ -520,7 +520,7 @@ ...
... ${echo('foo')} ...
''') - >>> print tmpl.generate() + >>> print(tmpl.generate())
foo
@@ -529,7 +529,7 @@ def __call__(self, stream, directives, ctxt, **vars): def _generate(): - if _eval_expr(self.expr, ctxt, vars): + if not self.expr or _eval_expr(self.expr, ctxt, vars): stream.next() # skip start tag previous = stream.next() for event in stream: @@ -540,13 +540,6 @@ yield event return _apply_directives(_generate(), directives, ctxt, vars) - @classmethod - def attach(cls, template, stream, value, namespaces, pos): - if not value: - return None, stream[1:-1] - return super(StripDirective, cls).attach(template, stream, value, - namespaces, pos) - class ChooseDirective(Directive): """Implementation of the ``py:choose`` directive for conditionally selecting @@ -564,7 +557,7 @@ ... 1 ... 2 ...
''') - >>> print tmpl.generate() + >>> print(tmpl.generate())
1
@@ -578,7 +571,7 @@ ... 1 ... 2 ...
''') - >>> print tmpl.generate() + >>> print(tmpl.generate())
2
@@ -685,7 +678,7 @@ >>> tmpl = MarkupTemplate('''
... $x $y $z ...
''') - >>> print tmpl.generate(x=42) + >>> print(tmpl.generate(x=42))
42 7 52
@@ -731,4 +724,4 @@ ctxt.pop() def __repr__(self): - return '<%s>' % (self.__class__.__name__) + return '<%s>' % (type(self).__name__) diff --git a/genshi/template/eval.py b/genshi/template/eval.py --- a/genshi/template/eval.py +++ b/genshi/template/eval.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -120,7 +120,7 @@ return not self == other def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, self.source) + return '%s(%r)' % (type(self).__name__, self.source) class Expression(Code): @@ -229,7 +229,7 @@ False >>> list(foo) [] - >>> print foo + >>> print(foo) undefined However, calling an undefined variable, or trying to access an attribute @@ -266,7 +266,7 @@ return False def __repr__(self): - return '<%s %r>' % (self.__class__.__name__, self._name) + return '<%s %r>' % (type(self).__name__, self._name) def __str__(self): return 'undefined' @@ -277,6 +277,10 @@ raise UndefinedError(self._name, self._owner) __call__ = __getattr__ = __getitem__ = _die + # Hack around some behavior introduced in Python 2.6.2 + # http://genshi.edgewall.org/ticket/324 + __length_hint__ = None + class LookupBase(object): """Abstract base class for variable lookup implementations.""" @@ -495,7 +499,7 @@ names.add(node.asname or node.name) elif isinstance(node, _ast.Tuple): for elt in node.elts: - _process(node) + _process(elt) if hasattr(node, 'args'): for arg in node.args: _process(arg) @@ -558,15 +562,13 @@ # GeneratorExp(expr elt, comprehension* generators) def visit_GeneratorExp(self, node): gens = [] - # need to visit them in inverse order - for generator in node.generators[::-1]: + for generator in node.generators: # comprehension = (expr target, expr iter, expr* ifs) self.locals.append(set()) gen = _new(_ast.comprehension, self.visit(generator.target), - self.visit(generator.iter), - [self.visit(if_) for if_ in generator.ifs]) + self.visit(generator.iter), + [self.visit(if_) for if_ in generator.ifs]) gens.append(gen) - gens.reverse() # use node.__class__ to make it reusable as ListComp ret = _new(node.__class__, self.visit(node.elt), gens) diff --git a/genshi/template/inline.py b/genshi/template/inline.py --- a/genshi/template/inline.py +++ b/genshi/template/inline.py @@ -181,7 +181,7 @@ w.unshift() else: - raise NotImplementedError + raise NotImplementedError, '%r not supported' % d.tagname yield w() @@ -287,7 +287,7 @@ return x*x ?> -
+
Hello, $name!
${sayhi()} diff --git a/genshi/template/interpolation.py b/genshi/template/interpolation.py --- a/genshi/template/interpolation.py +++ b/genshi/template/interpolation.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007-2008 Edgewall Software +# Copyright (C) 2007-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -36,6 +36,7 @@ PseudoToken )) + def interpolate(text, filepath=None, lineno=-1, offset=0, lookup='strict'): """Parse the given string and extract expressions. @@ -44,10 +45,10 @@ string. >>> for kind, data, pos in interpolate("hey ${foo}bar"): - ... print kind, repr(data) - TEXT u'hey ' + ... print('%s %r' % (kind, data)) + TEXT 'hey ' EXPR Expression('foo') - TEXT u'bar' + TEXT 'bar' :param text: the text to parse :param filepath: absolute path to the file in which the text was found @@ -68,7 +69,7 @@ for is_expr, chunk in chain(lex(text, pos, filepath), [(True, '')]): if is_expr: if textbuf: - yield TEXT, u''.join(textbuf), textpos + yield TEXT, ''.join(textbuf), textpos del textbuf[:] textpos = None if chunk: @@ -91,6 +92,7 @@ else: pos[2] += len(chunk) + def lex(text, textpos, filepath): offset = pos = 0 end = len(text) diff --git a/genshi/template/loader.py b/genshi/template/loader.py --- a/genshi/template/loader.py +++ b/genshi/template/loader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -131,6 +131,15 @@ self._uptodate = {} self._lock = threading.RLock() + def __getstate__(self): + state = self.__dict__.copy() + state['_lock'] = None + return state + + def __setstate__(self, state): + self.__dict__ = state + self._lock = threading.RLock() + def load(self, filename, relative_to=None, cls=None, encoding=None): """Load the template with the given name. @@ -308,9 +317,9 @@ ... app1 = lambda filename: ('app1', filename, None, None), ... app2 = lambda filename: ('app2', filename, None, None) ... ) - >>> print load('app1/foo.html') + >>> print(load('app1/foo.html')) ('app1', 'app1/foo.html', None, None) - >>> print load('app2/bar.html') + >>> print(load('app2/bar.html')) ('app2', 'app2/bar.html', None, None) :param delegates: mapping of path prefixes to loader functions @@ -326,7 +335,7 @@ filename[len(prefix):].lstrip('/\\') ) return filepath, filename, fileobj, uptodate - raise TemplateNotFound(filename, delegates.keys()) + raise TemplateNotFound(filename, list(delegates.keys())) return _dispatch_by_prefix diff --git a/genshi/template/markup.py b/genshi/template/markup.py --- a/genshi/template/markup.py +++ b/genshi/template/markup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -36,7 +36,7 @@ >>> tmpl = MarkupTemplate('''
...
${item}
...
''') - >>> print tmpl.generate(items=[1, 2, 3]) + >>> print(tmpl.generate(items=[1, 2, 3]))

1
2
3

@@ -70,11 +70,8 @@ def _init_filters(self): Template._init_filters(self) # Make sure the include filter comes after the match filter - if self.loader: - self.filters.remove(self._include) - self.filters += [self._match] - if self.loader: - self.filters.append(self._include) + self.filters.remove(self._include) + self.filters += [self._match, self._include] def _parse(self, source, encoding): if not isinstance(source, Stream): @@ -131,7 +128,8 @@ self.filepath, pos[1]) args = dict([(name.localname, value) for name, value in attrs if not name.namespace]) - directives.append((cls, args, ns_prefix.copy(), pos)) + directives.append((factory.get_directive_index(cls), cls, + args, ns_prefix.copy(), pos)) strip = True new_attrs = [] @@ -143,14 +141,14 @@ self.filepath, pos[1]) if type(value) is list and len(value) == 1: value = value[0][1] - directives.append((cls, value, ns_prefix.copy(), - pos)) + directives.append((factory.get_directive_index(cls), + cls, value, ns_prefix.copy(), pos)) else: new_attrs.append((name, value)) new_attrs = Attrs(new_attrs) if directives: - directives.sort(self.compare_directives()) + directives.sort() dirmap[(depth, tag)] = (directives, len(new_stream), strip) @@ -245,7 +243,7 @@ cls = { 'xml': MarkupTemplate, 'text': NewTextTemplate - }[parse or 'xml'] + }.get(parse) or self.__class__ except KeyError: raise TemplateSyntaxError('Invalid value for "parse" ' 'attribute of include', @@ -311,10 +309,11 @@ match_templates = ctxt._match_templates tail = [] - def _strip(stream): + def _strip(stream, append=tail.append): depth = 1 + next = stream.next while 1: - event = stream.next() + event = next() if event[0] is START: depth += 1 elif event[0] is END: @@ -322,7 +321,7 @@ if depth > 0: yield event else: - tail[:] = [event] + append(event) break for event in stream: @@ -356,17 +355,19 @@ pre_end -= 1 inner = _strip(stream) if pre_end > 0: - inner = self._match(inner, ctxt, end=pre_end, **vars) + inner = self._match(inner, ctxt, start=start, + end=pre_end, **vars) content = self._include(chain([event], inner, tail), ctxt) if 'not_buffered' not in hints: content = list(content) + content = Stream(content) # Make the select() function available in the body of the # match template selected = [False] def select(path): selected[0] = True - return Stream(content).select(path, namespaces, ctxt) + return content.select(path, namespaces, ctxt) vars = dict(select=select) # Recursively process the output @@ -387,7 +388,7 @@ # Let the remaining match templates know about the last # event in the matched content, so they can update their # internal state accordingly - for test in [mt[0] for mt in match_templates]: + for test in [mt[0] for mt in match_templates[idx + 1:]]: test(tail[0], namespaces, ctxt, updateonly=True) break diff --git a/genshi/template/plugin.py b/genshi/template/plugin.py --- a/genshi/template/plugin.py +++ b/genshi/template/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # Copyright (C) 2006 Matthew Good # All rights reserved. # @@ -48,7 +48,8 @@ auto_reload = options.get('genshi.auto_reload', '1') if isinstance(auto_reload, basestring): auto_reload = auto_reload.lower() in ('1', 'on', 'yes', 'true') - search_path = filter(None, options.get('genshi.search_path', '').split(':')) + search_path = [p for p in + options.get('genshi.search_path', '').split(':') if p] self.use_package_naming = not search_path try: max_cache_size = int(options.get('genshi.max_cache_size', 25)) @@ -71,7 +72,7 @@ raise ConfigurationError('Invalid value for allow_exec "%s"' % options.get('genshi.allow_exec')) - self.loader = TemplateLoader(filter(None, search_path), + self.loader = TemplateLoader([p for p in search_path if p], auto_reload=auto_reload, max_cache_size=max_cache_size, default_class=self.template_class, diff --git a/genshi/template/tests/directives.py b/genshi/template/tests/directives.py --- a/genshi/template/tests/directives.py +++ b/genshi/template/tests/directives.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -12,6 +12,7 @@ # history and logs, available at http://genshi.edgewall.org/log/. import doctest +import re import sys import unittest @@ -32,7 +33,7 @@ items = [{'id': 1, 'class': 'foo'}, {'id': 2, 'class': 'bar'}] self.assertEqual(""" - """, str(tmpl.generate(items=items))) + """, tmpl.generate(items=items).render(encoding=None)) def test_update_existing_attr(self): """ @@ -44,7 +45,7 @@ """) self.assertEqual(""" - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_remove_existing_attr(self): """ @@ -56,7 +57,7 @@ """) self.assertEqual(""" - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) class ChooseDirectiveTestCase(unittest.TestCase): @@ -75,7 +76,7 @@
""") self.assertEqual("""
1 -
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) def test_otherwise(self): tmpl = MarkupTemplate("""
@@ -84,7 +85,7 @@
""") self.assertEqual("""
hello -
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) def test_nesting(self): """ @@ -104,7 +105,7 @@ 3

- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_complex_nesting(self): """ @@ -124,7 +125,7 @@ OK

- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_complex_nesting_otherwise(self): """ @@ -144,7 +145,7 @@ OK

- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_when_with_strip(self): """ @@ -158,7 +159,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_when_outside_choose(self): """ @@ -205,7 +206,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate(foo='Yeah'))) + """, tmpl.generate(foo='Yeah').render(encoding=None)) def test_otherwise_without_test(self): """ @@ -219,7 +220,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_as_element(self): """ @@ -234,7 +235,7 @@ """) self.assertEqual(""" 1 - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_in_text_template(self): """ @@ -251,7 +252,8 @@ 3 #end #end""") - self.assertEqual(""" 1\n""", str(tmpl.generate())) + self.assertEqual(""" 1\n""", + tmpl.generate().render(encoding=None)) class DefDirectiveTestCase(unittest.TestCase): @@ -270,7 +272,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_exec_in_replace(self): tmpl = MarkupTemplate("""
@@ -283,7 +285,7 @@
hello, world!
-
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) def test_as_element(self): """ @@ -297,7 +299,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_nested_defs(self): """ @@ -315,7 +317,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate(semantic=True))) + """, tmpl.generate(semantic=True).render(encoding=None)) def test_function_with_default_arg(self): """ @@ -327,7 +329,7 @@ """) self.assertEqual(""" foo - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_invocation_in_attribute(self): tmpl = MarkupTemplate(""" @@ -336,7 +338,7 @@ """) self.assertEqual("""
bar
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_invocation_in_attribute_none(self): tmpl = MarkupTemplate(""" @@ -345,7 +347,7 @@ """) self.assertEqual("""
bar
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_function_raising_typeerror(self): def badfunc(): @@ -368,7 +370,7 @@ """) self.assertEqual(""" True - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_in_text_template(self): """ @@ -383,7 +385,7 @@ self.assertEqual(""" Hi, you! - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_function_with_star_args(self): """ @@ -402,7 +404,7 @@ [1, 2] {'a': 3, 'b': 4}
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) class ForDirectiveTestCase(unittest.TestCase): @@ -424,7 +426,7 @@ 3 4 5 - """, str(tmpl.generate(items=range(1, 6)))) + """, tmpl.generate(items=range(1, 6)).render(encoding=None)) def test_as_element(self): """ @@ -441,7 +443,7 @@ 3 4 5 - """, str(tmpl.generate(items=range(1, 6)))) + """, tmpl.generate(items=range(1, 6)).render(encoding=None)) def test_multi_assignment(self): """ @@ -455,7 +457,8 @@ self.assertEqual("""
key=a, value=1

key=b, value=2
- """, str(tmpl.generate(items=dict(a=1, b=2).items()))) + """, tmpl.generate(items=dict(a=1, b=2).items()) + .render(encoding=None)) def test_nested_assignment(self): """ @@ -469,7 +472,8 @@ self.assertEqual("""
0: key=a, value=1

1: key=b, value=2
- """, str(tmpl.generate(items=enumerate(dict(a=1, b=2).items())))) + """, tmpl.generate(items=enumerate(dict(a=1, b=2).items())) + .render(encoding=None)) def test_not_iterable(self): """ @@ -528,7 +532,7 @@ """) self.assertEqual(""" Hello - """, str(tmpl.generate(foo=True, bar='Hello'))) + """, tmpl.generate(foo=True, bar='Hello').render(encoding=None)) def test_as_element(self): """ @@ -539,7 +543,7 @@ """) self.assertEqual(""" Hello - """, str(tmpl.generate(foo=True, bar='Hello'))) + """, tmpl.generate(foo=True, bar='Hello').render(encoding=None)) class MatchDirectiveTestCase(unittest.TestCase): @@ -558,7 +562,7 @@ """) self.assertEqual("""
Hey Joe
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_without_strip(self): """ @@ -575,7 +579,7 @@
Hey Joe
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_as_element(self): """ @@ -589,7 +593,7 @@ """) self.assertEqual("""
Hey Joe
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_recursive_match_1(self): """ @@ -619,7 +623,7 @@
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_recursive_match_2(self): """ @@ -645,7 +649,7 @@
Foo

- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_recursive_match_3(self): tmpl = MarkupTemplate(""" @@ -671,7 +675,7 @@
12
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_not_match_self(self): """ @@ -693,7 +697,7 @@ Hello! Goodbye! - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_select_text_in_element(self): """ @@ -719,7 +723,7 @@ Goodbye! - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_select_all_attrs(self): tmpl = MarkupTemplate(""" @@ -732,7 +736,7 @@
Hey Joe
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_select_all_attrs_empty(self): tmpl = MarkupTemplate(""" @@ -745,7 +749,7 @@
Hey Joe
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_select_all_attrs_in_body(self): tmpl = MarkupTemplate(""" @@ -758,7 +762,7 @@
Hey Joe Cool
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_def_in_match(self): tmpl = MarkupTemplate(""" @@ -768,7 +772,7 @@ """) self.assertEqual(""" True - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_match_with_xpath_variable(self): tmpl = MarkupTemplate("""
@@ -781,10 +785,10 @@ Hello Dude -
""", str(tmpl.generate(tagname='greeting'))) +
""", tmpl.generate(tagname='greeting').render(encoding=None)) self.assertEqual("""
-
""", str(tmpl.generate(tagname='sayhello'))) +
""", tmpl.generate(tagname='sayhello').render(encoding=None)) def test_content_directive_in_match(self): tmpl = MarkupTemplate(""" @@ -793,7 +797,7 @@ """) self.assertEqual("""
I said bar.
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_cascaded_matches(self): tmpl = MarkupTemplate(""" @@ -806,7 +810,7 @@ self.assertEqual(""" Welcome to Markup
Are you ready to mark up?
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_multiple_matches(self): tmpl = MarkupTemplate(""" @@ -836,7 +840,8 @@ Hello_4
- """, str(tmpl.generate(fields=fields, values=values))) + """, tmpl.generate(fields=fields, values=values) + .render(encoding=None)) def test_namespace_context(self): tmpl = MarkupTemplate("""
Foo
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_match_with_position_predicate(self): tmpl = MarkupTemplate(""" @@ -863,7 +868,7 @@
Foo

Bar
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_match_with_closure(self): tmpl = MarkupTemplate(""" @@ -878,7 +883,7 @@
Foo

Bar
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_match_without_closure(self): tmpl = MarkupTemplate(""" @@ -893,7 +898,7 @@
Foo

Bar
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_match_with_once_attribute(self): tmpl = MarkupTemplate(""" @@ -918,7 +923,7 @@
Bar
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_match_with_recursive_attribute(self): tmpl = MarkupTemplate(""" @@ -941,7 +946,64 @@
- """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) + + # See http://genshi.edgewall.org/ticket/254/ + def test_triple_match_produces_no_duplicate_items(self): + tmpl = MarkupTemplate(""" +
+
+ ${select('*')} +
+ + + ${select('*|text()')} + + + ${select('*|text()')} + + + +
+
Ticket X
+
+ + """) + output = tmpl.generate().render('xhtml', doctype='xhtml') + matches = re.findall("tabbed_pane", output) + self.assertNotEqual(None, matches) + self.assertEqual(1, len(matches)) + + def test_match_multiple_times1(self): + # See http://genshi.edgewall.org/ticket/370 + tmpl = MarkupTemplate(""" + + + + + + """) + self.assertEqual(""" + + + """, tmpl.generate().render()) + + def test_match_multiple_times2(self): + # See http://genshi.edgewall.org/ticket/370 + tmpl = MarkupTemplate(""" + + + + + +
Foo
+ + """) + self.assertEqual(""" + + + + """, tmpl.generate().render()) # FIXME #def test_match_after_step(self): @@ -955,7 +1017,7 @@ # # Hello Dude # - #
""", str(tmpl.generate())) + #
""", tmpl.generate().render(encoding=None)) class ContentDirectiveTestCase(unittest.TestCase): @@ -995,7 +1057,7 @@
""", filename='test.html') self.assertEqual("""
Test -
""", str(tmpl.generate(title='Test'))) +
""", tmpl.generate(title='Test').render(encoding=None)) class StripDirectiveTestCase(unittest.TestCase): @@ -1007,7 +1069,7 @@
""") self.assertEqual("""

foo
-
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) def test_strip_empty(self): tmpl = MarkupTemplate("""
@@ -1015,7 +1077,7 @@
""") self.assertEqual("""
foo -
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) class WithDirectiveTestCase(unittest.TestCase): @@ -1031,7 +1093,7 @@ 42 84 42 -
""", str(tmpl.generate(x=42))) +
""", tmpl.generate(x=42).render(encoding=None)) def test_as_element(self): tmpl = MarkupTemplate("""
@@ -1039,7 +1101,7 @@
""") self.assertEqual("""
84 -
""", str(tmpl.generate(x=42))) +
""", tmpl.generate(x=42).render(encoding=None)) def test_multiple_vars_same_name(self): tmpl = MarkupTemplate("""
@@ -1052,7 +1114,7 @@
""") self.assertEqual("""
baz -
""", str(tmpl.generate(x=42))) +
""", tmpl.generate(x=42).render(encoding=None)) def test_multiple_vars_single_assignment(self): tmpl = MarkupTemplate("""
@@ -1060,7 +1122,7 @@
""") self.assertEqual("""
1 1 1 -
""", str(tmpl.generate(x=42))) +
""", tmpl.generate(x=42).render(encoding=None)) def test_nested_vars_single_assignment(self): tmpl = MarkupTemplate("""
@@ -1068,7 +1130,7 @@
""") self.assertEqual("""
1 2 3 -
""", str(tmpl.generate(x=42))) +
""", tmpl.generate(x=42).render(encoding=None)) def test_multiple_vars_trailing_semicolon(self): tmpl = MarkupTemplate("""
@@ -1076,7 +1138,7 @@
""") self.assertEqual("""
84 42 -
""", str(tmpl.generate(x=42))) +
""", tmpl.generate(x=42).render(encoding=None)) def test_semicolon_escape(self): tmpl = MarkupTemplate("""
@@ -1088,7 +1150,7 @@ self.assertEqual("""
here is a semicolon: ; here are two semicolons: ;; -
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) def test_ast_transformation(self): """ @@ -1104,19 +1166,19 @@ 42 -
""", str(tmpl.generate(foo={'bar': 42}))) +
""", tmpl.generate(foo={'bar': 42}).render(encoding=None)) def test_unicode_expr(self): - tmpl = MarkupTemplate("""
+ tmpl = MarkupTemplate(u"""
$weeks
""") - self.assertEqual("""
+ self.assertEqual(u"""
一二三四五六日 -
""", str(tmpl.generate())) +
""", tmpl.generate().render(encoding=None)) def test_with_empty_value(self): """ @@ -1126,7 +1188,7 @@ Text
""") self.assertEqual("""
- Text
""", str(tmpl.generate())) + Text
""", tmpl.generate().render(encoding=None)) def suite(): diff --git a/genshi/template/tests/eval.py b/genshi/template/tests/eval.py --- a/genshi/template/tests/eval.py +++ b/genshi/template/tests/eval.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -12,9 +12,11 @@ # history and logs, available at http://genshi.edgewall.org/log/. import doctest +import os import pickle from StringIO import StringIO import sys +from tempfile import mkstemp import unittest from genshi.core import Markup @@ -246,6 +248,11 @@ expr = Expression("filter(lambda x: x > 2, items)") self.assertEqual([3, 4], expr.evaluate(data)) + def test_lambda_tuple_arg(self): + data = {'items': [(1, 2), (2, 1)]} + expr = Expression("filter(lambda (x, y): x > y, items)") + self.assertEqual([(2, 1)], expr.evaluate(data)) + def test_list_comprehension(self): expr = Expression("[n for n in numbers if n < 2]") self.assertEqual([0, 1], expr.evaluate({'numbers': range(5)})) @@ -258,6 +265,14 @@ self.assertEqual([2, 3, 4, 5, 6], expr.evaluate({'numbers': range(5), 'offset': 2})) + expr = Expression("[n for group in groups for n in group]") + self.assertEqual([0, 1, 0, 1, 2], + expr.evaluate({'groups': [range(2), range(3)]})) + + expr = Expression("[(a, b) for a in x for b in y]") + self.assertEqual([('x0', 'y0'), ('x0', 'y1'), ('x1', 'y0'), ('x1', 'y1')], + expr.evaluate({'x': ['x0', 'x1'], 'y': ['y0', 'y1']})) + def test_list_comprehension_with_getattr(self): items = [{'name': 'a', 'value': 1}, {'name': 'b', 'value': 2}] expr = Expression("[i.name for i in items if i.value > 1]") @@ -280,6 +295,14 @@ self.assertEqual([2, 3, 4, 5, 6], expr.evaluate({'numbers': range(5), 'offset': 2})) + expr = Expression("list(n for group in groups for n in group)") + self.assertEqual([0, 1, 0, 1, 2], + expr.evaluate({'groups': [range(2), range(3)]})) + + expr = Expression("list((a, b) for a in x for b in y)") + self.assertEqual([('x0', 'y0'), ('x0', 'y1'), ('x1', 'y0'), ('x1', 'y1')], + expr.evaluate({'x': ['x0', 'x1'], 'y': ['y0', 'y1']})) + def test_generator_expression_with_getattr(self): items = [{'name': 'a', 'value': 1}, {'name': 'b', 'value': 2}] expr = Expression("list(i.name for i in items if i.value > 1)") @@ -570,6 +593,21 @@ suite.execute(data) self.assertEqual(['foo', 'bar'], data['x']) + def test_def_with_decorator(self): + suite = Suite(""" +def lower(fun): + return lambda: fun().lower() + +@lower +def say_hi(): + return 'Hi!' + +result = say_hi() +""") + data = {} + suite.execute(data) + self.assertEqual('hi!', data['result']) + def test_delete(self): suite = Suite("""foo = 42 del foo @@ -621,7 +659,7 @@ def test_import_in_def(self): suite = Suite("""def fun(): from itertools import ifilter - return ifilter(None, xrange(3)) + return ifilter(None, range(3)) """) data = Context() suite.execute(data) @@ -767,6 +805,46 @@ Suite("del d['k']").execute({'d': d}) self.failIf('k' in d, repr(d)) + if sys.version_info >= (2, 5): + def test_with_statement(self): + fd, path = mkstemp() + f = os.fdopen(fd, "w") + try: + f.write('foo\nbar\n') + f.seek(0) + f.close() + + d = {'path': path} + suite = Suite("""from __future__ import with_statement +lines = [] +with open(path) as file: + for line in file: + lines.append(line) +""") + suite.execute(d) + self.assertEqual(['foo\n', 'bar\n'], d['lines']) + finally: + os.remove(path) + + def test_yield_expression(self): + d = {} + suite = Suite("""results = [] +def counter(maximum): + i = 0 + while i < maximum: + val = (yield i) + if val is not None: + i = val + else: + i += 1 +it = counter(5) +results.append(it.next()) +results.append(it.send(3)) +results.append(it.next()) +""") + suite.execute(d) + self.assertEqual([0, 3, 4], d['results']) + def suite(): suite = unittest.TestSuite() diff --git a/genshi/template/tests/loader.py b/genshi/template/tests/loader.py --- a/genshi/template/tests/loader.py +++ b/genshi/template/tests/loader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -58,7 +58,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_relative_include_subdir(self): os.mkdir(os.path.join(self.dirname, 'sub')) @@ -80,7 +80,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_relative_include_parentdir(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') @@ -102,7 +102,7 @@ tmpl = loader.load('sub/tmpl2.html') self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_relative_include_samesubdir(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') @@ -130,7 +130,7 @@ tmpl = loader.load('sub/tmpl2.html') self.assertEqual("""
Included sub/tmpl1.html
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_relative_include_without_search_path(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') @@ -151,7 +151,51 @@ tmpl = loader.load(os.path.join(self.dirname, 'tmpl2.html')) self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) + + def test_relative_include_without_loader(self): + file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') + try: + file1.write("""
Included
""") + finally: + file1.close() + + file2 = open(os.path.join(self.dirname, 'tmpl2.html'), 'w') + try: + file2.write(""" + + """) + finally: + file2.close() + + tmpl = MarkupTemplate(""" + + """, os.path.join(self.dirname, 'tmpl2.html'), 'tmpl2.html') + self.assertEqual(""" +
Included
+ """, tmpl.generate().render(encoding=None)) + + def test_relative_include_without_loader_relative(self): + file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') + try: + file1.write("""
Included
""") + finally: + file1.close() + + file2 = open(os.path.join(self.dirname, 'tmpl2.html'), 'w') + try: + file2.write(""" + + """) + finally: + file2.close() + + tmpl = MarkupTemplate(""" + + """, filename=os.path.join(self.dirname, 'tmpl2.html')) + self.assertEqual(""" +
Included
+ """, tmpl.generate().render(encoding=None)) def test_relative_include_without_search_path_nested(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') @@ -182,7 +226,7 @@

Included

- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_relative_include_from_inmemory_template(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') @@ -198,7 +242,7 @@ self.assertEqual("""
Included
- """, tmpl2.generate().render()) + """, tmpl2.generate().render(encoding=None)) def test_relative_absolute_template_preferred(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') @@ -227,7 +271,7 @@ 'tmpl2.html'))) self.assertEqual("""
Included from sub
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_abspath_caching(self): abspath = os.path.join(self.dirname, 'abs') @@ -258,7 +302,7 @@ tmpl1 = loader.load(os.path.join(abspath, 'tmpl1.html')) self.assertEqual("""
Included from searchpath.
- """, tmpl1.generate().render()) + """, tmpl1.generate().render(encoding=None)) assert 'tmpl2.html' in loader._cache def test_abspath_include_caching_without_search_path(self): @@ -295,11 +339,11 @@ tmpl1 = loader.load(os.path.join(self.dirname, 'tmpl1.html')) self.assertEqual("""
Included
- """, tmpl1.generate().render()) + """, tmpl1.generate().render(encoding=None)) tmpl2 = loader.load(os.path.join(self.dirname, 'sub', 'tmpl1.html')) self.assertEqual("""
Included from sub
- """, tmpl2.generate().render()) + """, tmpl2.generate().render(encoding=None)) assert 'tmpl2.html' not in loader._cache def test_load_with_default_encoding(self): @@ -341,13 +385,13 @@ tmpl = loader.load('tmpl.html') self.assertEqual("""
Hello, hello
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) # Make sure the filter is only added once tmpl = loader.load('tmpl.html') self.assertEqual("""
Hello, hello
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_prefix_delegation_to_directories(self): """ @@ -393,7 +437,7 @@ tmpl = loader.load('sub1/tmpl1.html') self.assertEqual("""
Included foo
from sub1 - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_prefix_delegation_to_directories_with_subdirs(self): """ @@ -449,7 +493,7 @@
Included foo
from sub1
tmpl2
from sub1
bar/tmpl3
from sub1 - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def suite(): diff --git a/genshi/template/tests/markup.py b/genshi/template/tests/markup.py --- a/genshi/template/tests/markup.py +++ b/genshi/template/tests/markup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -211,23 +211,23 @@ Verify that a code block processing instruction with trailing space does not cause a syntax error (see ticket #127). """ - MarkupTemplate(u""" + MarkupTemplate(""" """) def test_exec_import(self): - tmpl = MarkupTemplate(u""" + tmpl = MarkupTemplate("""
${timedelta(days=2)}
""") - self.assertEqual(u"""
+ self.assertEqual("""
2 days, 0:00:00
""", str(tmpl.generate())) def test_exec_def(self): - tmpl = MarkupTemplate(u""" + tmpl = MarkupTemplate(""" ${foo()}
""") - self.assertEqual(u"""
+ self.assertEqual("""
42
""", str(tmpl.generate())) @@ -277,7 +277,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
Included 0
Included 1
Included 2
- """, tmpl.generate(name='tmpl1').render()) + """, tmpl.generate(name='tmpl1').render(encoding=None)) finally: shutil.rmtree(dirname) @@ -303,7 +303,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
Included
- """, tmpl.generate(name='tmpl1').render()) + """, tmpl.generate(name='tmpl1').render(encoding=None)) finally: shutil.rmtree(dirname) @@ -332,7 +332,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
1
2
3
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -358,7 +358,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -395,7 +395,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual(""" Missing - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -415,7 +415,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual(""" Missing - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -446,7 +446,7 @@ tmpl = loader.load('tmpl3.html') self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -471,7 +471,7 @@ tmpl = loader.load('tmpl3.html') self.assertEqual(""" Missing - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -500,7 +500,7 @@ tmpl = loader.load('tmpl3.html') self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -522,7 +522,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual(""" tmpl1.html not found - """, tmpl.generate(debug=True).render()) + """, tmpl.generate(debug=True).render(encoding=None)) finally: shutil.rmtree(dirname) @@ -550,7 +550,7 @@ self.assertEqual(7, len(tmpl.stream)) self.assertEqual("""
Included
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -576,7 +576,7 @@ tmpl = loader.load('tmpl2.html') self.assertEqual("""
Included 0
Included 1
Included 2
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -620,7 +620,7 @@ wakka wakka wakka - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_with_in_match(self): xml = (""" @@ -636,7 +636,7 @@
bar

bar
- """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_nested_include_matches(self): # See ticket #157 @@ -688,7 +688,7 @@
-""", tmpl.generate().render()) +""", tmpl.generate().render(encoding=None)) finally: shutil.rmtree(dirname) @@ -711,7 +711,7 @@ Foo And some other stuff... - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def test_match_without_select(self): # See @@ -730,7 +730,7 @@ This replaces the other text. - """, tmpl.generate().render()) + """, tmpl.generate().render(encoding=None)) def suite(): diff --git a/genshi/template/tests/text.py b/genshi/template/tests/text.py --- a/genshi/template/tests/text.py +++ b/genshi/template/tests/text.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -33,32 +33,35 @@ def test_escaping(self): tmpl = OldTextTemplate('\\#escaped') - self.assertEqual('#escaped', str(tmpl.generate())) + self.assertEqual('#escaped', tmpl.generate().render(encoding=None)) def test_comment(self): tmpl = OldTextTemplate('## a comment') - self.assertEqual('', str(tmpl.generate())) + self.assertEqual('', tmpl.generate().render(encoding=None)) def test_comment_escaping(self): tmpl = OldTextTemplate('\\## escaped comment') - self.assertEqual('## escaped comment', str(tmpl.generate())) + self.assertEqual('## escaped comment', + tmpl.generate().render(encoding=None)) def test_end_with_args(self): tmpl = OldTextTemplate(""" #if foo bar #end 'if foo'""") - self.assertEqual('\n', str(tmpl.generate(foo=False))) + self.assertEqual('\n', tmpl.generate(foo=False).render(encoding=None)) def test_latin1_encoded(self): text = u'$foo\xf6$bar'.encode('iso-8859-1') tmpl = OldTextTemplate(text, encoding='iso-8859-1') - self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) + self.assertEqual(u'x\xf6y', + tmpl.generate(foo='x', bar='y').render(encoding=None)) def test_unicode_input(self): text = u'$foo\xf6$bar' tmpl = OldTextTemplate(text) - self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) + self.assertEqual(u'x\xf6y', + tmpl.generate(foo='x', bar='y').render(encoding=None)) def test_empty_lines1(self): tmpl = OldTextTemplate("""Your items: @@ -71,7 +74,7 @@ * 0 * 1 * 2 -""", tmpl.generate(items=range(3)).render()) +""", tmpl.generate(items=range(3)).render(encoding=None)) def test_empty_lines2(self): tmpl = OldTextTemplate("""Your items: @@ -88,7 +91,7 @@ * 2 -""", tmpl.generate(items=range(3)).render()) +""", tmpl.generate(items=range(3)).render(encoding=None)) def test_include(self): file1 = open(os.path.join(self.dirname, 'tmpl1.txt'), 'w') @@ -110,7 +113,7 @@ self.assertEqual("""----- Included data below this line ----- Included ----- Included data above this line -----""", - tmpl.generate().render()) + tmpl.generate().render(encoding=None)) class NewTextTemplateTestCase(unittest.TestCase): @@ -124,32 +127,36 @@ def test_escaping(self): tmpl = NewTextTemplate('\\{% escaped %}') - self.assertEqual('{% escaped %}', str(tmpl.generate())) + self.assertEqual('{% escaped %}', + tmpl.generate().render(encoding=None)) def test_comment(self): tmpl = NewTextTemplate('{# a comment #}') - self.assertEqual('', str(tmpl.generate())) + self.assertEqual('', tmpl.generate().render(encoding=None)) def test_comment_escaping(self): tmpl = NewTextTemplate('\\{# escaped comment #}') - self.assertEqual('{# escaped comment #}', str(tmpl.generate())) + self.assertEqual('{# escaped comment #}', + tmpl.generate().render(encoding=None)) def test_end_with_args(self): tmpl = NewTextTemplate(""" {% if foo %} bar {% end 'if foo' %}""") - self.assertEqual('\n', str(tmpl.generate(foo=False))) + self.assertEqual('\n', tmpl.generate(foo=False).render(encoding=None)) def test_latin1_encoded(self): text = u'$foo\xf6$bar'.encode('iso-8859-1') tmpl = NewTextTemplate(text, encoding='iso-8859-1') - self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) + self.assertEqual(u'x\xf6y', + tmpl.generate(foo='x', bar='y').render(encoding=None)) def test_unicode_input(self): text = u'$foo\xf6$bar' tmpl = NewTextTemplate(text) - self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) + self.assertEqual(u'x\xf6y', + tmpl.generate(foo='x', bar='y').render(encoding=None)) def test_empty_lines1(self): tmpl = NewTextTemplate("""Your items: @@ -162,7 +169,7 @@ * 0 * 1 * 2 -""", tmpl.generate(items=range(3)).render()) +""", tmpl.generate(items=range(3)).render(encoding=None)) def test_empty_lines2(self): tmpl = NewTextTemplate("""Your items: @@ -179,37 +186,37 @@ * 2 -""", tmpl.generate(items=range(3)).render()) +""", tmpl.generate(items=range(3)).render(encoding=None)) def test_exec_with_trailing_space(self): """ Verify that a code block with trailing space does not cause a syntax error (see ticket #127). """ - NewTextTemplate(u""" + NewTextTemplate(""" {% python bar = 42 $} """) def test_exec_import(self): - tmpl = NewTextTemplate(u"""{% python from datetime import timedelta %} + tmpl = NewTextTemplate("""{% python from datetime import timedelta %} ${timedelta(days=2)} """) self.assertEqual(""" 2 days, 0:00:00 - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_exec_def(self): - tmpl = NewTextTemplate(u"""{% python + tmpl = NewTextTemplate("""{% python def foo(): return 42 %} ${foo()} """) - self.assertEqual(u""" + self.assertEqual(""" 42 - """, str(tmpl.generate())) + """, tmpl.generate().render(encoding=None)) def test_include(self): file1 = open(os.path.join(self.dirname, 'tmpl1.txt'), 'w') @@ -230,7 +237,8 @@ tmpl = loader.load('tmpl2.txt', cls=NewTextTemplate) self.assertEqual("""----- Included data below this line ----- Included ------ Included data above this line -----""", tmpl.generate().render()) +----- Included data above this line -----""", + tmpl.generate().render(encoding=None)) def test_include_expr(self): file1 = open(os.path.join(self.dirname, 'tmpl1.txt'), 'w') @@ -251,7 +259,8 @@ tmpl = loader.load('tmpl2.txt', cls=NewTextTemplate) self.assertEqual("""----- Included data below this line ----- Included - ----- Included data above this line -----""", tmpl.generate().render()) + ----- Included data above this line -----""", + tmpl.generate().render(encoding=None)) def suite(): diff --git a/genshi/template/text.py b/genshi/template/text.py --- a/genshi/template/text.py +++ b/genshi/template/text.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -61,7 +61,7 @@ ... * ${'Item %d' % item} ... {% end %} ... ''') - >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + >>> print(tmpl.generate(name='Joe', items=[1, 2, 3]).render(encoding=None)) Dear Joe, @@ -86,7 +86,7 @@ ... * $item ... {% end %}\ ... ''') - >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + >>> print(tmpl.generate(name='Joe', items=[1, 2, 3]).render(encoding=None)) Dear Joe, We have the following items for you: @@ -106,7 +106,7 @@ ... * $item ... {% end %}\ ... ''') - >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + >>> print(tmpl.generate(name='Joe', items=[1, 2, 3]).render(encoding=None)) Dear Joe, {# This is a comment #} @@ -144,10 +144,10 @@ raise ValueError('delimiers tuple must have exactly four elements') self._delims = delims self._directive_re = re.compile(self._DIRECTIVE_RE % tuple( - map(re.escape, delims) + [re.escape(d) for d in delims] ), re.DOTALL) self._escape_re = re.compile(self._ESCAPE_RE % tuple( - map(re.escape, delims[::2]) + [re.escape(d) for d in delims[::2]] )) delimiters = property(_get_delims, _set_delims, """\ The delimiters for directives and comments. This should be a four item tuple @@ -169,7 +169,7 @@ _escape_sub = self._escape_re.sub def _escape_repl(mo): - groups = filter(None, mo.groups()) + groups = [g for g in mo.groups() if g] if not groups: return '' return groups[0] @@ -219,7 +219,7 @@ cls = self.get_directive(command) if cls is None: raise BadDirectiveError(command) - directive = cls, value, None, (self.filepath, lineno, 0) + directive = 0, cls, value, None, (self.filepath, lineno, 0) dirmap[depth] = (directive, len(stream)) depth += 1 @@ -248,7 +248,7 @@ ... ... All the best, ... Foobar''') - >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + >>> print(tmpl.generate(name='Joe', items=[1, 2, 3]).render(encoding=None)) Dear Joe, We have the following items for you: @@ -315,7 +315,7 @@ cls = self.get_directive(command) if cls is None: raise BadDirectiveError(command) - directive = cls, value, None, (self.filepath, lineno, 0) + directive = 0, cls, value, None, (self.filepath, lineno, 0) dirmap[depth] = (directive, len(stream)) depth += 1 diff --git a/genshi/tests/builder.py b/genshi/tests/builder.py --- a/genshi/tests/builder.py +++ b/genshi/tests/builder.py @@ -12,7 +12,6 @@ # history and logs, available at http://genshi.edgewall.org/log/. import doctest -from HTMLParser import HTMLParseError import unittest from genshi.builder import Element, tag @@ -24,12 +23,12 @@ def test_link(self): link = tag.a(href='#', title='Foo', accesskey=None)('Bar') - bits = iter(link.generate()) + events = list(link.generate()) self.assertEqual((Stream.START, ('a', Attrs([('href', "#"), ('title', "Foo")])), - (None, -1, -1)), bits.next()) - self.assertEqual((Stream.TEXT, u'Bar', (None, -1, -1)), bits.next()) - self.assertEqual((Stream.END, 'a', (None, -1, -1)), bits.next()) + (None, -1, -1)), events[0]) + self.assertEqual((Stream.TEXT, 'Bar', (None, -1, -1)), events[1]) + self.assertEqual((Stream.END, 'a', (None, -1, -1)), events[2]) def test_nonstring_attributes(self): """ @@ -37,41 +36,40 @@ non-string type), it is coverted to a string when the stream is generated. """ - event = iter(tag.foo(id=3)).next() + events = list(tag.foo(id=3)) self.assertEqual((Stream.START, ('foo', Attrs([('id', '3')])), - (None, -1, -1)), - event) + (None, -1, -1)), events[0]) def test_duplicate_attributes(self): link = tag.a(href='#1', href_='#2')('Bar') - bits = iter(link.generate()) - self.assertEqual((Stream.START, - ('a', Attrs([('href', "#1")])), - (None, -1, -1)), bits.next()) - self.assertEqual((Stream.TEXT, u'Bar', (None, -1, -1)), bits.next()) - self.assertEqual((Stream.END, 'a', (None, -1, -1)), bits.next()) + events = list(link.generate()) + self.assertEqual((Stream.START, ('a', Attrs([('href', "#1")])), + (None, -1, -1)), events[0]) + self.assertEqual((Stream.TEXT, 'Bar', (None, -1, -1)), events[1]) + self.assertEqual((Stream.END, 'a', (None, -1, -1)), events[2]) def test_stream_as_child(self): - xml = list(tag.span(XML('Foo')).generate()) - self.assertEqual(5, len(xml)) - self.assertEqual((Stream.START, ('span', ())), xml[0][:2]) - self.assertEqual((Stream.START, ('b', ())), xml[1][:2]) - self.assertEqual((Stream.TEXT, 'Foo'), xml[2][:2]) - self.assertEqual((Stream.END, 'b'), xml[3][:2]) - self.assertEqual((Stream.END, 'span'), xml[4][:2]) + events = list(tag.span(XML('Foo')).generate()) + self.assertEqual(5, len(events)) + self.assertEqual((Stream.START, ('span', ())), events[0][:2]) + self.assertEqual((Stream.START, ('b', ())), events[1][:2]) + self.assertEqual((Stream.TEXT, 'Foo'), events[2][:2]) + self.assertEqual((Stream.END, 'b'), events[3][:2]) + self.assertEqual((Stream.END, 'span'), events[4][:2]) def test_markup_escape(self): - from genshi.core import Markup m = Markup('See %s') % tag.a('genshi', href='http://genshi.edgwall.org') self.assertEqual(m, Markup('See ' 'genshi')) + def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(Element.__module__)) suite.addTest(unittest.makeSuite(ElementFactoryTestCase, 'test')) return suite + if __name__ == '__main__': unittest.main(defaultTest='suite') diff --git a/genshi/tests/core.py b/genshi/tests/core.py --- a/genshi/tests/core.py +++ b/genshi/tests/core.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -57,7 +57,7 @@ pickle.dump(xml, buf, 2) buf.seek(0) xml = pickle.load(buf) - self.assertEquals('
Foo
', xml.render()) + self.assertEquals('
Foo
', xml.render(encoding=None)) class MarkupTestCase(unittest.TestCase): @@ -175,16 +175,32 @@ self.assertEquals("Attrs([('attr1', 'foo'), ('attr2', 'bar')])", repr(unpickled)) + def test_non_ascii(self): + attrs_tuple = Attrs([("attr1", u"föö"), ("attr2", u"bär")]).totuple() + self.assertEqual(u'fööbär', attrs_tuple[1]) + class NamespaceTestCase(unittest.TestCase): + def test_repr(self): + self.assertEqual("Namespace('http://www.example.org/namespace')", + repr(Namespace('http://www.example.org/namespace'))) + + def test_repr_eval(self): + ns = Namespace('http://www.example.org/namespace') + self.assertEqual(eval(repr(ns)), ns) + + def test_repr_eval_non_ascii(self): + ns = Namespace(u'http://www.example.org/nämespäcé') + self.assertEqual(eval(repr(ns)), ns) + def test_pickle(self): ns = Namespace('http://www.example.org/namespace') buf = StringIO() pickle.dump(ns, buf, 2) buf.seek(0) unpickled = pickle.load(buf) - self.assertEquals('', + self.assertEquals("Namespace('http://www.example.org/namespace')", repr(unpickled)) self.assertEquals('http://www.example.org/namespace', unpickled.uri) @@ -203,10 +219,18 @@ self.assertEquals('elem', unpickled.localname) def test_repr(self): - self.assertEqual("QName(u'elem')", repr(QName('elem'))) - self.assertEqual("QName(u'http://www.example.org/namespace}elem')", + self.assertEqual("QName('elem')", repr(QName('elem'))) + self.assertEqual("QName('http://www.example.org/namespace}elem')", repr(QName('http://www.example.org/namespace}elem'))) + def test_repr_eval(self): + qn = QName('elem') + self.assertEqual(eval(repr(qn)), qn) + + def test_repr_eval_non_ascii(self): + qn = QName(u'élem') + self.assertEqual(eval(repr(qn)), qn) + def test_leading_curly_brace(self): qname = QName('{http://www.example.org/namespace}elem') self.assertEquals('http://www.example.org/namespace', qname.namespace) diff --git a/genshi/tests/input.py b/genshi/tests/input.py --- a/genshi/tests/input.py +++ b/genshi/tests/input.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -27,7 +27,7 @@ events = list(XMLParser(StringIO(text))) kind, data, pos = events[1] self.assertEqual(Stream.TEXT, kind) - self.assertEqual(u'foo bar', data) + self.assertEqual('foo bar', data) self.assertEqual((None, 1, 6), pos) def test_text_node_pos_multi_line(self): @@ -36,7 +36,7 @@ events = list(XMLParser(StringIO(text))) kind, data, pos = events[1] self.assertEqual(Stream.TEXT, kind) - self.assertEqual(u'foo\nbar', data) + self.assertEqual('foo\nbar', data) self.assertEqual((None, 1, -1), pos) def test_element_attribute_order(self): @@ -45,10 +45,10 @@ kind, data, pos = events[0] self.assertEqual(Stream.START, kind) tag, attrib = data - self.assertEqual(u'elem', tag) - self.assertEqual((u'title', u'baz'), attrib[0]) - self.assertEqual((u'id', u'foo'), attrib[1]) - self.assertEqual((u'class', u'bar'), attrib[2]) + self.assertEqual('elem', tag) + self.assertEqual(('title', 'baz'), attrib[0]) + self.assertEqual(('id', 'foo'), attrib[1]) + self.assertEqual(('class', 'bar'), attrib[2]) def test_unicode_input(self): text = u'
\u2013
' @@ -120,7 +120,7 @@ events = list(HTMLParser(StringIO(text))) kind, data, pos = events[1] self.assertEqual(Stream.TEXT, kind) - self.assertEqual(u'foo bar', data) + self.assertEqual('foo bar', data) self.assertEqual((None, 1, 6), pos) def test_text_node_pos_multi_line(self): @@ -129,7 +129,7 @@ events = list(HTMLParser(StringIO(text))) kind, data, pos = events[1] self.assertEqual(Stream.TEXT, kind) - self.assertEqual(u'foo\nbar', data) + self.assertEqual('foo\nbar', data) self.assertEqual((None, 1, 6), pos) def test_input_encoding_text(self): @@ -174,15 +174,15 @@ events = list(HTMLParser(StringIO(text))) kind, (target, data), pos = events[0] self.assertEqual(Stream.PI, kind) - self.assertEqual(u'php', target) - self.assertEqual(u'echo "Foobar"', data) + self.assertEqual('php', target) + self.assertEqual('echo "Foobar"', data) def test_xmldecl(self): text = '' events = list(XMLParser(StringIO(text))) kind, (version, encoding, standalone), pos = events[0] self.assertEqual(Stream.XML_DECL, kind) - self.assertEqual(u'1.0', version) + self.assertEqual('1.0', version) self.assertEqual(None, encoding) self.assertEqual(-1, standalone) @@ -191,8 +191,8 @@ events = list(XMLParser(StringIO(text))) kind, (version, encoding, standalone), pos = events[0] self.assertEqual(Stream.XML_DECL, kind) - self.assertEqual(u'1.0', version) - self.assertEqual(u'utf-8', encoding) + self.assertEqual('1.0', version) + self.assertEqual('utf-8', encoding) self.assertEqual(-1, standalone) def test_xmldecl_standalone(self): @@ -200,7 +200,7 @@ events = list(XMLParser(StringIO(text))) kind, (version, encoding, standalone), pos = events[0] self.assertEqual(Stream.XML_DECL, kind) - self.assertEqual(u'1.0', version) + self.assertEqual('1.0', version) self.assertEqual(None, encoding) self.assertEqual(1, standalone) @@ -209,8 +209,8 @@ events = list(HTMLParser(StringIO(text))) kind, (target, data), pos = events[0] self.assertEqual(Stream.PI, kind) - self.assertEqual(u'php', target) - self.assertEqual(u'echo "Foobar" ?', data) + self.assertEqual('php', target) + self.assertEqual('echo "Foobar" ?', data) def test_out_of_order_tags1(self): text = 'Foobar' diff --git a/genshi/tests/output.py b/genshi/tests/output.py --- a/genshi/tests/output.py +++ b/genshi/tests/output.py @@ -25,7 +25,7 @@ def test_with_xml_decl(self): stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))]) - output = stream.render(XMLSerializer, doctype='xhtml') + output = stream.render(XMLSerializer, doctype='xhtml', encoding=None) self.assertEqual('\n' '\n', @@ -44,7 +44,7 @@ stream = Stream([(Stream.DOCTYPE, ('html', '-//W3C//DTD HTML 4.01//EN', None), (None, -1, -1))]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual('\n', output) @@ -54,7 +54,7 @@ ('html', None, 'http://www.w3.org/TR/html4/strict.dtd'), (None, -1, -1)) ]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual('\n', output) @@ -62,12 +62,13 @@ def test_doctype_in_stream_no_pubid_or_sysid(self): stream = Stream([(Stream.DOCTYPE, ('html', None, None), (None, -1, -1))]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual('\n', output) def test_serializer_doctype(self): stream = Stream([]) - output = stream.render(XMLSerializer, doctype=DocType.HTML_STRICT) + output = stream.render(XMLSerializer, doctype=DocType.HTML_STRICT, + encoding=None) self.assertEqual('\n', @@ -77,7 +78,8 @@ stream = Stream([ (Stream.DOCTYPE, ('html', None, None), (None, -1, -1)) ]) - output = stream.render(XMLSerializer, doctype=DocType.HTML_STRICT) + output = stream.render(XMLSerializer, doctype=DocType.HTML_STRICT, + encoding=None) self.assertEqual('\n', @@ -85,12 +87,12 @@ def test_comment(self): stream = Stream([(Stream.COMMENT, 'foo bar', (None, -1, -1))]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual('', output) def test_processing_instruction(self): stream = Stream([(Stream.PI, ('python', 'x = 2'), (None, -1, -1))]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual('', output) def test_nested_default_namespaces(self): @@ -111,7 +113,7 @@ (Stream.END, QName('http://example.org/}div'), (None, -1, -1)), (Stream.END_NS, '', (None, -1, -1)) ]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual("""

@@ -135,7 +137,7 @@ (Stream.END, QName('http://example.org/}div'), (None, -1, -1)), (Stream.END_NS, 'x', (None, -1, -1)) ]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual(""" @@ -157,7 +159,7 @@ (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.END, QName('div'), (None, -1, -1)), ]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual("""

@@ -179,7 +181,7 @@ (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.END, QName('div'), (None, -1, -1)), ]) - output = stream.render(XMLSerializer) + output = stream.render(XMLSerializer, encoding=None) self.assertEqual("""
@@ -197,7 +199,7 @@ """ - output = XML(text).render(XMLSerializer) + output = XML(text).render(XMLSerializer, encoding=None) self.assertEqual(text, output) @@ -205,7 +207,7 @@ def test_xml_decl_dropped(self): stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))]) - output = stream.render(XHTMLSerializer, doctype='xhtml') + output = stream.render(XHTMLSerializer, doctype='xhtml', encoding=None) self.assertEqual('\n', @@ -214,7 +216,7 @@ def test_xml_decl_included(self): stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))]) output = stream.render(XHTMLSerializer, doctype='xhtml', - drop_xml_decl=False) + drop_xml_decl=False, encoding=None) self.assertEqual('\n' 'English text
' - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual('
English text
', output) def test_xml_lang_nodup(self): text = '
English text
' - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual('
English text
', output) def test_textarea_whitespace(self): content = '\nHey there. \n\n I am indented.\n' stream = XML('%s' % content) - output = stream.render(XHTMLSerializer) + output = stream.render(XHTMLSerializer, encoding=None) self.assertEqual('%s' % content, output) def test_pre_whitespace(self): content = '\nHey there. \n\n I am indented.\n' stream = XML('
%s
' % content) - output = stream.render(XHTMLSerializer) + output = stream.render(XHTMLSerializer, encoding=None) self.assertEqual('
%s
' % content, output) def test_xml_space(self): text = ' Do not mess \n\n with me ' - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(' Do not mess \n\n with me ', output) def test_empty_script(self): text = """ """, output) @@ -261,28 +263,28 @@ text = """""" - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(text, output) def test_script_escaping_with_namespace(self): text = """""" - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(text, output) def test_style_escaping(self): text = """""" - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(text, output) def test_style_escaping_with_namespace(self): text = """""" - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(text, output) def test_embedded_svg(self): @@ -295,14 +297,14 @@ """ - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(text, output) def test_xhtml_namespace_prefix(self): text = """
Hello
""" - output = XML(text).render(XHTMLSerializer) + output = XML(text).render(XHTMLSerializer, encoding=None) self.assertEqual(text, output) def test_nested_default_namespaces(self): @@ -323,7 +325,7 @@ (Stream.END, QName('div'), (None, -1, -1)), (Stream.END_NS, '', (None, -1, -1)) ]) - output = stream.render(XHTMLSerializer) + output = stream.render(XHTMLSerializer, encoding=None) self.assertEqual("""

@@ -347,7 +349,7 @@ (Stream.END, QName('div'), (None, -1, -1)), (Stream.END_NS, 'x', (None, -1, -1)) ]) - output = stream.render(XHTMLSerializer) + output = stream.render(XHTMLSerializer, encoding=None) self.assertEqual("""

@@ -355,7 +357,8 @@ def test_html5_doctype(self): stream = HTML('') - output = stream.render(XHTMLSerializer, doctype=DocType.HTML5) + output = stream.render(XHTMLSerializer, doctype=DocType.HTML5, + encoding=None) self.assertEqual('\n', output) @@ -363,39 +366,39 @@ def test_xml_lang(self): text = '
English text
' - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual('
English text
', output) def test_xml_lang_nodup(self): text = '
English text
' - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual('
English text
', output) def test_textarea_whitespace(self): content = '\nHey there. \n\n I am indented.\n' stream = XML('%s' % content) - output = stream.render(HTMLSerializer) + output = stream.render(HTMLSerializer, encoding=None) self.assertEqual('%s' % content, output) def test_pre_whitespace(self): content = '\nHey there. \n\n I am indented.\n' stream = XML('
%s
' % content) - output = stream.render(HTMLSerializer) + output = stream.render(HTMLSerializer, encoding=None) self.assertEqual('
%s
' % content, output) def test_xml_space(self): text = ' Do not mess \n\n with me ' - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual(' Do not mess \n\n with me ', output) def test_empty_script(self): text = '', output) def test_script_escaping(self): text = '' - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual('', output) @@ -403,14 +406,14 @@ text = """""" - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual("""""", output) def test_style_escaping(self): text = '' - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual('', output) @@ -418,14 +421,15 @@ text = """""" - output = XML(text).render(HTMLSerializer) + output = XML(text).render(HTMLSerializer, encoding=None) self.assertEqual("""""", output) def test_html5_doctype(self): stream = HTML('') - output = stream.render(HTMLSerializer, doctype=DocType.HTML5) + output = stream.render(HTMLSerializer, doctype=DocType.HTML5, + encoding=None) self.assertEqual('\n', output) diff --git a/genshi/tests/path.py b/genshi/tests/path.py --- a/genshi/tests/path.py +++ b/genshi/tests/path.py @@ -25,45 +25,10 @@ def test(self, ignore_context = False): return self.strategy.test(ignore_context) + class PathTestCase(unittest.TestCase): strategies = [GenericStrategy, SingleStepStrategy, SimplePathStrategy] - def _create_path(self, expression, expected): - return path - - def _test_strategies(self, stream, path, render, - namespaces=None, variables=None): - for strategy in self.strategies: - if not strategy.supports(path): - continue - s = strategy(path) - rendered = FakePath(s).select(stream,namespaces=namespaces, - variables=variables).render() - msg = "Bad render using %s strategy"%str(strategy) - msg += "\nExpected:\t'%s'"%render - msg += "\nRendered:\t'%s'"%rendered - self.assertEqual(render, rendered, msg) - - def _test_expression(self, text, expected, stream=None, render="", - namespaces=None, variables=None): - path = Path(text) - if expected is not None: - self.assertEqual(expected, repr(path)) - - if stream is None: - return - - rendered = path.select(stream, namespaces=namespaces, - variables=variables).render() - msg = "Bad render using whole path" - msg += "\nExpected:\t'%s'"%render - msg += "\nRendered:\t'%s'"%rendered - self.assertEqual(render, rendered, msg) - - if len(path.paths) == 1: - self._test_strategies(stream, path.paths[0], render, - namespaces=namespaces, variables=variables) - def test_error_no_absolute_path(self): self.assertRaises(PathSyntaxError, Path, '/root') @@ -74,442 +39,542 @@ def test_1step(self): xml = XML('') - - self._test_expression( 'elem', - '', - xml, - '') - - self._test_expression( 'elem', - '', - xml, - '') - - self._test_expression( 'child::elem', - '', - xml, - '') - - self._test_expression( '//elem', - '', - xml, - '') - - self._test_expression( 'descendant::elem', - '', - xml, - '') + self._test_eval( + path = 'elem', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'elem', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'child::elem', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = '//elem', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'descendant::elem', + equiv = '', + input = xml, + output = '' + ) def test_1step_self(self): xml = XML('') - - self._test_expression( '.', - '', - xml, - '') - - self._test_expression( 'self::node()', - '', - xml, - '') + self._test_eval( + path = '.', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'self::node()', + equiv = '', + input = xml, + output = '' + ) def test_1step_wildcard(self): xml = XML('') - - self._test_expression( '*', - '', - xml, - '') - - self._test_expression( 'child::*', - '', - xml, - '') - - self._test_expression( 'child::node()', - '', - xml, - '') - - self._test_expression( '//*', - '', - xml, - '') + self._test_eval( + path = '*', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'child::*', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'child::node()', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = '//*', + equiv = '', + input = xml, + output = '' + ) def test_1step_attribute(self): - self._test_expression( '@foo', - '', - XML(''), - '') - + self._test_eval( + path = '@foo', + equiv = '', + input = XML(''), + output = '' + ) xml = XML('') - - self._test_expression( '@foo', - '', - xml, - 'bar') - - self._test_expression( './@foo', - '', - xml, - 'bar') + self._test_eval( + path = '@foo', + equiv = '', + input = xml, + output = 'bar' + ) + self._test_eval( + path = './@foo', + equiv = '', + input = xml, + output = 'bar' + ) def test_1step_text(self): xml = XML('Hey') - - self._test_expression( 'text()', - '', - xml, - 'Hey') - - self._test_expression( './text()', - '', - xml, - 'Hey') - - self._test_expression( '//text()', - '', - xml, - 'Hey') - - self._test_expression( './/text()', - '', - xml, - 'Hey') + self._test_eval( + path = 'text()', + equiv = '', + input = xml, + output = 'Hey' + ) + self._test_eval( + path = './text()', + equiv = '', + input = xml, + output = 'Hey' + ) + self._test_eval( + path = '//text()', + equiv = '', + input = xml, + output = 'Hey' + ) + self._test_eval( + path = './/text()', + equiv = '', + input = xml, + output = 'Hey' + ) def test_2step(self): xml = XML('') - self._test_expression('*', None, xml, '') - self._test_expression('bar', None, xml, '') - self._test_expression('baz', None, xml, '') + self._test_eval('*', input=xml, output='') + self._test_eval('bar', input=xml, output='') + self._test_eval('baz', input=xml, output='') def test_2step_attribute(self): xml = XML('Hey Joe') - self._test_expression('@*', None, xml, 'x') - self._test_expression('./@*', None, xml, 'x') - self._test_expression('.//@*', None, xml, 'xjoe') - self._test_expression('*/@*', None, xml, 'joe') + self._test_eval('@*', input=xml, output='x') + self._test_eval('./@*', input=xml, output='x') + self._test_eval('.//@*', input=xml, output='xjoe') + self._test_eval('*/@*', input=xml, output='joe') xml = XML('') - self._test_expression('@*', None, xml, '') - self._test_expression('foo/@*', None, xml, '12') + self._test_eval('@*', input=xml, output='') + self._test_eval('foo/@*', input=xml, output='12') def test_2step_complex(self): xml = XML('') - - self._test_expression( 'foo/bar', - '', - xml, - '') - - self._test_expression( './bar', - '', - xml, - '') - - self._test_expression( 'foo/*', - '', - xml, - '') - + self._test_eval( + path = 'foo/bar', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = './bar', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'foo/*', + equiv = '', + input = xml, + output = '' + ) xml = XML('') - self._test_expression( './bar', - '', - xml, - '') + self._test_eval( + path = './bar', + equiv = '', + input = xml, + output = '' + ) + xml = XML(''' + + +
1 One
2 Two
''') + self._test_eval( + path = 'tr/td[1]', + input = xml, + output = '12' + ) + xml = XML('''
+
item1 +
subitem11
+
+
item2 +
subitem21
+
+
''') + self._test_eval( + path = 'li[2]/ul', + input = xml, + output = '
subitem21
' + ) def test_2step_text(self): xml = XML('Foo') - - self._test_expression( 'item/text()', - '', - xml, - 'Foo') - - self._test_expression( '*/text()', - '', - xml, - 'Foo') - - self._test_expression( '//text()', - '', - xml, - 'Foo') - - self._test_expression( './text()', - '', - xml, - '') - + self._test_eval( + path = 'item/text()', + equiv = '', + input = xml, + output = 'Foo' + ) + self._test_eval( + path = '*/text()', + equiv = '', + input = xml, + output = 'Foo' + ) + self._test_eval( + path = '//text()', + equiv = '', + input = xml, + output = 'Foo' + ) + self._test_eval( + path = './text()', + equiv = '', + input = xml, + output = '' + ) xml = XML('FooBar') - self._test_expression( 'item/text()', - '', - xml, - 'FooBar') + self._test_eval( + path = 'item/text()', + equiv = '', + input = xml, + output = 'FooBar' + ) + xml = XML('Foo_Bar') + self._test_eval( + path = 'item/name/text()', + equiv = '', + input = xml, + output = 'Foo' + ) def test_3step(self): xml = XML('') - self._test_expression( 'foo/*', - '', - xml, - '') + self._test_eval( + path = 'foo/*', + equiv = '', + input = xml, + output = '' + ) def test_3step_complex(self): - xml = XML('') - self._test_expression( '*/bar', - '', - xml, - '') + self._test_eval( + path = '*/bar', + equiv = '', + input = XML(''), + output = '' + ) + self._test_eval( + path = '//bar', + equiv = '', + input = XML(''), + output = '' + ) - xml = XML('') - self._test_expression( '//bar', - '', - xml, - '') + def test_3step_complex_text(self): + xml = XML('Some text in here.') + self._test_eval( + path = 'item/bar/text()', + equiv = '', + input = xml, + output = 'Some text ' + ) + self._test_eval( + path = 'item//bar/text()', + equiv = '', + input = xml, + output = 'Some text in here.' + ) def test_node_type_comment(self): xml = XML('') - self._test_expression( 'comment()', - '', - xml, - '') + self._test_eval( + path = 'comment()', + equiv = '', + input = xml, + output = '' + ) def test_node_type_text(self): xml = XML('Some text
in here.') - self._test_expression( 'text()', - '', - xml, - 'Some text in here.') + self._test_eval( + path = 'text()', + equiv = '', + input = xml, + output = 'Some text in here.' + ) def test_node_type_node(self): xml = XML('Some text
in here.') - self._test_expression( 'node()', - '', - xml, - 'Some text
in here.',) + self._test_eval( + path = 'node()', + equiv = '', + input = xml, + output = 'Some text
in here.' + ) def test_node_type_processing_instruction(self): xml = XML('') - - self._test_expression( '//processing-instruction()', - '', - xml, - '') - - self._test_expression( 'processing-instruction()', - '', - xml, - '') - - self._test_expression( 'processing-instruction("php")', - '', - xml, - '') + self._test_eval( + path = '//processing-instruction()', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'processing-instruction()', + equiv = '', + input = xml, + output = '' + ) + self._test_eval( + path = 'processing-instruction("php")', + equiv = '', + input = xml, + output = '' + ) def test_simple_union(self): xml = XML("""1
2
3
""") - self._test_expression( '*|text()', - '', - xml, - '1
2
3
') + self._test_eval( + path = '*|text()', + equiv = '', + input = xml, + output = '1
2
3
' + ) def test_predicate_name(self): xml = XML('') - self._test_expression('*[name()="foo"]', None, xml, '') + self._test_eval('*[name()="foo"]', input=xml, output='') def test_predicate_localname(self): xml = XML('') - self._test_expression('*[local-name()="foo"]', None, xml, - '') + self._test_eval('*[local-name()="foo"]', input=xml, + output='') def test_predicate_namespace(self): xml = XML('') - self._test_expression('*[namespace-uri()="NS"]', None, xml, - '') + self._test_eval('*[namespace-uri()="NS"]', input=xml, + output='') def test_predicate_not_name(self): xml = XML('') - self._test_expression('*[not(name()="foo")]', None, xml, '') + self._test_eval('*[not(name()="foo")]', input=xml, + output='') def test_predicate_attr(self): xml = XML('') - self._test_expression('item[@important]', None, xml, - '') - self._test_expression('item[@important="very"]', None, xml, - '') + self._test_eval('item[@important]', input=xml, + output='') + self._test_eval('item[@important="very"]', input=xml, + output='') def test_predicate_attr_equality(self): xml = XML('') - self._test_expression('item[@important="very"]', None, xml, '') - self._test_expression('item[@important!="very"]', None, xml, - '') + self._test_eval('item[@important="very"]', input=xml, output='') + self._test_eval('item[@important!="very"]', input=xml, + output='') def test_predicate_attr_greater_than(self): xml = XML('') - self._test_expression('item[@priority>3]', None, xml, '') - self._test_expression('item[@priority>2]', None, xml, - '') + self._test_eval('item[@priority>3]', input=xml, output='') + self._test_eval('item[@priority>2]', input=xml, + output='') def test_predicate_attr_less_than(self): xml = XML('') - self._test_expression('item[@priority<3]', None, xml, '') - self._test_expression('item[@priority<4]', None, xml, - '') + self._test_eval('item[@priority<3]', input=xml, output='') + self._test_eval('item[@priority<4]', input=xml, + output='') def test_predicate_attr_and(self): xml = XML('') - self._test_expression('item[@important and @important="very"]', - None, xml, '') - self._test_expression('item[@important and @important="notso"]', - None, xml, '') + self._test_eval('item[@important and @important="very"]', + input=xml, output='') + self._test_eval('item[@important and @important="notso"]', + input=xml, output='') def test_predicate_attr_or(self): xml = XML('') - self._test_expression('item[@urgent or @important]', None, xml, - '') - self._test_expression('item[@urgent or @notso]', None, xml, '') + self._test_eval('item[@urgent or @important]', input=xml, + output='') + self._test_eval('item[@urgent or @notso]', input=xml, output='') def test_predicate_boolean_function(self): xml = XML('bar') - self._test_expression('*[boolean("")]', None, xml, '') - self._test_expression('*[boolean("yo")]', None, xml, 'bar') - self._test_expression('*[boolean(0)]', None, xml, '') - self._test_expression('*[boolean(42)]', None, xml, 'bar') - self._test_expression('*[boolean(false())]', None, xml, '') - self._test_expression('*[boolean(true())]', None, xml, - 'bar') + self._test_eval('*[boolean("")]', input=xml, output='') + self._test_eval('*[boolean("yo")]', input=xml, + output='bar') + self._test_eval('*[boolean(0)]', input=xml, output='') + self._test_eval('*[boolean(42)]', input=xml, + output='bar') + self._test_eval('*[boolean(false())]', input=xml, output='') + self._test_eval('*[boolean(true())]', input=xml, + output='bar') def test_predicate_ceil_function(self): xml = XML('bar') - self._test_expression('*[ceiling("4.5")=5]', None, xml, - 'bar') + self._test_eval('*[ceiling("4.5")=5]', input=xml, + output='bar') def test_predicate_concat_function(self): xml = XML('bar') - self._test_expression('*[name()=concat("f", "oo")]', None, xml, - 'bar') + self._test_eval('*[name()=concat("f", "oo")]', input=xml, + output='bar') def test_predicate_contains_function(self): xml = XML('bar') - self._test_expression('*[contains(name(), "oo")]', None, xml, - 'bar') + self._test_eval('*[contains(name(), "oo")]', input=xml, + output='bar') def test_predicate_matches_function(self): xml = XML('barfoo') - self._test_expression('*[matches(name(), "foo|bar")]', None, xml, - 'barfoo') + self._test_eval('*[matches(name(), "foo|bar")]', input=xml, + output='barfoo') def test_predicate_false_function(self): xml = XML('bar') - self._test_expression('*[false()]', None, xml, '') + self._test_eval('*[false()]', input=xml, output='') def test_predicate_floor_function(self): xml = XML('bar') - self._test_expression('*[floor("4.5")=4]', None, xml, - 'bar') + self._test_eval('*[floor("4.5")=4]', input=xml, + output='bar') def test_predicate_normalize_space_function(self): xml = XML('bar') - self._test_expression('*[normalize-space(" foo bar ")="foo bar"]', - None, xml, 'bar') + self._test_eval('*[normalize-space(" foo bar ")="foo bar"]', + input=xml, output='bar') def test_predicate_number_function(self): xml = XML('bar') - self._test_expression('*[number("3.0")=3]', None, xml, - 'bar') - self._test_expression('*[number("3.0")=3.0]', None, xml, - 'bar') - self._test_expression('*[number("0.1")=.1]', None, xml, - 'bar') + self._test_eval('*[number("3.0")=3]', input=xml, + output='bar') + self._test_eval('*[number("3.0")=3.0]', input=xml, + output='bar') + self._test_eval('*[number("0.1")=.1]', input=xml, + output='bar') def test_predicate_round_function(self): xml = XML('bar') - self._test_expression('*[round("4.4")=4]', None, xml, - 'bar') - self._test_expression('*[round("4.6")=5]', None, xml, - 'bar') + self._test_eval('*[round("4.4")=4]', input=xml, + output='bar') + self._test_eval('*[round("4.6")=5]', input=xml, + output='bar') def test_predicate_starts_with_function(self): xml = XML('bar') - self._test_expression('*[starts-with(name(), "f")]', None, xml, - 'bar') - self._test_expression('*[starts-with(name(), "b")]', None, xml, '') + self._test_eval('*[starts-with(name(), "f")]', input=xml, + output='bar') + self._test_eval('*[starts-with(name(), "b")]', input=xml, + output='') def test_predicate_string_length_function(self): xml = XML('bar') - self._test_expression('*[string-length(name())=3]', None, xml, - 'bar') + self._test_eval('*[string-length(name())=3]', input=xml, + output='bar') def test_predicate_substring_function(self): xml = XML('bar') - self._test_expression('*[substring(name(), 1)="oo"]', None, xml, - 'bar') - self._test_expression('*[substring(name(), 1, 1)="o"]', None, xml, - 'bar') + self._test_eval('*[substring(name(), 1)="oo"]', input=xml, + output='bar') + self._test_eval('*[substring(name(), 1, 1)="o"]', input=xml, + output='bar') def test_predicate_substring_after_function(self): xml = XML('bar') - self._test_expression('*[substring-after(name(), "f")="oo"]', None, xml, - 'bar') + self._test_eval('*[substring-after(name(), "f")="oo"]', input=xml, + output='bar') def test_predicate_substring_before_function(self): xml = XML('bar') - self._test_expression('*[substring-before(name(), "oo")="f"]', - None, xml, 'bar') + self._test_eval('*[substring-before(name(), "oo")="f"]', + input=xml, output='bar') def test_predicate_translate_function(self): xml = XML('bar') - self._test_expression('*[translate(name(), "fo", "ba")="baa"]', - None, xml, 'bar') + self._test_eval('*[translate(name(), "fo", "ba")="baa"]', + input=xml, output='bar') def test_predicate_true_function(self): xml = XML('bar') - self._test_expression('*[true()]', None, xml, 'bar') + self._test_eval('*[true()]', input=xml, output='bar') def test_predicate_variable(self): xml = XML('bar') - variables = {'bar': 'foo'} - self._test_expression('*[name()=$bar]', None, xml, 'bar', - variables = variables) + self._test_eval( + path = '*[name()=$bar]', + input = xml, + output = 'bar', + variables = {'bar': 'foo'} + ) def test_predicate_position(self): xml = XML('') - self._test_expression('*[2]', None, xml, '') + self._test_eval('*[2]', input=xml, output='') def test_predicate_attr_and_position(self): xml = XML('') - self._test_expression('*[@id][2]', None, xml, '') + self._test_eval('*[@id][2]', input=xml, output='') def test_predicate_position_and_attr(self): xml = XML('') - self._test_expression('*[1][@id]', None, xml, '') - self._test_expression('*[2][@id]', None, xml, '') + self._test_eval('*[1][@id]', input=xml, output='') + self._test_eval('*[2][@id]', input=xml, output='') def test_predicate_advanced_position(self): xml = XML('') - self._test_expression( 'descendant-or-self::*/' + self._test_eval( 'descendant-or-self::*/' 'descendant-or-self::*/' 'descendant-or-self::*[2]/' - 'self::*/descendant::*[3]', None, xml, - '') + 'self::*/descendant::*[3]', input=xml, + output='') def test_predicate_child_position(self): xml = XML('\ 123 45') - self._test_expression('//a/b[2]', None, xml, '25') - self._test_expression('//a/b[3]', None, xml, '3') + self._test_eval('//a/b[2]', input=xml, output='25') + self._test_eval('//a/b[3]', input=xml, output='3') def test_name_with_namespace(self): xml = XML('bar') - self._test_expression('f:foo', '', xml, - 'bar', - namespaces = {'f': 'FOO'}) + self._test_eval( + path = 'f:foo', + equiv = '', + input = xml, + output = 'bar', + namespaces = {'f': 'FOO'} + ) def test_wildcard_with_namespace(self): xml = XML('bar') - self._test_expression('f:*', '', xml, - 'bar', - namespaces = {'f': 'FOO'}) + self._test_eval( + path = 'f:*', + equiv = '', + input = xml, + output = 'bar', + namespaces = {'f': 'FOO'} + ) def test_predicate_termination(self): """ @@ -517,54 +582,58 @@ cause an infinite loop. See . """ xml = XML('
a
b
') - self._test_expression('.[@flag="1"]/*', None, xml, - '
a
b
') + self._test_eval('.[@flag="1"]/*', input=xml, + output='
a
b
') xml = XML('
a
b
') - self._test_expression('.[@flag="0"]/*', None, xml, '') + self._test_eval('.[@flag="0"]/*', input=xml, output='') def test_attrname_with_namespace(self): xml = XML('') - self._test_expression('foo[@f:bar]', None, xml, - '', - namespaces={'f': 'FOO'}) + self._test_eval('foo[@f:bar]', input=xml, + output='', + namespaces={'f': 'FOO'}) def test_attrwildcard_with_namespace(self): xml = XML('') - self._test_expression('foo[@f:*]', None, xml, - '', - namespaces={'f': 'FOO'}) + self._test_eval('foo[@f:*]', input=xml, + output='', + namespaces={'f': 'FOO'}) + def test_self_and_descendant(self): xml = XML('') - self._test_expression('self::root', None, xml, '') - self._test_expression('self::foo', None, xml, '') - self._test_expression('descendant::root', None, xml, '') - self._test_expression('descendant::foo', None, xml, '') - self._test_expression('descendant-or-self::root', None, xml, - '') - self._test_expression('descendant-or-self::foo', None, xml, '') + self._test_eval('self::root', input=xml, output='') + self._test_eval('self::foo', input=xml, output='') + self._test_eval('descendant::root', input=xml, output='') + self._test_eval('descendant::foo', input=xml, output='') + self._test_eval('descendant-or-self::root', input=xml, + output='') + self._test_eval('descendant-or-self::foo', input=xml, output='') def test_long_simple_paths(self): xml = XML('!' '') - self._test_expression('//a/b/a/b/a/c', None, xml, '!') - self._test_expression('//a/b/a/c', None, xml, '!') - self._test_expression('//a/c', None, xml, '!') - self._test_expression('//c', None, xml, '!') + self._test_eval('//a/b/a/b/a/c', input=xml, output='!') + self._test_eval('//a/b/a/c', input=xml, output='!') + self._test_eval('//a/c', input=xml, output='!') + self._test_eval('//c', input=xml, output='!') # Please note that a//b is NOT the same as a/descendant::b # it is a/descendant-or-self::node()/b, which SimplePathStrategy # does NOT support - self._test_expression('a/b/descendant::a/c', None, xml, '!') - self._test_expression('a/b/descendant::a/d/descendant::a/c', - None, xml, '!') - self._test_expression('a/b/descendant::a/d/a/c', None, xml, '') - self._test_expression('//d/descendant::b/descendant::b/descendant::b' - '/descendant::c', None, xml, '!') - self._test_expression('//d/descendant::b/descendant::b/descendant::b' - '/descendant::b/descendant::c', None, xml, '') + self._test_eval('a/b/descendant::a/c', input=xml, output='!') + self._test_eval('a/b/descendant::a/d/descendant::a/c', + input=xml, output='!') + self._test_eval('a/b/descendant::a/d/a/c', input=xml, output='') + self._test_eval('//d/descendant::b/descendant::b/descendant::b' + '/descendant::c', input=xml, output='!') + self._test_eval('//d/descendant::b/descendant::b/descendant::b' + '/descendant::b/descendant::c', input=xml, + output='') + def _test_support(self, strategy_class, text): path = PathParser(text, None, -1).parse()[0] return strategy_class.supports(path) + def test_simple_strategy_support(self): self.assert_(self._test_support(SimplePathStrategy, 'a/b')) self.assert_(self._test_support(SimplePathStrategy, 'self::a/b')) @@ -582,11 +651,47 @@ self.assert_(not self._test_support(SimplePathStrategy, 'foo:bar')) self.assert_(not self._test_support(SimplePathStrategy, 'a/@foo:bar')) + def _test_strategies(self, input, path, output, + namespaces=None, variables=None): + for strategy in self.strategies: + if not strategy.supports(path): + continue + s = strategy(path) + rendered = FakePath(s).select(input, namespaces=namespaces, + variables=variables) \ + .render(encoding=None) + msg = 'Bad render using %s strategy' % str(strategy) + msg += '\nExpected:\t%r' % output + msg += '\nRendered:\t%r' % rendered + self.assertEqual(output, rendered, msg) + + def _test_eval(self, path, equiv=None, input=None, output='', + namespaces=None, variables=None): + path = Path(path) + if equiv is not None: + self.assertEqual(equiv, repr(path)) + + if input is None: + return + + rendered = path.select(input, namespaces=namespaces, + variables=variables).render(encoding=None) + msg = 'Bad output using whole path' + msg += '\nExpected:\t%r' % output + msg += '\nRendered:\t%r' % rendered + self.assertEqual(output, rendered, msg) + + if len(path.paths) == 1: + self._test_strategies(input, path.paths[0], output, + namespaces=namespaces, variables=variables) + + def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(Path.__module__)) suite.addTest(unittest.makeSuite(PathTestCase, 'test')) return suite + if __name__ == '__main__': unittest.main(defaultTest='suite') diff --git a/genshi/tests/util.py b/genshi/tests/util.py --- a/genshi/tests/util.py +++ b/genshi/tests/util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006,2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -29,8 +29,8 @@ item_a = cache._dict['A'] self.assertEqual('A', item_a.key) self.assertEqual(0, item_a.value) - self.assertEqual(None, item_a.previous) - self.assertEqual(None, item_a.next) + self.assertEqual(None, item_a.prv) + self.assertEqual(None, item_a.nxt) cache['B'] = 1 self.assertEqual(2, len(cache)) @@ -40,12 +40,12 @@ item_b = cache._dict['B'] self.assertEqual('A', item_a.key) self.assertEqual(0, item_a.value) - self.assertEqual(item_b, item_a.previous) - self.assertEqual(None, item_a.next) + self.assertEqual(item_b, item_a.prv) + self.assertEqual(None, item_a.nxt) self.assertEqual('B', item_b.key) self.assertEqual(1, item_b.value) - self.assertEqual(None, item_b.previous) - self.assertEqual(item_a, item_b.next) + self.assertEqual(None, item_b.prv) + self.assertEqual(item_a, item_b.nxt) cache['C'] = 2 self.assertEqual(2, len(cache)) @@ -55,12 +55,12 @@ item_c = cache._dict['C'] self.assertEqual('B', item_b.key) self.assertEqual(1, item_b.value) - self.assertEqual(item_c, item_b.previous) - self.assertEqual(None, item_b.next) + self.assertEqual(item_c, item_b.prv) + self.assertEqual(None, item_b.nxt) self.assertEqual('C', item_c.key) self.assertEqual(2, item_c.value) - self.assertEqual(None, item_c.previous) - self.assertEqual(item_b, item_c.next) + self.assertEqual(None, item_c.prv) + self.assertEqual(item_b, item_c.nxt) def test_getitem(self): cache = LRUCache(2) @@ -76,12 +76,12 @@ item_b = cache._dict['B'] self.assertEqual('A', item_a.key) self.assertEqual(0, item_a.value) - self.assertEqual(None, item_a.previous) - self.assertEqual(item_b, item_a.next) + self.assertEqual(None, item_a.prv) + self.assertEqual(item_b, item_a.nxt) self.assertEqual('B', item_b.key) self.assertEqual(1, item_b.value) - self.assertEqual(item_a, item_b.previous) - self.assertEqual(None, item_b.next) + self.assertEqual(item_a, item_b.prv) + self.assertEqual(None, item_b.nxt) def suite(): diff --git a/genshi/util.py b/genshi/util.py --- a/genshi/util.py +++ b/genshi/util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2009 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -13,7 +13,7 @@ """Various utility classes and functions.""" -import htmlentitydefs +import htmlentitydefs as entities import re __docformat__ = 'restructuredtext en' @@ -46,7 +46,7 @@ used: >>> for key in cache: - ... print key + ... print(key) D A C @@ -59,7 +59,7 @@ class _Item(object): def __init__(self, key, value): - self.previous = self.next = None + self.prv = self.nxt = None self.key = key self.value = value def __repr__(self): @@ -78,7 +78,7 @@ cur = self.head while cur: yield cur.key - cur = cur.next + cur = cur.nxt def __len__(self): return len(self._dict) @@ -103,10 +103,10 @@ return repr(self._dict) def _insert_item(self, item): - item.previous = None - item.next = self.head + item.prv = None + item.nxt = self.head if self.head is not None: - self.head.previous = item + self.head.prv = item else: self.tail = item self.head = item @@ -117,8 +117,8 @@ olditem = self._dict[self.tail.key] del self._dict[self.tail.key] if self.tail != self.head: - self.tail = self.tail.previous - self.tail.next = None + self.tail = self.tail.prv + self.tail.nxt = None else: self.head = self.tail = None @@ -126,16 +126,16 @@ if self.head == item: return - previous = item.previous - previous.next = item.next - if item.next is not None: - item.next.previous = previous + prv = item.prv + prv.nxt = item.nxt + if item.nxt is not None: + item.nxt.prv = prv else: - self.tail = previous + self.tail = prv - item.previous = None - item.next = self.head - self.head.previous = self.head = item + item.prv = None + item.nxt = self.head + self.head.prv = self.head = item def flatten(items): @@ -158,9 +158,9 @@ retval.append(item) return retval + def plaintext(text, keeplinebreaks=True): - """Returns the text as a `unicode` string with all entities and tags - removed. + """Return the text with all entities and tags removed. >>> plaintext('1 < 2') u'1 < 2' @@ -179,9 +179,10 @@ """ text = stripentities(striptags(text)) if not keeplinebreaks: - text = text.replace(u'\n', u' ') + text = text.replace('\n', ' ') return text + _STRIPENTITIES_RE = re.compile(r'&(?:#((?:\d+)|(?:[xX][0-9a-fA-F]+));?|(\w+);)') def stripentities(text, keepxmlentities=False): """Return a copy of the given text with any character or numeric entities @@ -213,16 +214,17 @@ else: # character entity ref = match.group(2) if keepxmlentities and ref in ('amp', 'apos', 'gt', 'lt', 'quot'): - return u'&%s;' % ref + return '&%s;' % ref try: - return unichr(htmlentitydefs.name2codepoint[ref]) + return unichr(entities.name2codepoint[ref]) except KeyError: if keepxmlentities: - return u'&%s;' % ref + return '&%s;' % ref else: return ref return _STRIPENTITIES_RE.sub(_replace_entity, text) + _STRIPTAGS_RE = re.compile(r'(|<[^>]*>)') def striptags(text): """Return a copy of the text with any XML/HTML tags removed. @@ -243,3 +245,30 @@ :return: the text with tags removed """ return _STRIPTAGS_RE.sub('', text) + + +def stringrepr(string): + ascii = string.encode('ascii', 'backslashreplace') + quoted = "'" + ascii.replace("'", "\\'") + "'" + if len(ascii) > len(string): + return 'u' + quoted + return quoted + + +# Compatibility fallback implementations for older Python versions + +try: + all = all + any = any +except NameError: + def any(S): + for x in S: + if x: + return True + return False + + def all(S): + for x in S: + if not x: + return False + return True diff --git a/scripts/ast_generator.py b/scripts/ast_generator.py --- a/scripts/ast_generator.py +++ b/scripts/ast_generator.py @@ -20,24 +20,24 @@ print_class(base) bnames.append(base.__name__) elif base.__module__ == '__builtin__': - bnames.append("%s"%base.__name__) + bnames.append("%s" % base.__name__) else: - bnames.append("%s.%s"%(base.__module__,base.__name__)) - print "class %s(%s):"%(cls.__name__, ", ".join(bnames)) + bnames.append("%s.%s" % (base.__module__,base.__name__)) + print("class %s(%s):" % (cls.__name__, ", ".join(bnames))) written = False for attr in cls.__dict__: if attr not in IGNORE_ATTRS: written = True - print "\t%s = %s"%(attr, repr(cls.__dict__[attr]),) + print("\t%s = %s" % (attr, repr(cls.__dict__[attr]),)) if not written: - print "\tpass" + print("\tpass") done.add(cls) -print "# Generated automatically, please do not edit" -print "# Generator can be found in Genshi SVN, scripts/ast-generator.py" -print -print "__version__ = %s" % _ast.__version__ -print +print('# Generated automatically, please do not edit') +print('# Generator can be found in Genshi SVN, scripts/ast_generator.py') +print('') +print('__version__ = %s' % _ast.__version__) +print('') for name in dir(_ast): cls = getattr(_ast, name) diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2008 Edgewall Software +# Copyright (C) 2006-2010 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -35,6 +35,7 @@ _speedup_available = False + class optional_build_ext(build_ext): # This class allows C extension building to fail. def run(self): @@ -52,18 +53,18 @@ self._unavailable(e) def _unavailable(self, exc): - print '*' * 70 - print """WARNING: + print('*' * 70) + print("""WARNING: An optional C extension could not be compiled, speedups will not be -available.""" - print '*' * 70 - print exc +available.""") + print('*' * 70) + print(exc) if Feature: speedups = Feature( - "optionial C speed-enhancements", - standard = True, + "optional C speed-enhancements", + standard = False, ext_modules = [ Extension('genshi._speedups', ['genshi/_speedups.c']), ], @@ -87,7 +88,7 @@ setup( name = 'Genshi', - version = '0.6', + version = '0.7', description = 'A toolkit for generation of output for the web', long_description = \ """Genshi is a Python library that provides an integrated set of