# HG changeset patch # User cmlenz # Date 1167329830 0 # Node ID a816755902581023180356b03d38231189e1dd46 # Parent e9101f20b796f3294483d1448468d6391614ab81 inline branch: Merged [439:479/trunk]. diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -21,17 +21,37 @@ value. * Unsuccessful attribute or item lookups now return `Undefined` objects for nicer error messages. - * Fix XPath traversal in match templates. Previously, `div/p` would be treated - the same as `div//p`, i.e. it would match all descendants and not just the - immediate children. * Split up the `genshi.template` module into multiple modules inside the new `genshi.template` package. * Results of expression evaluation are no longer implicitly called if they are callable. * Instances of the `genshi.core.Attrs` class are now immutable (they are subclasses of `tuple` instead of `list`). - * Preserve whitespace in HTML `
` elements also when they contained any
-   child elements.
+ * `MarkupTemplate`s can now be instantiated from markup streams, in addition
+   to strings and file-like objects (ticket #69).
+ * Improve handling of incorrectly nested tags in the HTML parser.
+ * Template includes can you be nested inside fallback content.
+
+
+Version 0.3.6
+http://svn.edgewall.org/repos/genshi/tags/0.3.6/
+(Dec 11 2006, from branches/stable/0.3.x)
+
+ * The builder API now accepts streams as children of elements and fragments.
+
+
+Version 0.3.5
+http://svn.edgewall.org/repos/genshi/tags/0.3.5/
+(Nov 22 2006, from branches/stable/0.3.x)
+
+ * Fix XPath traversal in match templates. Previously, `div/p` would be treated 
+   the same as `div//p`, i.e. it would match all descendants and not just the
+   immediate children.
+ * Preserve whitespace in HTML `
` elements also when they contain child
+   elements.
+ * Match templates no longer match their own output (ticket #77).
+ * Blank lines before directives in text templates are now preserved as
+   expected (ticket #62).
 
 
 Version 0.3.4
diff --git a/doc/Makefile b/doc/Makefile
deleted file mode 100644
--- a/doc/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-HTML_FILES = \
-	builder.html \
-	index.html \
-	streams.html \
-	text-templates.html \
-	xml-templates.html \
-	xpath.html
-
-all: $(HTML_FILES)
-
-%.html: %.txt
-	rst2html.py --exit-status=3 $< $@
-
-clean:
-	rm -rf $(HTML_FILES)
diff --git a/doc/builder.txt b/doc/builder.txt
--- a/doc/builder.txt
+++ b/doc/builder.txt
@@ -52,7 +52,7 @@
 
   >>> stream = doc.generate()
   >>> stream
-  
+  
   >>> print stream
   
Some text and a link.

 
@@ -65,7 +65,7 @@
 for creating snippets of markup that are attached to a parent element later (for
 example in a template). Fragments are created by calling the ``tag`` object::
 
-  >>> fragment = tag('Hello, ', tag.em('word'), '!')
+  >>> fragment = tag('Hello, ', tag.em('world'), '!')
   >>> fragment
   
   >>> print fragment
diff --git a/doc/streams.txt b/doc/streams.txt
--- a/doc/streams.txt
+++ b/doc/streams.txt
@@ -8,7 +8,7 @@
 
 
 .. contents:: Contents
-   :depth: 2
+   :depth: 1
 .. sectnum::
 
 
@@ -30,7 +30,7 @@
   ...              'a link.'
   ...              '
')
   >>> stream
-  
+  
 
 The stream is the result of parsing the text into events. Each event is a tuple
 of the form ``(kind, data, pos)``, where:
@@ -38,7 +38,7 @@
 * ``kind`` defines what kind of event it is (such as the start of an element,
   text, a comment, etc).
 * ``data`` is the actual data associated with the event. How this looks depends
-  on the event kind.
+  on the event kind (see  `event kinds`_)
 * ``pos`` is a ``(filename, lineno, column)`` tuple that describes where the
   event “comes from”.
 
@@ -47,15 +47,15 @@
   >>> for kind, data, pos in stream:
   ...     print kind, `data`, pos
   ... 
-  START (u'p', [(u'class', u'intro')]) ('', 1, 0)
-  TEXT u'Some text and ' ('', 1, 31)
-  START (u'a', [(u'href', u'http://example.org/')]) ('', 1, 31)
-  TEXT u'a link' ('', 1, 67)
-  END u'a' ('', 1, 67)
-  TEXT u'.' ('', 1, 72)
-  START (u'br', []) ('', 1, 72)
-  END u'br' ('', 1, 77)
-  END u'p' ('', 1, 77)
+  START (QName(u'p'), Attrs([(QName(u'class'), u'intro')])) (None, 1, 0)
+  TEXT u'Some text and ' (None, 1, 17)
+  START (QName(u'a'), Attrs([(QName(u'href'), u'http://example.org/')])) (None, 1, 31)
+  TEXT u'a link' (None, 1, 61)
+  END QName(u'a') (None, 1, 67)
+  TEXT u'.' (None, 1, 71)
+  START (QName(u'br'), Attrs()) (None, 1, 72)
+  END QName(u'br') (None, 1, 77)
+  END QName(u'p') (None, 1, 77)
 
 
 Filtering
@@ -150,7 +150,7 @@
 
   >>> from genshi.filters import HTMLSanitizer
   >>> from genshi.output import TextSerializer
-  >>> print TextSerializer()(HTMLSanitizer()(stream))
+  >>> print ''.join(TextSerializer()(HTMLSanitizer()(stream)))
   Some text and a link.
 
 The pipe operator allows a nicer syntax::
@@ -158,6 +158,7 @@
   >>> print stream | HTMLSanitizer() | TextSerializer()
   Some text and a link.
 
+
 Using XPath
 ===========
 
@@ -166,7 +167,7 @@
 
   >>> substream = stream.select('a')
   >>> substream
-  
+  
   >>> print substream
   a link
 
@@ -178,10 +179,126 @@
   >>> from genshi import Stream
   >>> substream = Stream(list(stream.select('a')))
   >>> substream
-  
+  
   >>> print substream
   a link
   >>> print substream.select('@href')
   http://example.org/
   >>> print substream.select('text()')
   a link
+
+See `Using XPath in Genshi`_ for more information about the XPath support in
+Genshi.
+
+.. _`Using XPath in Genshi`: xpath.html
+
+
+.. _`event kinds`:
+
+Event Kinds
+===========
+
+Every event in a stream is of one of several *kinds*, which also determines
+what the ``data`` item of the event tuple looks like. The different kinds of
+events are documented below.
+
+.. note:: The ``data`` item is generally immutable. If the data is to be
+   modified when processing a stream, it must be replaced by a new tuple.
+   Effectively, this means the entire event tuple is immutable.
+
+START
+-----
+The opening tag of an element.
+
+For this kind of event, the ``data`` item is a tuple of the form
+``(tagname, attrs)``, where ``tagname`` is a ``QName`` instance describing the
+qualified name of the tag, and ``attrs`` is an ``Attrs`` instance containing
+the attribute names and values associated with the tag (excluding namespace
+declarations)::
+
+  START, (QName(u'p'), Attrs([(u'class', u'intro')])), pos
+
+END
+---
+The closing tag of an element.
+
+The ``data`` item of end events consists of just a ``QName`` instance
+describing the qualified name of the tag::
+
+  END, QName(u'p'), pos
+
+TEXT
+----
+Character data outside of elements and comments.
+
+For text events, the ``data`` item should be a unicode object::
+
+  TEXT, u'Hello, world!', pos
+
+START_NS
+--------
+The start of a namespace mapping, binding a namespace prefix to a URI.
+
+The ``data`` item of this kind of event is a tuple of the form
+``(prefix, uri)``, where ``prefix`` is the namespace prefix and ``uri`` is the
+full URI to which the prefix is bound. Both should be unicode objects. If the
+namespace is not bound to any prefix, the ``prefix`` item is an empty string::
+
+  START_NS, (u'svg', u'http://www.w3.org/2000/svg'), pos
+
+END_NS
+------
+The end of a namespace mapping.
+
+The ``data`` item of such events consists of only the namespace prefix (a
+unicode object)::
+
+  END_NS, u'svg', pos
+
+DOCTYPE
+-------
+A document type declaration.
+
+For this type of event, the ``data`` item is a tuple of the form
+``(name, pubid, sysid)``, where ``name`` is the name of the root element,
+``pubid`` is the public identifier of the DTD (or ``None``), and ``sysid`` is
+the system identifier of the DTD (or ``None``)::
+
+  DOCTYPE, (u'html', u'-//W3C//DTD XHTML 1.0 Transitional//EN', \
+            u'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'), pos
+
+COMMENT
+-------
+A comment.
+
+For such events, the ``data`` item is a unicode object containing all character
+data between the comment delimiters::
+
+  COMMENT, u'Commented out', pos
+
+PI
+--
+A processing instruction.
+
+The ``data`` item is a tuple of the form ``(target, data)`` for processing
+instructions, where ``target`` is the target of the PI (used to identify the
+application by which the instruction should be processed), and ``data`` is text
+following the target (excluding the terminating question mark)::
+
+  PI, (u'php', u'echo "Yo" '), pos
+
+START_CDATA
+-----------
+Marks the beginning of a ``CDATA`` section.
+
+The ``data`` item for such events is always ``None``::
+
+  START_CDATA, None, pos
+
+END_CDATA
+---------
+Marks the end of a ``CDATA`` section.
+
+The ``data`` item for such events is always ``None``::
+
+  END_CDATA, None, pos
diff --git a/doc/style/edgewall.css b/doc/style/edgewall.css
--- a/doc/style/edgewall.css
+++ b/doc/style/edgewall.css
@@ -7,13 +7,15 @@
 }
 pre, code, tt { font-size: medium; }
 h1, h2, h3, h4 {
+  border-bottom: 1px solid #ccc;
   font-family: Arial,Verdana,'Bitstream Vera Sans',Helvetica,sans-serif;
   font-weight: bold; letter-spacing: -0.018em;
 }
-h1 { font-size: 19px; margin: 2em 1em 0 0; }
-h2 { font-size: 16px; }
-h3 { font-size: 14px; }
+h1 { font-size: 19px; margin: 2em 0 .5em; }
+h2 { font-size: 16px; margin: 1.5em 0 .5em; }
+h3 { font-size: 14px; margin: 1.2em 0 .5em; }
 hr { border: none;  border-top: 1px solid #ccb; margin: 2em 0; }
+p { margin: 0 0 1em; }
 
 :link, :visited { text-decoration: none; border-bottom: 1px dotted #bbb;
   color: #b00;
@@ -26,21 +28,21 @@
 }
 
 div.document { background: #fff url(shadow.gif) right top repeat-y;
-  border-left: 1px solid #000; margin: 0 auto 0 80px;
+  border-left: 1px solid #000; margin: 0 auto 0 40px;
   min-height: 100%; width: 54em; padding: 0 180px 1px 20px;
 }
-h1.title, div.document#genshi h1 { color: #666; font-size: x-large;
-  margin: 0 -20px 1em; padding: 2em 20px 0;
+h1.title, div.document#genshi h1 { border: none; color: #666;
+  font-size: x-large; margin: 0 -20px 1em; padding: 2em 20px 0;
 }
 h1.title { background: url(vertbars.png) repeat-x; }
 div.document#genshi h1.title { text-indent: -4000px; }
 div.document#genshi h1 { text-align: center; }
-pre.literal-block { background: #f7f7f7; border: 1px solid #d7d7d7;
-  margin: 1em 0; padding: .25em; overflow: auto;
+pre.literal-block { background: #d7d7d7; border: 1px solid #e6e6e6; color: #000;
+  margin: 1em 1em; padding: .25em; overflow: auto;
 }
 
-div.contents { position: absolute; position: fixed; margin-left: 80px;
-  left: 57.3em; top: 30px; right: 0;
+div.contents { font-size: 90%; position: absolute; position: fixed;
+  margin-left: 80px; left: 60em; top: 30px; right: 0;
 }
 div.contents .topic-title { display: none; }
 div.contents ul { list-style: none; padding-left: 0; }
@@ -50,3 +52,6 @@
 div.contents :link:hover, div.contents :visited:hover { background: #000;
   color: #fff;
 }
+
+p.admonition-title { font-weight: bold; margin-bottom: 0; }
+div.note { font-style: italic; margin-left: 2em; margin-right: 2em; }
diff --git a/doc/text-templates.txt b/doc/text-templates.txt
--- a/doc/text-templates.txt
+++ b/doc/text-templates.txt
@@ -72,9 +72,10 @@
 
 .. _python: http://www.python.org/
 
-If the expression starts with a letter and contains only letters and digits,
-the curly braces may be omitted. In all other cases, the braces are required
-so that the template processors knows where the expression ends::
+If the expression starts with a letter and contains only letters, digits, dots,
+and underscores, the curly braces may be omitted. In all other cases, the
+braces are required so that the template processor knows where the expression
+ends::
 
   >>> from genshi.template import TextTemplate
   >>> tmpl = TextTemplate('${items[0].capitalize()} item')
diff --git a/doc/xml-templates.txt b/doc/xml-templates.txt
--- a/doc/xml-templates.txt
+++ b/doc/xml-templates.txt
@@ -74,9 +74,10 @@
 Expressions need to prefixed with a dollar sign (``$``) and usually enclosed in
 curly braces (``{…}``).
 
-If the expression starts with a letter and contains only letters and digits,
-the curly braces may be omitted. In all other cases, the braces are required so
-that the template processors knows where the expression ends::
+If the expression starts with a letter and contains only letters, digits, dots,
+and underscores, the curly braces may be omitted. In all other cases, the
+braces are required so that the template processor knows where the expression
+ends::
 
   >>> from genshi.template import MarkupTemplate
   >>> tmpl = MarkupTemplate('${items[0].capitalize()} item')
@@ -123,7 +124,7 @@
 template is rendered in a number of ways: Genshi provides directives for
 conditionals and looping, among others.
 
-To use directives in a template, the namespace should be declared, which is
+To use directives in a template, the namespace must be declared, which is
 usually done on the root element::
 
   
   
 
-If a macro doesn't require parameters, it can be defined as well as called
-without the parenthesis. For example::
+If a macro doesn't require parameters, it can be defined without the 
+parenthesis. For example::
 
   

     

       Hello, world!
     

-    ${greeting}
+    ${greeting()}
   

 
 The above would be rendered to::
diff --git a/doc/xpath.txt b/doc/xpath.txt
--- a/doc/xpath.txt
+++ b/doc/xpath.txt
@@ -36,7 +36,7 @@
 for I don't know). Basically, any path expression that would require buffering
 of the stream is not supported.
 
-Predicates are of course supported, but Path expressions *inside* predicates
+Predicates are of course supported, but path expressions *inside* predicates
 are restricted to attribute lookups (again due to the lack of buffering).
 
 Most of the XPath functions and operators are supported, however they
diff --git a/examples/webpy/hello.py b/examples/webpy/hello.py
--- a/examples/webpy/hello.py
+++ b/examples/webpy/hello.py
@@ -23,5 +23,5 @@
 
 
 if __name__ == '__main__':
-    web.internalerror = web.debugerror
-    web.run(urls)
+    web.webapi.internalerror = web.debugerror
+    web.run(urls, globals())
diff --git a/genshi/builder.py b/genshi/builder.py
--- a/genshi/builder.py
+++ b/genshi/builder.py
@@ -46,7 +46,7 @@
 
     def append(self, node):
         """Append an element or string as child node."""
-        if isinstance(node, (Element, basestring, int, float, long)):
+        if isinstance(node, (Stream, Element, basestring, int, float, long)):
             # For objects of a known/primitive type, we avoid the check for
             # whether it is iterable for better performance
             self.children.append(node)
@@ -63,6 +63,9 @@
             if isinstance(child, Fragment):
                 for event in child._generate():
                     yield event
+            elif isinstance(child, Stream):
+                for event in child:
+                    yield event
             else:
                 if not isinstance(child, basestring):
                     child = unicode(child)
diff --git a/genshi/core.py b/genshi/core.py
--- a/genshi/core.py
+++ b/genshi/core.py
@@ -17,7 +17,8 @@
 import operator
 import re
 
-__all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Namespace', 'QName']
+__all__ = ['Stream', 'Markup', 'escape', 'unescape', 'Attrs', 'Namespace',
+           'QName']
 
 
 class StreamEventKind(str):
@@ -399,7 +400,7 @@
         return Markup(num * unicode(self))
 
     def __repr__(self):
-        return '<%s "%s">' % (self.__class__.__name__, self)
+        return '<%s %r>' % (self.__class__.__name__, unicode(self))
 
     def join(self, seq, escape_quotes=True):
         return Markup(unicode(self).join([escape(item, quotes=escape_quotes)
diff --git a/genshi/filters.py b/genshi/filters.py
--- a/genshi/filters.py
+++ b/genshi/filters.py
@@ -19,10 +19,10 @@
     from sets import ImmutableSet as frozenset
 import re
 
-from genshi.core import Attrs, Namespace, stripentities
-from genshi.core import END, END_NS, START, START_NS, TEXT
+from genshi.core import Attrs, stripentities
+from genshi.core import END, START, TEXT
 
-__all__ = ['HTMLFormFiller', 'HTMLSanitizer', 'IncludeFilter']
+__all__ = ['HTMLFormFiller', 'HTMLSanitizer']
 
 
 class HTMLFormFiller(object):
@@ -284,78 +284,3 @@
             else:
                 if not waiting_for:
                     yield kind, data, pos
-
-
-class IncludeFilter(object):
-    """Template filter providing (very) basic XInclude support
-    (see http://www.w3.org/TR/xinclude/) in templates.
-    """
-
-    NAMESPACE = Namespace('http://www.w3.org/2001/XInclude')
-
-    def __init__(self, loader):
-        """Initialize the filter.
-        
-        @param loader: the `TemplateLoader` to use for resolving references to
-            external template files
-        """
-        self.loader = loader
-
-    def __call__(self, stream, ctxt=None):
-        """Filter the stream, processing any XInclude directives it may
-        contain.
-        
-        @param stream: the markup event stream to filter
-        @param ctxt: the template context
-        """
-        from genshi.template import TemplateError, TemplateNotFound
-
-        namespace = self.NAMESPACE
-        ns_prefixes = []
-        in_fallback = False
-        include_href = fallback_stream = None
-
-        for kind, data, pos in stream:
-
-            if kind is START and not in_fallback and data[0] in namespace:
-                tag, attrs = data
-                if tag.localname == 'include':
-                    include_href = attrs.get('href')
-                elif tag.localname == 'fallback':
-                    in_fallback = True
-                    fallback_stream = []
-
-            elif kind is END and data in namespace:
-                if data.localname == 'include':
-                    try:
-                        if not include_href:
-                            raise TemplateError('Include misses required '
-                                                'attribute "href"')
-                        template = self.loader.load(include_href,
-                                                    relative_to=pos[0])
-                        for event in template.generate(ctxt):
-                            yield event
-
-                    except TemplateNotFound:
-                        if fallback_stream is None:
-                            raise
-                        for event in fallback_stream:
-                            yield event
-
-                    include_href = None
-                    fallback_stream = None
-
-                elif data.localname == 'fallback':
-                    in_fallback = False
-
-            elif in_fallback:
-                fallback_stream.append((kind, data, pos))
-
-            elif kind is START_NS and data[1] == namespace:
-                ns_prefixes.append(data[0])
-
-            elif kind is END_NS and data in ns_prefixes:
-                ns_prefixes.pop()
-
-            else:
-                yield kind, data, pos
diff --git a/genshi/input.py b/genshi/input.py
--- a/genshi/input.py
+++ b/genshi/input.py
@@ -327,10 +327,9 @@
         if tag not in self._EMPTY_ELEMS:
             while self._open_tags:
                 open_tag = self._open_tags.pop()
+                self._enqueue(END, QName(open_tag))
                 if open_tag.lower() == tag.lower():
                     break
-                self._enqueue(END, QName(open_tag))
-            self._enqueue(END, QName(tag))
 
     def handle_data(self, text):
         if not isinstance(text, unicode):
@@ -349,8 +348,9 @@
         self._enqueue(TEXT, text)
 
     def handle_pi(self, data):
-        target, data = data.split(maxsplit=1)
-        data = data.rstrip('?')
+        target, data = data.split(None, 1)
+        if data.endswith('?'):
+            data = data[:-1]
         self._enqueue(PI, (target.strip(), data.strip()))
 
     def handle_comment(self, text):
diff --git a/genshi/path.py b/genshi/path.py
--- a/genshi/path.py
+++ b/genshi/path.py
@@ -156,8 +156,8 @@
         >>> test = Path('child').test()
         >>> for event in xml:
         ...     if test(event, {}, {}):
-        ...         print event
-        ('START', (QName(u'child'), Attrs([(QName(u'id'), u'2')])), (None, 1, 34))
+        ...         print event[0], repr(event[1])
+        START (QName(u'child'), Attrs([(QName(u'id'), u'2')]))
         """
         paths = [(p, len(p), [0], [], [0] * len(p)) for p in [
             (ignore_context and [_DOTSLASHSLASH] or []) + p for p in self.paths
@@ -237,15 +237,15 @@
 
                         elif steps[cursor][0] is ATTRIBUTE:
                             # If the axis of the next location step is the
-                            # attribute axis, we need to move on to
-                            # processing that step without waiting for the
-                            # next markup event
+                            # attribute axis, we need to move on to processing
+                            # that step without waiting for the next markup
+                            # event
                             continue
 
                     # We're done with this step if it's the last step or the
                     # axis isn't "self"
-                    if last_step or not (axis is SELF or
-                                         axis is DESCENDANT_OR_SELF):
+                    if not matched or last_step or not (
+                            axis is SELF or axis is DESCENDANT_OR_SELF):
                         break
 
                 if (retval or not matched) and kind is START and \
@@ -550,7 +550,7 @@
         return '%s:*' % self.prefix
 
 class LocalNameTest(object):
-    """Node test that matches any event with the given prinipal type and
+    """Node test that matches any event with the given principal type and
     local name.
     """
     __slots__ = ['principal_type', 'name']
@@ -567,7 +567,7 @@
         return self.name
 
 class QualifiedNameTest(object):
-    """Node test that matches any event with the given prinipal type and
+    """Node test that matches any event with the given principal type and
     qualified name.
     """
     __slots__ = ['principal_type', 'prefix', 'name']
diff --git a/genshi/template/core.py b/genshi/template/core.py
--- a/genshi/template/core.py
+++ b/genshi/template/core.py
@@ -139,62 +139,6 @@
         """Pop the top-most scope from the stack."""
 
 
-class Directive(object):
-    """Abstract base class for template directives.
-    
-    A directive is basically a callable that takes three positional arguments:
-    `ctxt` is the template data context, `stream` is an iterable over the
-    events that the directive applies to, and `directives` is is a list of
-    other directives on the same stream that need to be applied.
-    
-    Directives can be "anonymous" or "registered". Registered directives can be
-    applied by the template author using an XML attribute with the
-    corresponding name in the template. Such directives should be subclasses of
-    this base class that can  be instantiated with the value of the directive
-    attribute as parameter.
-    
-    Anonymous directives are simply functions conforming to the protocol
-    described above, and can only be applied programmatically (for example by
-    template filters).
-    """
-    __slots__ = ['expr']
-
-    def __init__(self, value, namespaces=None, filename=None, lineno=-1,
-                 offset=-1):
-        try:
-            self.expr = value and Expression(value, filename, lineno) or None
-        except SyntaxError, err:
-            err.msg += ' in expression "%s" of "%s" directive' % (value,
-                                                                  self.tagname)
-            raise TemplateSyntaxError(err, filename, lineno,
-                                      offset + (err.offset or 0))
-
-    def __call__(self, stream, ctxt, directives):
-        raise NotImplementedError
-
-    def __repr__(self):
-        expr = ''
-        if self.expr is not None:
-            expr = ' "%s"' % self.expr.source
-        return '<%s%s>' % (self.__class__.__name__, expr)
-
-    def prepare(self, directives, stream):
-        """Called after the template stream has been completely parsed.
-        
-        The part of the template stream associated with the directive will be
-        replaced by what this function returns. This allows the directive to
-        optimize the template or validate the way the directive is used.
-        """
-        return stream
-
-    def tagname(self):
-        """Return the local tag name of the directive as it is used in
-        templates.
-        """
-        return self.__class__.__name__.lower().replace('directive', '')
-    tagname = property(tagname)
-
-
 def _apply_directives(stream, ctxt, directives):
     """Apply the given directives to the stream."""
     if directives:
@@ -227,25 +171,25 @@
     def __init__(self, source, basedir=None, filename=None, loader=None,
                  encoding=None):
         """Initialize a template from either a string or a file-like object."""
-        if isinstance(source, basestring):
-            self.source = StringIO(source)
-        else:
-            self.source = source
         self.basedir = basedir
         self.filename = filename
         if basedir and filename:
             self.filepath = os.path.join(basedir, filename)
         else:
             self.filepath = filename
+        self.loader = loader
 
+        if isinstance(source, basestring):
+            source = StringIO(source)
+        else:
+            source = source
+        self.stream = list(self._prepare(self._parse(source, encoding)))
         self.filters = [self._flatten, self._eval]
 
-        self.stream = list(self._prepare(self._parse(encoding)))
-
     def __repr__(self):
         return '<%s "%s">' % (self.__class__.__name__, self.filename)
 
-    def _parse(self, encoding):
+    def _parse(self, source, encoding):
         """Parse the template.
         
         The parsing stage parses the template and constructs a list of
@@ -299,15 +243,16 @@
     _interpolate = classmethod(_interpolate)
 
     def _prepare(self, stream):
-        """Call the `prepare` method of every directive instance in the
-        template so that various optimization and validation tasks can be
-        performed.
-        """
+        """Call the `attach` method of every directive found in the template."""
         for kind, data, pos in stream:
             if kind is SUB:
-                directives, substream = data
-                for directive in directives[:]:
-                    substream = directive.prepare(directives, substream)
+                directives = []
+                substream = data[1]
+                for cls, value, namespaces, pos in data[0]:
+                    directive, substream = cls.attach(self, substream, value,
+                                                      namespaces, pos)
+                    if directive:
+                        directives.append(directive)
                 substream = self._prepare(substream)
                 if directives:
                     yield kind, (directives, list(substream)), pos
diff --git a/genshi/template/directives.py b/genshi/template/directives.py
--- a/genshi/template/directives.py
+++ b/genshi/template/directives.py
@@ -17,8 +17,8 @@
 
 from genshi.core import Attrs, Stream
 from genshi.path import Path
-from genshi.template.core import EXPR, Directive, TemplateRuntimeError, \
-                                 TemplateSyntaxError, _apply_directives
+from genshi.template.core import TemplateRuntimeError, TemplateSyntaxError, \
+                                 EXPR, _apply_directives
 from genshi.template.eval import Expression, _parse
 
 __all__ = ['AttrsDirective', 'ChooseDirective', 'ContentDirective',
@@ -27,6 +27,88 @@
            'WhenDirective', 'WithDirective']
 
 
+class DirectiveMeta(type):
+    """Meta class for template directives."""
+
+    def __new__(cls, name, bases, d):
+        d['tagname'] = name.lower().replace('directive', '')
+        return type.__new__(cls, name, bases, d)
+
+
+class Directive(object):
+    """Abstract base class for template directives.
+    
+    A directive is basically a callable that takes three positional arguments:
+    `ctxt` is the template data context, `stream` is an iterable over the
+    events that the directive applies to, and `directives` is is a list of
+    other directives on the same stream that need to be applied.
+    
+    Directives can be "anonymous" or "registered". Registered directives can be
+    applied by the template author using an XML attribute with the
+    corresponding name in the template. Such directives should be subclasses of
+    this base class that can  be instantiated with the value of the directive
+    attribute as parameter.
+    
+    Anonymous directives are simply functions conforming to the protocol
+    described above, and can only be applied programmatically (for example by
+    template filters).
+    """
+    __metaclass__ = DirectiveMeta
+    __slots__ = ['expr']
+
+    def __init__(self, value, namespaces=None, filename=None, lineno=-1,
+                 offset=-1):
+        self.expr = self._parse_expr(value, filename, lineno, offset)
+
+    def attach(cls, template, stream, value, namespaces, pos):
+        """Called after the template stream has been completely parsed.
+        
+        @param template: the `Template` object
+        @param stream: the event stream associated with the directive
+        @param value: the argument value for the directive
+        @param namespaces: a mapping of namespace URIs to prefixes
+        @param pos: a `(filename, lineno, offset)` tuple describing the location
+            where the directive was found in the source
+        
+        This class method should return a `(directive, stream)` tuple. If
+        `directive` is not `None`, it should be an instance of the `Directive`
+        class, and gets added to the list of directives applied to the substream
+        at runtime. `stream` is an event stream that replaces the original
+        stream associated with the directive.
+        """
+        return cls(value, namespaces, template.filename, *pos[1:]), stream
+    attach = classmethod(attach)
+
+    def __call__(self, stream, ctxt, directives):
+        """Apply the directive to the given stream.
+        
+        @param stream: the event stream
+        @param ctxt: the context data
+        @param directives: a list of the remaining directives that should
+            process the stream
+        """
+        raise NotImplementedError
+
+    def __repr__(self):
+        expr = ''
+        if getattr(self, 'expr', None) is not None:
+            expr = ' "%s"' % self.expr.source
+        return '<%s%s>' % (self.__class__.__name__, expr)
+
+    def _parse_expr(cls, expr, filename=None, lineno=-1, offset=-1):
+        """Parses the given expression, raising a useful error message when a
+        syntax error is encountered.
+        """
+        try:
+            return expr and Expression(expr, filename, lineno) or None
+        except SyntaxError, err:
+            err.msg += ' in expression "%s" of "%s" directive' % (expr,
+                                                                  cls.tagname)
+            raise TemplateSyntaxError(err, filename, lineno,
+                                      offset + (err.offset or 0))
+    _parse_expr = classmethod(_parse_expr)
+
+
 def _assignment(ast):
     """Takes the AST representation of an assignment, and returns a function
     that applies the assignment of a given value to a dictionary.
@@ -114,9 +196,10 @@
     """
     __slots__ = []
 
-    def prepare(self, directives, stream):
-        directives.remove(self)
-        return [stream[0], (EXPR, self.expr, (None, -1, --1)),  stream[-1]]
+    def attach(cls, template, stream, value, namespaces, pos):
+        expr = cls._parse_expr(value, template.filename, *pos[1:])
+        return None, [stream[0], (EXPR, expr, pos),  stream[-1]]
+    attach = classmethod(attach)
 
 
 class DefDirective(Directive):
@@ -319,9 +402,7 @@
                  offset=-1):
         Directive.__init__(self, None, namespaces, filename, lineno, offset)
         self.path = Path(value, filename, lineno)
-        if namespaces is None:
-            namespaces = {}
-        self.namespaces = namespaces.copy()
+        self.namespaces = namespaces or {}
 
     def __call__(self, stream, ctxt, directives):
         ctxt._match_templates.append((self.path.test(ignore_context=True),
@@ -361,9 +442,10 @@
     """
     __slots__ = []
 
-    def prepare(self, directives, stream):
-        directives.remove(self)
-        return [(EXPR, self.expr, (None, -1, -1))]
+    def attach(cls, template, stream, value, namespaces, pos):
+        expr = cls._parse_expr(value, template.filename, *pos[1:])
+        return None, [(EXPR, expr, pos)]
+    attach = classmethod(attach)
 
 
 class StripDirective(Directive):
@@ -412,11 +494,12 @@
                     yield event
         return _apply_directives(_generate(), ctxt, directives)
 
-    def prepare(self, directives, stream):
-        if not self.expr:
-            directives.remove(self)
-            return stream[1:-1]
-        return stream
+    def attach(cls, template, stream, value, namespaces, pos):
+        if not value:
+            return None, stream[1:-1]
+        return super(StripDirective, cls).attach(template, stream, value,
+                                                 namespaces, pos)
+    attach = classmethod(attach)
 
 
 class ChooseDirective(Directive):
diff --git a/genshi/template/eval.py b/genshi/template/eval.py
--- a/genshi/template/eval.py
+++ b/genshi/template/eval.py
@@ -201,14 +201,15 @@
 
 BUILTINS = __builtin__.__dict__.copy()
 BUILTINS['Undefined'] = Undefined
+_UNDEF = Undefined(None)
 
 def _lookup_name(data, name):
     __traceback_hide__ = True
-    val = data.get(name, Undefined)
-    if val is Undefined:
+    val = data.get(name, _UNDEF)
+    if val is _UNDEF:
         val = BUILTINS.get(name, val)
-        if val is Undefined:
-            return val(name)
+        if val is _UNDEF:
+            return Undefined(name)
     return val
 
 def _lookup_attr(data, obj, key):
@@ -232,8 +233,8 @@
         return obj[key]
     except (KeyError, IndexError, TypeError), e:
         if isinstance(key, basestring):
-            val = getattr(obj, key, Undefined)
-            if val is Undefined:
+            val = getattr(obj, key, _UNDEF)
+            if val is _UNDEF:
                 val = Undefined(key)
             return val
         raise
@@ -309,6 +310,12 @@
     visitUnaryAdd = visitUnarySub = visitNot = visitInvert = _visitUnaryOp
     visitBackquote = _visitUnaryOp
 
+    def visitIfExp(self, node):
+        node.test = self.visit(node.test)
+        node.then = self.visit(node.then)
+        node.else_ = self.visit(node.else_)
+        return node
+
     # Identifiers, Literals and Comprehensions
 
     def _visitDefault(self, node):
diff --git a/genshi/template/loader.py b/genshi/template/loader.py
--- a/genshi/template/loader.py
+++ b/genshi/template/loader.py
@@ -20,7 +20,6 @@
     import dummy_threading as threading
 
 from genshi.template.core import TemplateError
-from genshi.template.markup import MarkupTemplate
 from genshi.util import LRUCache
 
 __all__ = ['TemplateLoader', 'TemplateNotFound']
@@ -53,6 +52,7 @@
     template has already been loaded. If not, it attempts to locate the
     template file, and returns the corresponding `Template` object:
     
+    >>> from genshi.template import MarkupTemplate
     >>> template = loader.load(os.path.basename(path))
     >>> isinstance(template, MarkupTemplate)
     True
@@ -66,7 +66,7 @@
     >>> os.remove(path)
     """
     def __init__(self, search_path=None, auto_reload=False,
-                 default_encoding=None, max_cache_size=25):
+                 default_encoding=None, max_cache_size=25, default_class=None):
         """Create the template laoder.
         
         @param search_path: a list of absolute path names that should be
@@ -78,7 +78,11 @@
             templates; defaults to UTF-8
         @param max_cache_size: the maximum number of templates to keep in the
             cache
+        @param default_class: the default `Template` subclass to use when
+            instantiating templates
         """
+        from genshi.template.markup import MarkupTemplate
+
         self.search_path = search_path
         if self.search_path is None:
             self.search_path = []
@@ -86,12 +90,12 @@
             self.search_path = [self.search_path]
         self.auto_reload = auto_reload
         self.default_encoding = default_encoding
+        self.default_class = default_class or MarkupTemplate
         self._cache = LRUCache(max_cache_size)
         self._mtime = {}
         self._lock = threading.Lock()
 
-    def load(self, filename, relative_to=None, cls=MarkupTemplate,
-             encoding=None):
+    def load(self, filename, relative_to=None, cls=None, encoding=None):
         """Load the template with the given name.
         
         If the `filename` parameter is relative, this method searches the search
@@ -119,6 +123,8 @@
         @param encoding: the encoding of the template to load; defaults to the
             `default_encoding` of the loader instance
         """
+        if cls is None:
+            cls = self.default_class
         if encoding is None:
             encoding = self.default_encoding
         if relative_to and not os.path.isabs(relative_to):
diff --git a/genshi/template/markup.py b/genshi/template/markup.py
--- a/genshi/template/markup.py
+++ b/genshi/template/markup.py
@@ -15,12 +15,12 @@
 
 from itertools import chain
 
-from genshi.core import Attrs, Namespace, Stream
+from genshi.core import Attrs, Namespace, Stream, StreamEventKind
 from genshi.core import START, END, START_NS, END_NS, TEXT, COMMENT
-from genshi.filters import IncludeFilter
 from genshi.input import XMLParser
-from genshi.template.core import BadDirectiveError, Template, _apply_directives
-from genshi.template.core import SUB
+from genshi.template.core import BadDirectiveError, Template, \
+                                 _apply_directives, SUB
+from genshi.template.loader import TemplateNotFound
 from genshi.template.directives import *
 
 
@@ -35,7 +35,10 @@
       
  • 1
  • 2
  • 3
    • 
     
     """
-    NAMESPACE = Namespace('http://genshi.edgewall.org/')
+    INCLUDE = StreamEventKind('INCLUDE')
+
+    DIRECTIVE_NAMESPACE = Namespace('http://genshi.edgewall.org/')
+    XINCLUDE_NAMESPACE = Namespace('http://www.w3.org/2001/XInclude')
 
     directives = [('def', DefDirective),
                   ('match', MatchDirective),
@@ -58,55 +61,61 @@
 
         self.filters.append(self._match)
         if loader:
-            self.filters.append(IncludeFilter(loader))
+            self.filters.append(self._include)
 
-    def _parse(self, encoding):
+    def _parse(self, source, encoding):
         """Parse the template from an XML document."""
-        stream = [] # list of events of the "compiled" template
+        streams = [[]] # stacked lists of events of the "compiled" template
         dirmap = {} # temporary mapping of directives to elements
         ns_prefix = {}
         depth = 0
+        in_fallback = 0
+        include_href = None
 
-        for kind, data, pos in XMLParser(self.source, filename=self.filename,
-                                         encoding=encoding):
+        if not isinstance(source, Stream):
+            source = XMLParser(source, filename=self.filename,
+                               encoding=encoding)
+
+        for kind, data, pos in source:
+            stream = streams[-1]
 
             if kind is START_NS:
                 # Strip out the namespace declaration for template directives
                 prefix, uri = data
                 ns_prefix[prefix] = uri
-                if uri != self.NAMESPACE:
+                if uri not in (self.DIRECTIVE_NAMESPACE,
+                               self.XINCLUDE_NAMESPACE):
                     stream.append((kind, data, pos))
 
             elif kind is END_NS:
                 uri = ns_prefix.pop(data, None)
-                if uri and uri != self.NAMESPACE:
+                if uri and uri not in (self.DIRECTIVE_NAMESPACE,
+                                       self.XINCLUDE_NAMESPACE):
                     stream.append((kind, data, pos))
 
             elif kind is START:
                 # Record any directive attributes in start tags
-                tag, attrib = data
+                tag, attrs = data
                 directives = []
                 strip = False
 
-                if tag in self.NAMESPACE:
+                if tag in self.DIRECTIVE_NAMESPACE:
                     cls = self._dir_by_name.get(tag.localname)
                     if cls is None:
                         raise BadDirectiveError(tag.localname, self.filepath,
                                                 pos[1])
-                    value = attrib.get(getattr(cls, 'ATTRIBUTE', None), '')
-                    directives.append(cls(value, ns_prefix, self.filepath,
-                                          pos[1], pos[2]))
+                    value = attrs.get(getattr(cls, 'ATTRIBUTE', None), '')
+                    directives.append((cls, value, ns_prefix.copy(), pos))
                     strip = True
 
-                new_attrib = []
-                for name, value in attrib:
-                    if name in self.NAMESPACE:
+                new_attrs = []
+                for name, value in attrs:
+                    if name in self.DIRECTIVE_NAMESPACE:
                         cls = self._dir_by_name.get(name.localname)
                         if cls is None:
                             raise BadDirectiveError(name.localname,
                                                     self.filepath, pos[1])
-                        directives.append(cls(value, ns_prefix, self.filepath,
-                                              pos[1], pos[2]))
+                        directives.append((cls, value, ns_prefix.copy(), pos))
                     else:
                         if value:
                             value = list(self._interpolate(value, self.basedir,
@@ -115,20 +124,40 @@
                                 value = value[0][1]
                         else:
                             value = [(TEXT, u'', pos)]
-                        new_attrib.append((name, value))
+                        new_attrs.append((name, value))
+                new_attrs = Attrs(new_attrs)
 
                 if directives:
                     index = self._dir_order.index
-                    directives.sort(lambda a, b: cmp(index(a.__class__),
-                                                     index(b.__class__)))
+                    directives.sort(lambda a, b: cmp(index(a[0]), index(b[0])))
                     dirmap[(depth, tag)] = (directives, len(stream), strip)
 
-                stream.append((kind, (tag, Attrs(new_attrib)), pos))
+                if tag in self.XINCLUDE_NAMESPACE:
+                    if tag.localname == 'include':
+                        include_href = new_attrs.get('href')
+                        if not include_href:
+                            raise TemplateSyntaxError('Include misses required '
+                                                      'attribute "href"', *pos)
+                        streams.append([])
+                    elif tag.localname == 'fallback':
+                        in_fallback += 1
+
+                else:
+                    stream.append((kind, (tag, new_attrs), pos))
+
                 depth += 1
 
             elif kind is END:
                 depth -= 1
-                stream.append((kind, data, pos))
+
+                if in_fallback and data == self.XINCLUDE_NAMESPACE['fallback']:
+                    in_fallback -= 1
+                elif data == self.XINCLUDE_NAMESPACE['include']:
+                    fallback = streams.pop()
+                    stream = streams[-1]
+                    stream.append((INCLUDE, (include_href, fallback), pos))
+                else:
+                    stream.append((kind, data, pos))
 
                 # If there have have directive attributes with the corresponding
                 # start tag, move the events inbetween into a "subprogram"
@@ -152,7 +181,41 @@
             else:
                 stream.append((kind, data, pos))
 
-        return stream
+        assert len(streams) == 1
+        return streams[0]
+
+    def _prepare(self, stream):
+        for kind, data, pos in Template._prepare(self, stream):
+            if kind is INCLUDE:
+                data = data[0], list(self._prepare(data[1]))
+            yield kind, data, pos
+
+    def _include(self, stream, ctxt):
+        """Internal stream filter that performs inclusion of external
+        template files.
+        """
+        for event in stream:
+            if event[0] is INCLUDE:
+                href, fallback = event[1]
+                if not isinstance(href, basestring):
+                    parts = []
+                    for subkind, subdata, subpos in self._eval(href, ctxt):
+                        if subkind is TEXT:
+                            parts.append(subdata)
+                    href = u''.join([x for x in parts if x is not None])
+                try:
+                    tmpl = self.loader.load(href, relative_to=event[2][0])
+                    for event in tmpl.generate(ctxt):
+                        yield event
+                except TemplateNotFound:
+                    if fallback is None:
+                        raise
+                    for filter_ in self.filters:
+                        fallback = filter_(iter(fallback), ctxt)
+                    for event in fallback:
+                        yield event
+            else:
+                yield event
 
     def _match(self, stream, ctxt, match_templates=None):
         """Internal stream filter that applies any defined match templates
@@ -197,11 +260,11 @@
 
                     # Consume and store all events until an end event
                     # corresponding to this start event is encountered
-                    content = chain([event], self._match(_strip(stream), ctxt),
+                    content = chain([event],
+                                    self._match(_strip(stream), ctxt,
+                                                [match_templates[idx]]),
                                     tail)
-                    for filter_ in self.filters[3:]:
-                        content = filter_(content, ctxt)
-                    content = list(content)
+                    content = list(self._include(content, ctxt))
 
                     for test in [mt[0] for mt in match_templates]:
                         test(tail[0], namespaces, ctxt, updateonly=True)
@@ -226,3 +289,6 @@
 
             else: # no matches
                 yield event
+
+
+INCLUDE = MarkupTemplate.INCLUDE
diff --git a/genshi/template/plugin.py b/genshi/template/plugin.py
--- a/genshi/template/plugin.py
+++ b/genshi/template/plugin.py
@@ -59,7 +59,8 @@
 
         self.loader = TemplateLoader(filter(None, search_path),
                                      auto_reload=auto_reload,
-                                     max_cache_size=max_cache_size)
+                                     max_cache_size=max_cache_size,
+                                     default_class=self.template_class)
 
     def load_template(self, templatename, template_string=None):
         """Find a template specified in python 'dot' notation, or load one from
@@ -74,7 +75,7 @@
             basename = templatename[divider + 1:] + self.extension
             templatename = resource_filename(package, basename)
 
-        return self.loader.load(templatename, cls=self.template_class)
+        return self.loader.load(templatename)
 
     def _get_render_options(self, format=None):
         if format is None:
diff --git a/genshi/template/tests/directives.py b/genshi/template/tests/directives.py
--- a/genshi/template/tests/directives.py
+++ b/genshi/template/tests/directives.py
@@ -380,7 +380,8 @@
           #end
           ${echo('Hi', name='you')}
         """)
-        self.assertEqual("""                      Hi, you!
+        self.assertEqual("""
+                      Hi, you!
         """, str(tmpl.generate()))
 
 
@@ -599,6 +600,54 @@
           
         """, str(tmpl.generate()))
 
+    def test_not_match_self(self):
+        """
+        See http://genshi.edgewall.org/ticket/77
+        """
+        tmpl = MarkupTemplate("""
+          
+          
    
+            ${select('text()')}
+            Goodbye!
+          
    
+          
+            
    Hello!
    
+          
+        """)
+        self.assertEqual("""
+          
    
+            Hello!
+            Goodbye!
+          
    
+        """, str(tmpl.generate()))
+
+    def test_select_text_in_element(self):
+        """
+        See http://genshi.edgewall.org/ticket/77#comment:1
+        """
+        tmpl = MarkupTemplate("""
+          
+          
    
+            
+              ${select('text()')}
+            
+            Goodbye!
+          
    
+          
+            
    Hello!
    
+          
+        """)
+        self.assertEqual("""
+          
    
+            
+              Hello!
+            
+            Goodbye!
+          
    
+        """, str(tmpl.generate()))
+
     def test_select_all_attrs(self):
         tmpl = MarkupTemplate("""
           
    
diff --git a/genshi/template/tests/eval.py b/genshi/template/tests/eval.py
--- a/genshi/template/tests/eval.py
+++ b/genshi/template/tests/eval.py
@@ -278,6 +278,12 @@
             expr = Expression("list(i['name'] for i in items if i['value'] > 1)")
             self.assertEqual(['b'], expr.evaluate({'items': items}))
 
+    if sys.version_info >= (2, 5):
+        def test_conditional_expression(self):
+            expr = Expression("'T' if foo else 'F'")
+            self.assertEqual('T', expr.evaluate({'foo': True}))
+            self.assertEqual('F', expr.evaluate({'foo': False}))
+
     def test_slice(self):
         expr = Expression("numbers[0:2]")
         self.assertEqual([0, 1], expr.evaluate({'numbers': range(5)}))
diff --git a/genshi/template/tests/markup.py b/genshi/template/tests/markup.py
--- a/genshi/template/tests/markup.py
+++ b/genshi/template/tests/markup.py
@@ -12,17 +12,33 @@
 # history and logs, available at http://genshi.edgewall.org/log/.
 
 import doctest
+import os
+import shutil
+from StringIO import StringIO
 import sys
+import tempfile
 import unittest
 
 from genshi.core import Markup
+from genshi.input import XML
 from genshi.template.core import BadDirectiveError, TemplateSyntaxError
+from genshi.template.loader import TemplateLoader
 from genshi.template.markup import MarkupTemplate
 
 
 class MarkupTemplateTestCase(unittest.TestCase):
     """Tests for markup template processing."""
 
+    def test_parse_fileobj(self):
+        fileobj = StringIO(' ${var} $var')
+        tmpl = MarkupTemplate(fileobj)
+        self.assertEqual(' 42 42', str(tmpl.generate(var=42)))
+
+    def test_parse_stream(self):
+        stream = XML(' ${var} $var')
+        tmpl = MarkupTemplate(stream)
+        self.assertEqual(' 42 42', str(tmpl.generate(var=42)))
+
     def test_interpolate_mixed3(self):
         tmpl = MarkupTemplate(' ${var} $var')
         self.assertEqual(' 42 42', str(tmpl.generate(var=42)))
@@ -179,6 +195,211 @@
           \xf6
         
    """, unicode(tmpl.generate()))
 
+    def test_include_in_loop(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
+            try:
+                file1.write("""
    Included $idx
    """)
+            finally:
+                file1.close()
+
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""
+                  
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl2.html')
+            self.assertEqual("""
+                  
    Included 0
    Included 1
    Included 2
    
+                """, tmpl.generate(name='tmpl1').render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_dynamic_inlude_href(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
+            try:
+                file1.write("""
    Included
    """)
+            finally:
+                file1.close()
+
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""
+                  
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl2.html')
+            self.assertEqual("""
+                  
    Included
    
+                """, tmpl.generate(name='tmpl1').render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_select_inluded_elements(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
+            try:
+                file1.write("""
  • $item
    • """)
+            finally:
+                file1.close()
+
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""
+                  
      ${select('li')}
    
+                  
      
+                    
+                  
    
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl2.html')
+            self.assertEqual("""
+                  
    • 1
    • 2
    • 3
    
+                """, tmpl.generate().render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_fallback_when_include_found(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
+            try:
+                file1.write("""
    Included
    """)
+            finally:
+                file1.close()
+
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""
+                  
+                    Missing
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl2.html')
+            self.assertEqual("""
+                  
    Included
    
+                """, tmpl.generate().render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_fallback_when_include_not_found(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""
+                  
+                  Missing
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl2.html')
+            self.assertEqual("""
+                  Missing
+                """, tmpl.generate().render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_include_in_fallback(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
+            try:
+                file1.write("""
    Included
    """)
+            finally:
+                file1.close()
+
+            file2 = open(os.path.join(dirname, 'tmpl3.html'), 'w')
+            try:
+                file2.write("""
+                  
+                    
+                      
+                        Missing
+                      
+                    
+                  
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl3.html')
+            self.assertEqual("""
+                  
    Included
    
+                """, tmpl.generate().render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_nested_include_fallback(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file2 = open(os.path.join(dirname, 'tmpl3.html'), 'w')
+            try:
+                file2.write("""
+                  
+                    
+                      
+                        Missing
+                      
+                    
+                  
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl3.html')
+            self.assertEqual("""
+                        Missing
+                """, tmpl.generate().render())
+        finally:
+            shutil.rmtree(dirname)
+
+    def test_include_fallback_with_directive(self):
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""
+                  
+                    tmpl1.html not found
+                  
+                """)
+            finally:
+                file2.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl2.html')
+            self.assertEqual("""
+                    tmpl1.html not found
+                """, tmpl.generate(debug=True).render())
+        finally:
+            shutil.rmtree(dirname)
+
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/template/tests/text.py b/genshi/template/tests/text.py
--- a/genshi/template/tests/text.py
+++ b/genshi/template/tests/text.py
@@ -37,26 +37,42 @@
         #if foo
           bar
         #end 'if foo'""")
-        self.assertEqual('', str(tmpl.generate()))
+        self.assertEqual('\n', str(tmpl.generate()))
 
     def test_latin1_encoded(self):
         text = u'$foo\xf6$bar'.encode('iso-8859-1')
         tmpl = TextTemplate(text, encoding='iso-8859-1')
         self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y')))
 
-    # FIXME
-    #def test_empty_lines(self):
-    #    tmpl = TextTemplate("""Your items:
-    #
-    #    #for item in items
-    #      * ${item}
-    #
-    #    #end""")
-    #    self.assertEqual("""Your items:
-    #      * 0
-    #      * 1
-    #      * 2
-    #    """, tmpl.generate(items=range(3)).render('text'))
+    def test_empty_lines1(self):
+        tmpl = TextTemplate("""Your items:
+
+        #for item in items
+          * ${item}
+        #end""")
+        self.assertEqual("""Your items:
+
+          * 0
+          * 1
+          * 2
+""", tmpl.generate(items=range(3)).render('text'))
+
+    def test_empty_lines2(self):
+        tmpl = TextTemplate("""Your items:
+
+        #for item in items
+          * ${item}
+
+        #end""")
+        self.assertEqual("""Your items:
+
+          * 0
+
+          * 1
+
+          * 2
+
+""", tmpl.generate(items=range(3)).render('text'))
 
 
 def suite():
diff --git a/genshi/template/text.py b/genshi/template/text.py
--- a/genshi/template/text.py
+++ b/genshi/template/text.py
@@ -50,9 +50,11 @@
                   ('choose', ChooseDirective),
                   ('with', WithDirective)]
 
-    _DIRECTIVE_RE = re.compile(r'^\s*(?Foo')).generate())
+        self.assertEqual(5, len(xml))
+        self.assertEqual((Stream.START, ('span', ())), xml[0][:2])
+        self.assertEqual((Stream.START, ('b', ())), xml[1][:2])
+        self.assertEqual((Stream.TEXT, 'Foo'), xml[2][:2])
+        self.assertEqual((Stream.END, 'b'), xml[3][:2])
+        self.assertEqual((Stream.END, 'span'), xml[4][:2])
+
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/tests/core.py b/genshi/tests/core.py
--- a/genshi/tests/core.py
+++ b/genshi/tests/core.py
@@ -48,7 +48,7 @@
 
     def test_repr(self):
         markup = Markup('foo')
-        self.assertEquals('', repr(markup))
+        self.assertEquals("", repr(markup))
 
     def test_escape(self):
         markup = escape('"&"')
@@ -131,7 +131,7 @@
         buf = StringIO()
         pickle.dump(markup, buf, 2)
         buf.seek(0)
-        self.assertEquals('', repr(pickle.load(buf)))
+        self.assertEquals("", repr(pickle.load(buf)))
 
 
 class NamespaceTestCase(unittest.TestCase):
diff --git a/genshi/tests/filters.py b/genshi/tests/filters.py
--- a/genshi/tests/filters.py
+++ b/genshi/tests/filters.py
@@ -12,16 +12,11 @@
 # history and logs, available at http://genshi.edgewall.org/log/.
 
 import doctest
-import os
-import shutil
-import tempfile
 import unittest
 
 from genshi import filters
-from genshi.core import Stream
 from genshi.input import HTML, ParseError
 from genshi.filters import HTMLFormFiller, HTMLSanitizer
-from genshi.template import TemplateLoader
 
 
 class HTMLFormFillerTestCase(unittest.TestCase):
@@ -374,46 +369,11 @@
         self.assertEquals(u'', unicode(html | HTMLSanitizer()))
 
 
-class IncludeFilterTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.dirname = tempfile.mkdtemp(suffix='markup_test')
-
-    def tearDown(self):
-        shutil.rmtree(self.dirname)
-
-    def test_select_inluded_elements(self):
-        file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w')
-        try:
-            file1.write("""
  • $item
    • """)
-        finally:
-            file1.close()
-
-        file2 = open(os.path.join(self.dirname, 'tmpl2.html'), 'w')
-        try:
-            file2.write("""
-              
      ${select('li')}
    
-              
      
-                
-              
    
-            """)
-        finally:
-            file2.close()
-
-        loader = TemplateLoader([self.dirname])
-        tmpl = loader.load('tmpl2.html')
-        self.assertEqual("""
-              
    • 1
    • 2
    • 3
    
-            """, tmpl.generate().render())
-
-
 def suite():
     suite = unittest.TestSuite()
     suite.addTest(doctest.DocTestSuite(filters))
     suite.addTest(unittest.makeSuite(HTMLFormFillerTestCase, 'test'))
     suite.addTest(unittest.makeSuite(HTMLSanitizerTestCase, 'test'))
-    suite.addTest(unittest.makeSuite(IncludeFilterTestCase, 'test'))
     return suite
 
 if __name__ == '__main__':
diff --git a/genshi/tests/input.py b/genshi/tests/input.py
--- a/genshi/tests/input.py
+++ b/genshi/tests/input.py
@@ -16,7 +16,7 @@
 import sys
 import unittest
 
-from genshi.core import Stream
+from genshi.core import Attrs, Stream
 from genshi.input import XMLParser, HTMLParser, ParseError
 
 
@@ -173,6 +173,55 @@
         self.assertEqual(Stream.TEXT, kind)
         self.assertEqual(u'\xa0', data)
 
+    def test_processing_instruction(self):
+        text = ''
+        events = list(HTMLParser(StringIO(text)))
+        kind, (target, data), pos = events[0]
+        self.assertEqual(Stream.PI, kind)
+        self.assertEqual(u'php', target)
+        self.assertEqual(u'echo "Foobar"', data)
+
+    def test_processing_instruction_trailing_qmark(self):
+        text = ''
+        events = list(HTMLParser(StringIO(text)))
+        kind, (target, data), pos = events[0]
+        self.assertEqual(Stream.PI, kind)
+        self.assertEqual(u'php', target)
+        self.assertEqual(u'echo "Foobar" ?', data)
+
+    def test_out_of_order_tags1(self):
+        text = 'Foobar'
+        events = list(HTMLParser(StringIO(text)))
+        self.assertEqual(5, len(events))
+        self.assertEqual((Stream.START, ('span', ())), events[0][:2])
+        self.assertEqual((Stream.START, ('b', ())), events[1][:2])
+        self.assertEqual((Stream.TEXT, 'Foobar'), events[2][:2])
+        self.assertEqual((Stream.END, 'b'), events[3][:2])
+        self.assertEqual((Stream.END, 'span'), events[4][:2])
+
+    def test_out_of_order_tags2(self):
+        text = 'Foobar'
+        events = list(HTMLParser(StringIO(text)))
+        self.assertEqual(7, len(events))
+        self.assertEqual((Stream.START, ('span', Attrs([('class', 'baz')]))),
+                         events[0][:2])
+        self.assertEqual((Stream.START, ('b', ())), events[1][:2])
+        self.assertEqual((Stream.START, ('i', ())), events[2][:2])
+        self.assertEqual((Stream.TEXT, 'Foobar'), events[3][:2])
+        self.assertEqual((Stream.END, 'i'), events[4][:2])
+        self.assertEqual((Stream.END, 'b'), events[5][:2])
+        self.assertEqual((Stream.END, 'span'), events[6][:2])
+
+    def test_out_of_order_tags3(self):
+        text = 'Foobar'
+        events = list(HTMLParser(StringIO(text)))
+        self.assertEqual(5, len(events))
+        self.assertEqual((Stream.START, ('span', ())), events[0][:2])
+        self.assertEqual((Stream.START, ('b', ())), events[1][:2])
+        self.assertEqual((Stream.TEXT, 'Foobar'), events[2][:2])
+        self.assertEqual((Stream.END, 'b'), events[3][:2])
+        self.assertEqual((Stream.END, 'span'), events[4][:2])
+
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/tests/path.py b/genshi/tests/path.py
--- a/genshi/tests/path.py
+++ b/genshi/tests/path.py
@@ -438,6 +438,19 @@
         self.assertEqual('bar',
                          path.select(xml, namespaces=namespaces).render())
 
+    def test_predicate_termination(self):
+        """
+        Verify that a patch matching the self axis with a predicate doesn't
+        cause an infinite loop. See .
+        """
+        xml = XML('
    • a
    • b
    ')
+        path = Path('.[@flag="1"]/*')
+        self.assertEqual('
  • a
  • b
    • ', path.select(xml).render())
+
+        xml = XML('
    • a
    • b
    ')
+        path = Path('.[@flag="0"]/*')
+        self.assertEqual('', path.select(xml).render())
+
     # FIXME: the following two don't work due to a problem in XML serialization:
     #        attributes that would need a namespace prefix that isn't in the
     #        prefix map would need to get an artificial prefix, but currently
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -12,11 +12,55 @@
 # individuals. For the exact contribution history, see the revision
 # history and logs, available at http://genshi.edgewall.org/log/.
 
+from distutils.cmd import Command
+import doctest
+from glob import glob
+import os
 try:
     from setuptools import setup
 except ImportError:
     from distutils.core import setup
 
+
+class build_doc(Command):
+    description = 'Builds the documentation'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        from docutils.core import publish_cmdline
+        conf = os.path.join('doc', 'docutils.conf')
+
+        for source in glob('doc/*.txt'):
+            dest = os.path.splitext(source)[0] + '.html'
+            if not os.path.exists(dest) or \
+                   os.path.getmtime(dest) < os.path.getmtime(source):
+                print 'building documentation file %s' % dest
+                publish_cmdline(writer_name='html',
+                                argv=['--config=%s' % conf, source, dest])
+
+
+class test_doc(Command):
+    description = 'Tests the code examples in the documentation'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        for filename in glob('doc/*.txt'):
+            print 'testing documentation file %s' % filename
+            doctest.testfile(filename, False, optionflags=doctest.ELLIPSIS)
+
+
 setup(
     name = 'Genshi',
     version = '0.4',
@@ -56,4 +100,6 @@
     genshi-markup = genshi.template.plugin:MarkupTemplateEnginePlugin[plugin]
     genshi-text = genshi.template.plugin:TextTemplateEnginePlugin[plugin]
     """,
+
+    cmdclass={'build_doc': build_doc, 'test_doc': test_doc}
 )