# HG changeset patch
# User aflett
# Date 1216682272 0
# Node ID 67d324a62cc00cddbbc9ed9277c05d2911f8537f
# Parent  8f2c7023af9434452404b9326dd7c279ae019f57
update to 0.5.x branch, up through r907
don't know how this fits in with SoC work, but I wanted to do due diligence and keep this branch working in case it someday gets considered for trunk

diff --git a/COPYING b/COPYING
--- a/COPYING
+++ b/COPYING
@@ -1,4 +1,4 @@
-Copyright (C) 2006-2007 Edgewall Software
+Copyright (C) 2006-2008 Edgewall Software
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,26 @@
+Version 0.5.1
+http://svn.edgewall.org/repos/genshi/tags/0.5.1/
+(Jul 9 2008, from branches/stable/0.5.x)
+
+ * Fix problem with nested match templates not being applied when buffering
+   on the outer `py:match` is disabled. Thanks to Erik Bray for reporting the
+   problem and providing a test case!
+ * Fix problem in `Translator` filter that would cause the translation of
+   text nodes to fail if the translation function returned an object that was
+   not directly a string, but rather something like an instance of the
+   `LazyProxy` class in Babel (ticket #145).
+ * Fix problem with match templates incorrectly being applied multiple times.
+ * Includes from templates loaded via an absolute path now include the correct
+   file in nested directories as long if no search path has been configured
+   (ticket #240).
+ * Unbuffered match templates could result in parts of the matched content
+   being included in the output if the match template didn't actually consume
+   it via one or more calls to the `select()` function (ticket #243).
+
+
 Version 0.5
 http://svn.edgewall.org/repos/genshi/tags/0.5.0/
-(?, from branches/stable/0.5.x)
+(Jun 9 2008, from branches/stable/0.5.x)
 
  * Added #include directive for text templates (ticket #115).
  * Added new markup transformation filter contributed by Alec Thomas. This
@@ -79,6 +99,12 @@
    base class (ticket #211).
  * The `Template` class and its subclasses, as well as the interpolation API,
    now take an `filepath` parameter instead of `basedir` (ticket #207).
+ * The `XHTMLSerializer` now has a `drop_xml_decl` option that defaults to
+   `True`. Setting it to `False` will cause any XML decl in the serialized
+   stream to be included in the output as it would for XML serialization.
+ * Add support for a protocol that would allow interoperability of different
+   Python packages that generate and/or consume markup, based on the special
+   `__html__()` method (ticket #202).
 
 
 Version 0.4.4
diff --git a/doc/plugin.txt b/doc/plugin.txt
--- a/doc/plugin.txt
+++ b/doc/plugin.txt
@@ -22,12 +22,11 @@
 Introduction
 ============
 
-Most Python web frameworks (with the notable exception of Django_) support
-a variety of different templating engines through the `Template Engine Plugin
-API`_, which was first developed by the Buffet_ and TurboGears_ projects.
+Some Python web frameworks support a variety of different templating engines
+through the `Template Engine Plugin API`_, which was first developed by the
+Buffet_ and TurboGears_ projects.
 
 .. _`Template Engine Plugin API`: http://docs.turbogears.org/1.0/TemplatePlugins
-.. _`Django`: http://www.djangoproject.com/
 .. _`Buffet`: http://projects.dowski.com/projects/buffet
 .. _`TurboGears`: http://www.turbogears.org/
 
@@ -72,8 +71,31 @@
 format when you want to produce an Atom feed or other XML content.
 
 
+Template Paths
+--------------
+
+How you specify template paths depends on whether you have a `search path`_ set
+up or not. The search path is a list of directories that Genshi should load
+templates from. Now when you request a template using a relative path such as
+``mytmpl.html`` or ``foo/mytmpl.html``, Genshi will look for that file in the
+directories on the search path.
+
+For mostly historical reasons, the Genshi template engine plugin uses a
+different approach when you **haven't** configured the template search path:
+you now load templates using *dotted notation*, for example ``mytmpl`` or
+``foo.mytmpl``.  Note how you've lost the ability to explicitly specify the
+file extension: you now have to use ``.html`` for markup templates, and
+``.txt`` for text templates.
+
+Using the search path is recommended for a number of reasons: First, it's
+the native Genshi model and is thus more robust and better supported.
+Second, a search path gives you much more flexibility for organizing your
+application templates. And as noted above, you aren't forced to use hardcoded
+filename extensions for your template files.
+
+
 Extra Implicit Objects
-======================
+----------------------
 
 The "genshi-markup" template engine plugin adds some extra functions that are
 made available to all templates implicitly, namely:
@@ -230,6 +252,8 @@
 In the version of Genshi, the default is to use the old syntax for
 backwards-compatibility, but that will change in a future release.
 
+.. _`search path`:
+
 ``genshi.search_path``
 ----------------------
 A colon-separated list of file-system path names that the template loader should
diff --git a/doc/streams.txt b/doc/streams.txt
--- a/doc/streams.txt
+++ b/doc/streams.txt
@@ -8,7 +8,7 @@
 
 
 .. contents:: Contents
-   :depth: 1
+   :depth: 2
 .. sectnum::
 
 
@@ -132,10 +132,11 @@
 events, which you'll need when you want to transmit or store the results of
 generating or otherwise processing markup.
 
-The ``Stream`` class provides two methods for serialization: ``serialize()`` and
-``render()``. The former is a generator that yields chunks of ``Markup`` objects
-(which are basically unicode strings that are considered safe for output on the
-web). The latter returns a single string, by default UTF-8 encoded.
+The ``Stream`` class provides two methods for serialization: ``serialize()``
+and ``render()``. The former is a generator that yields chunks of ``Markup``
+objects (which are basically unicode strings that are considered safe for
+output on the web). The latter returns a single string, by default UTF-8
+encoded.
 
 Here's the output from ``serialize()``:
 
@@ -161,8 +162,8 @@
   <p class="intro">Some text and <a href="http://example.org/">a link</a>.<br/></p>
 
 Both methods can be passed a ``method`` parameter that determines how exactly
-the events are serialzed to text. This parameter can be either “xml” (the
-default), “xhtml”, “html”, “text”, or a custom serializer class:
+the events are serialized to text. This parameter can be either a string or a 
+custom serializer class:
 
 .. code-block:: pycon
 
@@ -170,7 +171,7 @@
   <p class="intro">Some text and <a href="http://example.org/">a link</a>.<br></p>
 
 Note how the `<br>` element isn't closed, which is the right thing to do for
-HTML.
+HTML. See  `serialization methods`_ for more details.
 
 In addition, the ``render()`` method takes an ``encoding`` parameter, which
 defaults to “UTF-8”. If set to ``None``, the result will be a unicode string.
@@ -193,6 +194,54 @@
   Some text and a link.
 
 
+.. _`serialization methods`:
+
+Serialization Methods
+---------------------
+
+Genshi supports the use of different serialization methods to use for creating
+a text representation of a markup stream.
+
+``xml``
+  The ``XMLSerializer`` is the default serialization method and results in
+  proper XML output including namespace support, the XML declaration, CDATA
+  sections, and so on. It is not generally not suitable for serving HTML or
+  XHTML web pages (unless you want to use true XHTML 1.1), for which the
+  ``xhtml`` and ``html`` serializers described below should be preferred.
+
+``xhtml``
+  The ``XHTMLSerializer`` is a specialization of the generic ``XMLSerializer``
+  that understands the pecularities of producing XML-compliant output that can
+  also be parsed without problems by the HTML parsers found in modern web
+  browsers. Thus, the output by this serializer should be usable whether sent
+  as "text/html" or "application/xhtml+html" (although there are a lot of
+  subtle issues to pay attention to when switching between the two, in
+  particular with respect to differences in the DOM and CSS).
+
+  For example, instead of rendering a script tag as ``<script/>`` (which
+  confuses the HTML parser in many browsers), it will produce
+  ``<script></script>``. Also, it will normalize any boolean attributes values
+  that are minimized in HTML, so that for example ``<hr noshade="1"/>``
+  becomes ``<hr noshade="noshade" />``.
+
+  This serializer supports the use of namespaces for compound documents, for
+  example to use inline SVG inside an XHTML document.
+
+``html``
+  The ``HTMLSerializer`` produces proper HTML markup. The main differences
+  compared to ``xhtml`` serialization are that boolean attributes are
+  minimized, empty tags are not self-closing (so it's ``<br>`` instead of
+  ``<br />``), and that the contents of ``<script>`` and ``<style>`` elements
+  are not escaped.
+
+``text``
+  The ``TextSerializer`` produces plain text from markup streams. This is
+  useful primarily for `text templates`_, but can also be used to produce
+  plain text output from markup templates or other sources.
+
+.. _`text templates`: text-templates.html
+
+
 Serialization Options
 ---------------------
 
@@ -201,8 +250,8 @@
 options are supported by the built-in serializers:
 
 ``strip_whitespace``
-  Whether the serializer should remove trailing spaces and empty lines. Defaults
-  to ``True``.
+  Whether the serializer should remove trailing spaces and empty lines.
+  Defaults to ``True``.
 
   (This option is not available for serialization to plain text.)
 
@@ -212,6 +261,38 @@
   output. If provided, this declaration will override any ``DOCTYPE``
   declaration in the stream.
 
+  The parameter can also be specified as a string to refer to commonly used
+  doctypes:
+  
+  +-----------------------------+-------------------------------------------+
+  | Shorthand                   | DOCTYPE                                   |
+  +=============================+===========================================+
+  | ``html`` or                 | HTML 4.01 Strict                          |
+  | ``html-strict``             |                                           |
+  +-----------------------------+-------------------------------------------+
+  | ``html-transitional``       | HTML 4.01 Transitional                    |
+  +-----------------------------+-------------------------------------------+
+  | ``html-frameset``           | HTML 4.01 Frameset                        |
+  +-----------------------------+-------------------------------------------+
+  | ``html5``                   | DOCTYPE proposed for the work-in-progress |
+  |                             | HTML5 standard                            |
+  +-----------------------------+-------------------------------------------+
+  | ``xhtml`` or                | XHTML 1.0 Strict                          |
+  | ``xhtml-strict``            |                                           |
+  +-----------------------------+-------------------------------------------+
+  | ``xhtml-transitional``      | XHTML 1.0 Transitional                    |
+  +-----------------------------+-------------------------------------------+
+  | ``xhtml-frameset``          | XHTML 1.0 Frameset                        |
+  +-----------------------------+-------------------------------------------+
+  | ``xhtml11``                 | XHTML 1.1                                 |
+  +-----------------------------+-------------------------------------------+
+  | ``svg`` or ``svg-full``     | SVG 1.1                                   |
+  +-----------------------------+-------------------------------------------+
+  | ``svg-basic``               | SVG 1.1 Basic                             |
+  +-----------------------------+-------------------------------------------+
+  | ``svg-tiny``                | SVG 1.1 Tiny                              |
+  +-----------------------------+-------------------------------------------+
+
   (This option is not available for serialization to plain text.)
 
 ``namespace_prefixes``
@@ -220,6 +301,19 @@
 
   (This option is not available for serialization to HTML or plain text.)
 
+``drop_xml_decl``
+  Whether to remove the XML declaration (the ``<?xml ?>`` part at the
+  beginning of a document) when serializing. This defaults to ``True`` as an
+  XML declaration throws some older browsers into "Quirks" rendering mode.
+
+  (This option is only available for serialization to XHTML.)
+
+``strip_markup``
+  Whether the text serializer should detect and remove any tags or entity
+  encoded characters in the text.
+
+  (This option is only available for serialization to plain text.)
+
 
 
 Using XPath
@@ -285,7 +379,7 @@
 
 .. code-block:: python
 
-  START, (QName(u'p'), Attrs([(u'class', u'intro')])), pos
+  START, (QName(u'p'), Attrs([(QName(u'class'), u'intro')])), pos
 
 END
 ---
diff --git a/doc/upgrade.txt b/doc/upgrade.txt
--- a/doc/upgrade.txt
+++ b/doc/upgrade.txt
@@ -56,7 +56,7 @@
 ``Markup`` Constructor
 ----------------------
 
-The ``Markup`` class now longer has a specialized constructor. The old
+The ``Markup`` class no longer has a specialized constructor. The old
 (undocumented) constructor provided a shorthand for doing positional
 substitutions. If you have code like this:
 
@@ -64,7 +64,7 @@
 
   Markup('<b>%s</b>', name)
 
-You can simply replace it by the more explicit:
+You must replace it by the more explicit:
 
 .. code-block:: python
 
diff --git a/doc/xml-templates.txt b/doc/xml-templates.txt
--- a/doc/xml-templates.txt
+++ b/doc/xml-templates.txt
@@ -101,7 +101,8 @@
 ``py:if``
 ---------
 
-The element is only rendered if the expression evaluates to a truth value:
+The element and its content is only rendered if the expression evaluates to a
+truth value:
 
 .. code-block:: genshi
 
@@ -118,6 +119,13 @@
     <b>Hello</b>
   </div>
 
+But setting ``foo=False`` would result in the following output:
+
+.. code-block:: xml
+
+  <div>
+  </div>
+
 This directive can also be used as an element:
 
 .. code-block:: genshi
@@ -178,6 +186,15 @@
     <span>1</span>
   </div>
 
+These directives can also be used as elements:
+
+.. code-block:: genshi
+
+  <py:choose test="1">
+    <py:when test="0">0</py:when>
+    <py:when test="1">1</py:when>
+    <py:otherwise>2</py:otherwise>
+  </py:choose>
 
 Looping
 =======
diff --git a/genshi/__init__.py b/genshi/__init__.py
--- a/genshi/__init__.py
+++ b/genshi/__init__.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2006-2007 Edgewall Software
+# Copyright (C) 2006-2008 Edgewall Software
 # All rights reserved.
 #
 # This software is licensed as described in the file COPYING, which
@@ -21,9 +21,13 @@
 
 __docformat__ = 'restructuredtext en'
 try:
-    __version__ = __import__('pkg_resources').get_distribution('Genshi').version
+    from pkg_resources import get_distribution, ResolutionError
+    try:
+        __version__ = get_distribution('Genshi').version
+    except ResolutionError:
+        __version__ = None # unknown
 except ImportError:
-    pass
+    __version__ = None # unknown
 
 from genshi.core import *
 from genshi.input import ParseError, XML, HTML
diff --git a/genshi/_speedups.c b/genshi/_speedups.c
--- a/genshi/_speedups.c
+++ b/genshi/_speedups.c
@@ -61,6 +61,18 @@
         Py_INCREF(text);
         return text;
     }
+    if (PyObject_HasAttrString(text, "__html__")) {
+        ret = PyObject_CallMethod(text, "__html__", NULL);
+        args = PyTuple_New(1);
+        if (args == NULL) {
+            Py_DECREF(ret);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(args, 0, ret);
+        ret = MarkupType.tp_new(&MarkupType, args, NULL);
+        Py_DECREF(args);
+        return ret;
+    }
     in = (PyUnicodeObject *) PyObject_Unicode(text);
     if (in == NULL) {
         return NULL;
@@ -191,6 +203,13 @@
     return escape(text, quotes);
 }
 
+static PyObject *
+Markup_html(PyObject *self)
+{
+    Py_INCREF(self);
+    return self;
+}
+
 PyDoc_STRVAR(join__doc__,
 "Return a `Markup` object which is the concatenation of the strings\n\
 in the given sequence, where this `Markup` object is the separator\n\
@@ -520,6 +539,7 @@
 } MarkupObject;
 
 static PyMethodDef Markup_methods[] = {
+    {"__html__", (PyCFunction) Markup_html, METH_NOARGS, NULL},
     {"escape", (PyCFunction) Markup_escape,
      METH_VARARGS|METH_CLASS|METH_KEYWORDS, escape__doc__},
     {"join", (PyCFunction)Markup_join, METH_VARARGS|METH_KEYWORDS, join__doc__},
diff --git a/genshi/builder.py b/genshi/builder.py
--- a/genshi/builder.py
+++ b/genshi/builder.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2006-2007 Edgewall Software
+# Copyright (C) 2006-2008 Edgewall Software
 # All rights reserved.
 #
 # This software is licensed as described in the file COPYING, which
@@ -68,7 +68,13 @@
 Hello, <em>world</em>!
 """
 
-from genshi.core import Attrs, Namespace, QName, Stream, START, END, TEXT
+try:
+    set
+except NameError:
+    from sets import Set as set
+
+from genshi.core import Attrs, Markup, Namespace, QName, Stream, \
+                        START, END, TEXT
 
 __all__ = ['Fragment', 'Element', 'ElementFactory', 'tag']
 __docformat__ = 'restructuredtext en'
@@ -107,6 +113,9 @@
     def __unicode__(self):
         return unicode(self.generate())
 
+    def __html__(self):
+        return Markup(self.generate())
+
     def append(self, node):
         """Append an element or string as child node.
         
@@ -146,14 +155,15 @@
         return Stream(self._generate())
 
 
-def _value_to_unicode(value):
-    if isinstance(value, unicode):
-        return value
-    return unicode(value)
-
 def _kwargs_to_attrs(kwargs):
-    return [(QName(k.rstrip('_').replace('_', '-')), _value_to_unicode(v))
-            for k, v in kwargs.items() if v is not None]
+    attrs = []
+    names = set()
+    for name, value in kwargs.items():
+        name = name.rstrip('_').replace('_', '-')
+        if value is not None and name not in names:
+            attrs.append((QName(name), unicode(value)))
+            names.add(name)
+    return Attrs(attrs)
 
 
 class Element(Fragment):
@@ -240,7 +250,7 @@
     def __init__(self, tag_, **attrib):
         Fragment.__init__(self)
         self.tag = QName(tag_)
-        self.attrib = Attrs(_kwargs_to_attrs(attrib))
+        self.attrib = _kwargs_to_attrs(attrib)
 
     def __call__(self, *args, **kwargs):
         """Append any positional arguments as child nodes, and keyword arguments
@@ -250,7 +260,7 @@
         :rtype: `Element`
         :see: `Fragment.append`
         """
-        self.attrib |= Attrs(_kwargs_to_attrs(kwargs))
+        self.attrib |= _kwargs_to_attrs(kwargs)
         Fragment.__call__(self, *args)
         return self
 
diff --git a/genshi/core.py b/genshi/core.py
--- a/genshi/core.py
+++ b/genshi/core.py
@@ -245,6 +245,9 @@
     def __unicode__(self):
         return self.render(encoding=None)
 
+    def __html__(self):
+        return self
+
 
 START = Stream.START
 END = Stream.END
@@ -485,6 +488,9 @@
             return cls()
         if type(text) is cls:
             return text
+        if hasattr(text, '__html__'):
+            return Markup(text.__html__())
+
         text = unicode(text).replace('&', '&amp;') \
                             .replace('<', '&lt;') \
                             .replace('>', '&gt;')
diff --git a/genshi/filters/i18n.py b/genshi/filters/i18n.py
--- a/genshi/filters/i18n.py
+++ b/genshi/filters/i18n.py
@@ -180,7 +180,10 @@
                     msgbuf.append(kind, data, pos)
                     continue
                 elif i18n_msg in attrs:
-                    msgbuf = MessageBuffer()
+                    params = attrs.get(i18n_msg)
+                    if params and type(params) is list: # event tuple
+                        params = params[0][1]
+                    msgbuf = MessageBuffer(params)
                     attrs -= i18n_msg
 
                 yield kind, (tag, attrs), pos
@@ -189,11 +192,14 @@
                 if not msgbuf:
                     text = data.strip()
                     if text:
-                        data = data.replace(text, translate(text))
+                        data = data.replace(text, unicode(translate(text)))
                     yield kind, data, pos
                 else:
                     msgbuf.append(kind, data, pos)
 
+            elif msgbuf and kind is EXPR:
+                msgbuf.append(kind, data, pos)
+
             elif not skip and msgbuf and kind is END:
                 msgbuf.append(kind, data, pos)
                 if not msgbuf.depth:
@@ -301,7 +307,10 @@
                 if msgbuf:
                     msgbuf.append(kind, data, pos)
                 elif i18n_msg in attrs:
-                    msgbuf = MessageBuffer(pos[1])
+                    params = attrs.get(i18n_msg)
+                    if params and type(params) is list: # event tuple
+                        params = params[0][1]
+                    msgbuf = MessageBuffer(params, pos[1])
 
             elif not skip and search_text and kind is TEXT:
                 if not msgbuf:
@@ -318,6 +327,8 @@
                     msgbuf = None
 
             elif kind is EXPR or kind is EXEC:
+                if msgbuf:
+                    msgbuf.append(kind, data, pos)
                 for funcname, strings in extract_from_code(data,
                                                            gettext_functions):
                     yield pos[1], funcname, strings
@@ -332,26 +343,46 @@
 
 
 class MessageBuffer(object):
-    """Helper class for managing localizable mixed content.
+    """Helper class for managing internationalized mixed content.
     
     :since: version 0.5
     """
 
-    def __init__(self, lineno=-1):
+    def __init__(self, params=u'', lineno=-1):
+        """Initialize the message buffer.
+        
+        :param params: comma-separated list of parameter names
+        :type params: `basestring`
+        :param lineno: the line number on which the first stream event
+                       belonging to the message was found
+        """
+        self.params = [name.strip() for name in params.split(',')]
         self.lineno = lineno
-        self.strings = []
+        self.string = []
         self.events = {}
+        self.values = {}
         self.depth = 1
         self.order = 1
         self.stack = [0]
 
     def append(self, kind, data, pos):
+        """Append a stream event to the buffer.
+        
+        :param kind: the stream event kind
+        :param data: the event data
+        :param pos: the position of the event in the source
+        """
         if kind is TEXT:
-            self.strings.append(data)
+            self.string.append(data)
             self.events.setdefault(self.stack[-1], []).append(None)
+        elif kind is EXPR:
+            param = self.params.pop(0)
+            self.string.append('%%(%s)s' % param)
+            self.events.setdefault(self.stack[-1], []).append(None)
+            self.values[param] = (kind, data, pos)
         else:
             if kind is START:
-                self.strings.append(u'[%d:' % self.order)
+                self.string.append(u'[%d:' % self.order)
                 self.events.setdefault(self.order, []).append((kind, data, pos))
                 self.stack.append(self.order)
                 self.depth += 1
@@ -360,26 +391,83 @@
                 self.depth -= 1
                 if self.depth:
                     self.events[self.stack[-1]].append((kind, data, pos))
-                    self.strings.append(u']')
+                    self.string.append(u']')
                     self.stack.pop()
 
     def format(self):
-        return u''.join(self.strings).strip()
+        """Return a message identifier representing the content in the
+        buffer.
+        """
+        return u''.join(self.string).strip()
 
-    def translate(self, string):
+    def translate(self, string, regex=re.compile(r'%\((\w+)\)s')):
+        """Interpolate the given message translation with the events in the
+        buffer and return the translated stream.
+        
+        :param string: the translated message string
+        """
         parts = parse_msg(string)
         for order, string in parts:
             events = self.events[order]
             while events:
-                event = self.events[order].pop(0)
-                if not event:
+                event = events.pop(0)
+                if event:
+                    yield event
+                else:
                     if not string:
                         break
-                    yield TEXT, string, (None, -1, -1)
+                    for idx, part in enumerate(regex.split(string)):
+                        if idx % 2:
+                            yield self.values[part]
+                        elif part:
+                            yield TEXT, part, (None, -1, -1)
                     if not self.events[order] or not self.events[order][0]:
                         break
-                else:
-                    yield event
+
+
+def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|\]')):
+    """Parse a translated message using Genshi mixed content message
+    formatting.
+
+    >>> parse_msg("See [1:Help].")
+    [(0, 'See '), (1, 'Help'), (0, '.')]
+
+    >>> parse_msg("See [1:our [2:Help] page] for details.")
+    [(0, 'See '), (1, 'our '), (2, 'Help'), (1, ' page'), (0, ' for details.')]
+
+    >>> parse_msg("[2:Details] finden Sie in [1:Hilfe].")
+    [(2, 'Details'), (0, ' finden Sie in '), (1, 'Hilfe'), (0, '.')]
+
+    >>> parse_msg("[1:] Bilder pro Seite anzeigen.")
+    [(1, ''), (0, ' Bilder pro Seite anzeigen.')]
+
+    :param string: the translated message string
+    :return: a list of ``(order, string)`` tuples
+    :rtype: `list`
+    """
+    parts = []
+    stack = [0]
+    while True:
+        mo = regex.search(string)
+        if not mo:
+            break
+
+        if mo.start() or stack[-1]:
+            parts.append((stack[-1], string[:mo.start()]))
+        string = string[mo.end():]
+
+        orderno = mo.group(1)
+        if orderno is not None:
+            stack.append(int(orderno))
+        else:
+            stack.pop()
+        if not stack:
+            break
+
+    if string:
+        parts.append((stack[-1], string))
+
+    return parts
 
 
 def extract_from_code(code, gettext_functions):
@@ -425,46 +513,6 @@
                     yield funcname, strings
     return _walk(code.ast)
 
-def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|\]')):
-    """Parse a message using Genshi compound message formatting.
-
-    >>> parse_msg("See [1:Help].")
-    [(0, 'See '), (1, 'Help'), (0, '.')]
-
-    >>> parse_msg("See [1:our [2:Help] page] for details.")
-    [(0, 'See '), (1, 'our '), (2, 'Help'), (1, ' page'), (0, ' for details.')]
-
-    >>> parse_msg("[2:Details] finden Sie in [1:Hilfe].")
-    [(2, 'Details'), (0, ' finden Sie in '), (1, 'Hilfe'), (0, '.')]
-    
-    >>> parse_msg("[1:] Bilder pro Seite anzeigen.")
-    [(1, ''), (0, ' Bilder pro Seite anzeigen.')]
-    
-    :since: version 0.5
-    """
-    parts = []
-    stack = [0]
-    while True:
-        mo = regex.search(string)
-        if not mo:
-            break
-
-        if mo.start() or stack[-1]:
-            parts.append((stack[-1], string[:mo.start()]))
-        string = string[mo.end():]
-
-        orderno = mo.group(1)
-        if orderno is not None:
-            stack.append(int(orderno))
-        else:
-            stack.pop()
-        if not stack:
-            break
-
-    if string:
-        parts.append((stack[-1], string))
-
-    return parts
 
 def extract(fileobj, keywords, comment_tags, options):
     """Babel extraction method for Genshi templates.
diff --git a/genshi/filters/tests/i18n.py b/genshi/filters/tests/i18n.py
--- a/genshi/filters/tests/i18n.py
+++ b/genshi/filters/tests/i18n.py
@@ -11,6 +11,7 @@
 # individuals. For the exact contribution history, see the revision
 # history and logs, available at http://genshi.edgewall.org/log/.
 
+from datetime import datetime
 import doctest
 from StringIO import StringIO
 import unittest
@@ -262,6 +263,73 @@
           <p><input type="text" name="num"/> Eintr\xc3\xa4ge pro Seite, beginnend auf Seite <input type="text" name="num"/>.</p>
         </html>""", tmpl.generate().render())
 
+    def test_extract_i18n_msg_with_param(self):
+        tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"
+            xmlns:i18n="http://genshi.edgewall.org/i18n">
+          <p i18n:msg="name">
+            Hello, ${user.name}!
+          </p>
+        </html>""")
+        translator = Translator()
+        messages = list(translator.extract(tmpl.stream))
+        self.assertEqual(1, len(messages))
+        self.assertEqual('Hello, %(name)s!', messages[0][2])
+
+    def test_translate_i18n_msg_with_param(self):
+        tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"
+            xmlns:i18n="http://genshi.edgewall.org/i18n">
+          <p i18n:msg="name">
+            Hello, ${user.name}!
+          </p>
+        </html>""")
+        gettext = lambda s: u"Hallo, %(name)s!"
+        tmpl.filters.insert(0, Translator(gettext))
+        self.assertEqual("""<html>
+          <p>Hallo, Jim!</p>
+        </html>""", tmpl.generate(user=dict(name='Jim')).render())
+
+    def test_translate_i18n_msg_with_param_reordered(self):
+        tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"
+            xmlns:i18n="http://genshi.edgewall.org/i18n">
+          <p i18n:msg="name">
+            Hello, ${user.name}!
+          </p>
+        </html>""")
+        gettext = lambda s: u"%(name)s, sei gegrüßt!"
+        tmpl.filters.insert(0, Translator(gettext))
+        self.assertEqual("""<html>
+          <p>Jim, sei gegrüßt!</p>
+        </html>""", tmpl.generate(user=dict(name='Jim')).render())
+
+    def test_extract_i18n_msg_with_two_params(self):
+        tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"
+            xmlns:i18n="http://genshi.edgewall.org/i18n">
+          <p i18n:msg="name, time">
+            Posted by ${post.author} at ${entry.time.strftime('%H:%m')}
+          </p>
+        </html>""")
+        translator = Translator()
+        messages = list(translator.extract(tmpl.stream))
+        self.assertEqual(1, len(messages))
+        self.assertEqual('Posted by %(name)s at %(time)s', messages[0][2])
+
+    def test_translate_i18n_msg_with_two_params(self):
+        tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"
+            xmlns:i18n="http://genshi.edgewall.org/i18n">
+          <p i18n:msg="name, time">
+            Written by ${entry.author} at ${entry.time.strftime('%H:%M')}
+          </p>
+        </html>""")
+        gettext = lambda s: u"%(name)s schrieb dies um %(time)s"
+        tmpl.filters.insert(0, Translator(gettext))
+        entry = {
+            'author': 'Jim',
+            'time': datetime(2008, 4, 1, 14, 30)
+        }
+        self.assertEqual("""<html>
+          <p>Jim schrieb dies um 14:30</p>
+        </html>""", tmpl.generate(entry=entry).render())
+
     def test_extract_i18n_msg_with_directive(self):
         tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"
             xmlns:i18n="http://genshi.edgewall.org/i18n">
diff --git a/genshi/filters/tests/transform.py b/genshi/filters/tests/transform.py
--- a/genshi/filters/tests/transform.py
+++ b/genshi/filters/tests/transform.py
@@ -12,20 +12,1477 @@
 # history and logs, available at http://genshi.edgewall.org/log/.
 
 import doctest
+from pprint import pprint
 import unittest
 
+from genshi import HTML
+from genshi.builder import Element
+from genshi.core import START, END, TEXT, QName, Attrs
+from genshi.filters.transform import Transformer, StreamBuffer, ENTER, EXIT, \
+                                     OUTSIDE, INSIDE, ATTR, BREAK
 import genshi.filters.transform
 
 
+FOO = '<root>ROOT<foo name="foo">FOO</foo></root>'
+FOOBAR = '<root>ROOT<foo name="foo" size="100">FOO</foo><bar name="bar">BAR</bar></root>'
+
+
+def _simplify(stream, with_attrs=False):
+    """Simplify a marked stream."""
+    def _generate():
+        for mark, (kind, data, pos) in stream:
+            if kind is START:
+                if with_attrs:
+                    data = (unicode(data[0]), dict([(unicode(k), v)
+                                                    for k, v in data[1]]))
+                else:
+                    data = unicode(data[0])
+            elif kind is END:
+                data = unicode(data)
+            elif kind is ATTR:
+                kind = ATTR
+                data = dict([(unicode(k), v) for k, v in data[1]])
+            yield mark, kind, data
+    return list(_generate())
+
+
+def _transform(html, transformer, with_attrs=False):
+    """Apply transformation returning simplified marked stream."""
+    if isinstance(html, basestring):
+        html = HTML(html)
+    stream = transformer(html, keep_marks=True)
+    return _simplify(stream, with_attrs)
+
+
+class SelectTest(unittest.TestCase):
+    """Test .select()"""
+    def _select(self, select):
+        html = HTML(FOOBAR)
+        if isinstance(select, basestring):
+            select = [select]
+        transformer = Transformer(select[0])
+        for sel in select[1:]:
+            transformer = transformer.select(sel)
+        return _transform(html, transformer)
+
+    def test_select_single_element(self):
+        self.assertEqual(
+            self._select('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')],
+            )
+
+    def test_select_context(self):
+        self.assertEqual(
+            self._select('.'),
+            [(ENTER, START, u'root'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (INSIDE, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (INSIDE, END, u'bar'),
+             (EXIT, END, u'root')]
+            )
+
+    def test_select_inside_select(self):
+        self.assertEqual(
+            self._select(['.', 'foo']),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')],
+            )
+
+    def test_select_text(self):
+        self.assertEqual(
+            self._select('*/text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (OUTSIDE, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')],
+            )
+
+    def test_select_attr(self):
+        self.assertEqual(
+            self._select('foo/@name'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ATTR, ATTR, {'name': u'foo'}),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_select_text_context(self):
+        self.assertEqual(
+            list(Transformer('.')(HTML('foo'), keep_marks=True)),
+            [('OUTSIDE', ('TEXT', u'foo', (None, 1, 0)))],
+            )
+
+
+class InvertTest(unittest.TestCase):
+    def _invert(self, select):
+        return _transform(FOO, Transformer(select).invert())
+
+    def test_invert_element(self):
+        self.assertEqual(
+            self._invert('foo'),
+            [(OUTSIDE, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (OUTSIDE, END, u'root')]
+            )
+
+    def test_invert_inverted_element(self):
+        self.assertEqual(
+            _transform(FOO, Transformer('foo').invert().invert()),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (OUTSIDE, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (OUTSIDE, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_invert_text(self):
+        self.assertEqual(
+            self._invert('foo/text()'),
+            [(OUTSIDE, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (OUTSIDE, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (OUTSIDE, END, u'foo'),
+             (OUTSIDE, END, u'root')]
+            )
+
+    def test_invert_attribute(self):
+        self.assertEqual(
+            self._invert('foo/@name'),
+            [(OUTSIDE, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, ATTR, {'name': u'foo'}),
+             (OUTSIDE, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (OUTSIDE, END, u'foo'),
+             (OUTSIDE, END, u'root')]
+            )
+
+    def test_invert_context(self):
+        self.assertEqual(
+            self._invert('.'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_invert_text_context(self):
+        self.assertEqual(
+            _simplify(Transformer('.').invert()(HTML('foo'), keep_marks=True)),
+            [(None, 'TEXT', u'foo')],
+            )
+
+
+
+class EndTest(unittest.TestCase):
+    def test_end(self):
+        stream = _transform(FOO, Transformer('foo').end())
+        self.assertEqual(
+            stream,
+            [(OUTSIDE, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (OUTSIDE, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (OUTSIDE, END, u'foo'),
+             (OUTSIDE, END, u'root')]
+            )
+
+
+class EmptyTest(unittest.TestCase):
+    def _empty(self, select):
+        return _transform(FOO, Transformer(select).empty())
+
+    def test_empty_element(self):
+        self.assertEqual(
+            self._empty('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (EXIT, END, u'foo'),
+             (None, END, u'root')],
+            )
+
+    def test_empty_text(self):
+        self.assertEqual(
+            self._empty('foo/text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_empty_attr(self):
+        self.assertEqual(
+            self._empty('foo/@name'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ATTR, ATTR, {'name': u'foo'}),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_empty_context(self):
+        self.assertEqual(
+            self._empty('.'),
+            [(ENTER, START, u'root'),
+             (EXIT, END, u'root')]
+            )
+
+    def test_empty_text_context(self):
+        self.assertEqual(
+            _simplify(Transformer('.')(HTML('foo'), keep_marks=True)),
+            [(OUTSIDE, TEXT, u'foo')],
+            )
+
+
+class RemoveTest(unittest.TestCase):
+    def _remove(self, select):
+        return _transform(FOO, Transformer(select).remove())
+
+    def test_remove_element(self):
+        self.assertEqual(
+            self._remove('foo|bar'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, END, u'root')]
+            )
+
+    def test_remove_text(self):
+        self.assertEqual(
+            self._remove('//text()'),
+            [(None, START, u'root'),
+             (None, START, u'foo'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_remove_attr(self):
+        self.assertEqual(
+            self._remove('foo/@name'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_remove_context(self):
+        self.assertEqual(
+            self._remove('.'),
+            [],
+            )
+
+    def test_remove_text_context(self):
+        self.assertEqual(
+            _transform('foo', Transformer('.').remove()),
+            [],
+            )
+
+
+class UnwrapText(unittest.TestCase):
+    def _unwrap(self, select):
+        return _transform(FOO, Transformer(select).unwrap())
+
+    def test_unwrap_element(self):
+        self.assertEqual(
+            self._unwrap('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (INSIDE, TEXT, u'FOO'),
+             (None, END, u'root')]
+            )
+
+    def test_unwrap_text(self):
+        self.assertEqual(
+            self._unwrap('foo/text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_unwrap_attr(self):
+        self.assertEqual(
+            self._unwrap('foo/@name'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ATTR, ATTR, {'name': u'foo'}),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_unwrap_adjacent(self):
+        self.assertEqual(
+            _transform(FOOBAR, Transformer('foo|bar').unwrap()),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, TEXT, u'BAR'),
+             (None, END, u'root')]
+            )
+
+    def test_unwrap_root(self):
+        self.assertEqual(
+            self._unwrap('.'),
+            [(INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo')]
+            )
+
+    def test_unwrap_text_root(self):
+        self.assertEqual(
+            _transform('foo', Transformer('.').unwrap()),
+            [(OUTSIDE, TEXT, 'foo')],
+            )
+
+
+class WrapTest(unittest.TestCase):
+    def _wrap(self, select, wrap='wrap'):
+        return _transform(FOO, Transformer(select).wrap(wrap))
+
+    def test_wrap_element(self):
+        self.assertEqual(
+            self._wrap('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'wrap'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, END, u'wrap'),
+             (None, END, u'root')]
+            )
+
+    def test_wrap_adjacent_elements(self):
+        self.assertEqual(
+            _transform(FOOBAR, Transformer('foo|bar').wrap('wrap')),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'wrap'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, END, u'wrap'),
+             (None, START, u'wrap'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, END, u'wrap'),
+             (None, END, u'root')]
+            )
+
+    def test_wrap_text(self):
+        self.assertEqual(
+            self._wrap('foo/text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, START, u'wrap'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (None, END, u'wrap'),
+             (None, END, u'foo'),
+             (None, END, u'root')]
+            )
+
+    def test_wrap_root(self):
+        self.assertEqual(
+            self._wrap('.'),
+            [(None, START, u'wrap'),
+             (ENTER, START, u'root'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (EXIT, END, u'root'),
+             (None, END, u'wrap')]
+            )
+
+    def test_wrap_text_root(self):
+        self.assertEqual(
+            _transform('foo', Transformer('.').wrap('wrap')),
+            [(None, START, u'wrap'),
+             (OUTSIDE, TEXT, u'foo'),
+             (None, END, u'wrap')],
+            )
+
+    def test_wrap_with_element(self):
+        element = Element('a', href='http://localhost')
+        self.assertEqual(
+            _transform('foo', Transformer('.').wrap(element), with_attrs=True),
+            [(None, START, (u'a', {u'href': u'http://localhost'})),
+             (OUTSIDE, TEXT, u'foo'),
+             (None, END, u'a')]
+            )
+
+
+class FilterTest(unittest.TestCase):
+    def _filter(self, select, html=FOOBAR):
+        """Returns a list of lists of filtered elements."""
+        output = []
+        def filtered(stream):
+            interval = []
+            output.append(interval)
+            for event in stream:
+                interval.append(event)
+                yield event
+        _transform(html, Transformer(select).filter(filtered))
+        simplified = []
+        for sub in output:
+            simplified.append(_simplify([(None, event) for event in sub]))
+        return simplified
+
+    def test_filter_element(self):
+        self.assertEqual(
+            self._filter('foo'),
+            [[(None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo')]]
+            )
+
+    def test_filter_adjacent_elements(self):
+        self.assertEqual(
+            self._filter('foo|bar'),
+            [[(None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo')],
+             [(None, START, u'bar'),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar')]]
+            )
+
+    def test_filter_text(self):
+        self.assertEqual(
+            self._filter('*/text()'),
+            [[(None, TEXT, u'FOO')],
+             [(None, TEXT, u'BAR')]]
+            )
+    def test_filter_root(self):
+        self.assertEqual(
+            self._filter('.'),
+            [[(None, START, u'root'),
+              (None, TEXT, u'ROOT'),
+              (None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo'),
+              (None, START, u'bar'),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar'),
+              (None, END, u'root')]]
+            )
+
+    def test_filter_text_root(self):
+        self.assertEqual(
+            self._filter('.', 'foo'),
+            [[(None, TEXT, u'foo')]])
+
+
+class MapTest(unittest.TestCase):
+    def _map(self, select, kind=None):
+        data = []
+        def record(d):
+            data.append(d)
+            return d
+        _transform(FOOBAR, Transformer(select).map(record, kind))
+        return data
+
+    def test_map_element(self):
+        self.assertEqual(
+            self._map('foo'),
+            [(QName(u'foo'), Attrs([(QName(u'name'), u'foo'),
+                                    (QName(u'size'), u'100')])),
+             u'FOO',
+             QName(u'foo')]
+            )
+
+    def test_map_with_text_kind(self):
+        self.assertEqual(
+            self._map('.', TEXT),
+            [u'ROOT', u'FOO', u'BAR']
+            )
+
+    def test_map_with_root_and_end_kind(self):
+        self.assertEqual(
+            self._map('.', END),
+            [QName(u'foo'), QName(u'bar'), QName(u'root')]
+            )
+
+    def test_map_with_attribute(self):
+        self.assertEqual(
+            self._map('foo/@name'),
+            [(QName(u'foo@*'), Attrs([('name', u'foo')]))]
+            )
+
+
+class SubstituteTest(unittest.TestCase):
+    def _substitute(self, select, pattern, replace):
+        return _transform(FOOBAR, Transformer(select).substitute(pattern, replace))
+
+    def test_substitute_foo(self):
+        self.assertEqual(
+            self._substitute('foo', 'FOO|BAR', 'FOOOOO'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOOOOO'),
+             (EXIT, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_substitute_foobar_with_group(self):
+        self.assertEqual(
+            self._substitute('foo|bar', '(FOO|BAR)', r'(\1)'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'(FOO)'),
+             (EXIT, END, u'foo'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'(BAR)'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+
+class RenameTest(unittest.TestCase):
+    def _rename(self, select):
+        return _transform(FOOBAR, Transformer(select).rename('foobar'))
+
+    def test_rename_root(self):
+        self.assertEqual(
+            self._rename('.'),
+            [(ENTER, START, u'foobar'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (INSIDE, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (INSIDE, END, u'bar'),
+             (EXIT, END, u'foobar')]
+            )
+
+    def test_rename_element(self):
+        self.assertEqual(
+            self._rename('foo|bar'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foobar'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foobar'),
+             (ENTER, START, u'foobar'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'foobar'),
+             (None, END, u'root')]
+            )
+
+    def test_rename_text(self):
+        self.assertEqual(
+            self._rename('foo/text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (OUTSIDE, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+
+class ContentTestMixin(object):
+    def _apply(self, select, content=None, html=FOOBAR):
+        class Injector(object):
+            count = 0
+
+            def __iter__(self):
+                self.count += 1
+                return iter(HTML('CONTENT %i' % self.count))
+
+        if isinstance(html, basestring):
+            html = HTML(html)
+        if content is None:
+            content = Injector()
+        elif isinstance(content, basestring):
+            content = HTML(content)
+        return _transform(html, getattr(Transformer(select), self.operation)
+                                (content))
+
+
+class ReplaceTest(unittest.TestCase, ContentTestMixin):
+    operation = 'replace'
+
+    def test_replace_element(self):
+        self.assertEqual(
+            self._apply('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 1'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_replace_text(self):
+        self.assertEqual(
+            self._apply('text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'CONTENT 1'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_replace_context(self):
+        self.assertEqual(
+            self._apply('.'),
+            [(None, TEXT, u'CONTENT 1')],
+            )
+
+    def test_replace_text_context(self):
+        self.assertEqual(
+            self._apply('.', html='foo'),
+            [(None, TEXT, u'CONTENT 1')],
+            )
+
+    def test_replace_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('*'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 1'),
+             (None, TEXT, u'CONTENT 2'),
+             (None, END, u'root')],
+            )
+
+    def test_replace_all(self):
+        self.assertEqual(
+            self._apply('*|text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'CONTENT 1'),
+             (None, TEXT, u'CONTENT 2'),
+             (None, TEXT, u'CONTENT 3'),
+             (None, END, u'root')],
+            )
+
+    def test_replace_with_callback(self):
+        count = [0]
+        def content():
+            count[0] += 1
+            yield '%2i.' % count[0]
+        self.assertEqual(
+            self._apply('*', content),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, TEXT, u' 1.'),
+             (None, TEXT, u' 2.'),
+             (None, END, u'root')]
+            )
+
+
+class BeforeTest(unittest.TestCase, ContentTestMixin):
+    operation = 'before'
+
+    def test_before_element(self):
+        self.assertEqual(
+            self._apply('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 1'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_before_text(self):
+        self.assertEqual(
+            self._apply('text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'CONTENT 1'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_before_context(self):
+        self.assertEqual(
+            self._apply('.'),
+            [(None, TEXT, u'CONTENT 1'),
+             (ENTER, START, u'root'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (INSIDE, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (INSIDE, END, u'bar'),
+             (EXIT, END, u'root')]
+            )
+
+    def test_before_text_context(self):
+        self.assertEqual(
+            self._apply('.', html='foo'),
+            [(None, TEXT, u'CONTENT 1'),
+             (OUTSIDE, TEXT, u'foo')]
+            )
+
+    def test_before_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('*'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 1'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, TEXT, u'CONTENT 2'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+
+            )
+
+    def test_before_all(self):
+        self.assertEqual(
+            self._apply('*|text()'),
+            [(None, START, u'root'),
+             (None, TEXT, u'CONTENT 1'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 2'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, TEXT, u'CONTENT 3'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_before_with_callback(self):
+        count = [0]
+        def content():
+            count[0] += 1
+            yield '%2i.' % count[0]
+        self.assertEqual(
+            self._apply('foo/text()', content),
+            [(None, 'START', u'root'),
+             (None, 'TEXT', u'ROOT'),
+             (None, 'START', u'foo'),
+             (None, 'TEXT', u' 1.'),
+             ('OUTSIDE', 'TEXT', u'FOO'),
+             (None, 'END', u'foo'),
+             (None, 'START', u'bar'),
+             (None, 'TEXT', u'BAR'),
+             (None, 'END', u'bar'),
+             (None, 'END', u'root')]
+            )
+
+
+class AfterTest(unittest.TestCase, ContentTestMixin):
+    operation = 'after'
+
+    def test_after_element(self):
+        self.assertEqual(
+            self._apply('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, TEXT, u'CONTENT 1'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_after_text(self):
+        self.assertEqual(
+            self._apply('text()'),
+            [(None, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 1'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_after_context(self):
+        self.assertEqual(
+            self._apply('.'),
+            [(ENTER, START, u'root'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (INSIDE, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (INSIDE, END, u'bar'),
+             (EXIT, END, u'root'),
+             (None, TEXT, u'CONTENT 1')]
+            )
+
+    def test_after_text_context(self):
+        self.assertEqual(
+            self._apply('.', html='foo'),
+            [(OUTSIDE, TEXT, u'foo'),
+             (None, TEXT, u'CONTENT 1')]
+            )
+
+    def test_after_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('*'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, TEXT, u'CONTENT 1'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, TEXT, u'CONTENT 2'),
+             (None, END, u'root')]
+
+            )
+
+    def test_after_all(self):
+        self.assertEqual(
+            self._apply('*|text()'),
+            [(None, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, TEXT, u'CONTENT 1'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, TEXT, u'CONTENT 2'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, TEXT, u'CONTENT 3'),
+             (None, END, u'root')]
+            )
+
+    def test_after_with_callback(self):
+        count = [0]
+        def content():
+            count[0] += 1
+            yield '%2i.' % count[0]
+        self.assertEqual(
+            self._apply('foo/text()', content),
+            [(None, 'START', u'root'),
+             (None, 'TEXT', u'ROOT'),
+             (None, 'START', u'foo'),
+             ('OUTSIDE', 'TEXT', u'FOO'),
+             (None, 'TEXT', u' 1.'),
+             (None, 'END', u'foo'),
+             (None, 'START', u'bar'),
+             (None, 'TEXT', u'BAR'),
+             (None, 'END', u'bar'),
+             (None, 'END', u'root')]
+            )
+
+
+class PrependTest(unittest.TestCase, ContentTestMixin):
+    operation = 'prepend'
+
+    def test_prepend_element(self):
+        self.assertEqual(
+            self._apply('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (None, TEXT, u'CONTENT 1'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_prepend_text(self):
+        self.assertEqual(
+            self._apply('text()'),
+            [(None, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_prepend_context(self):
+        self.assertEqual(
+            self._apply('.'),
+            [(ENTER, START, u'root'),
+             (None, TEXT, u'CONTENT 1'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (INSIDE, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (INSIDE, END, u'bar'),
+             (EXIT, END, u'root')],
+            )
+
+    def test_prepend_text_context(self):
+        self.assertEqual(
+            self._apply('.', html='foo'),
+            [(OUTSIDE, TEXT, u'foo')]
+            )
+
+    def test_prepend_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('*'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (None, TEXT, u'CONTENT 1'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (ENTER, START, u'bar'),
+             (None, TEXT, u'CONTENT 2'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+
+            )
+
+    def test_prepend_all(self):
+        self.assertEqual(
+            self._apply('*|text()'),
+            [(None, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (None, TEXT, u'CONTENT 1'),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (ENTER, START, u'bar'),
+             (None, TEXT, u'CONTENT 2'),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_prepend_with_callback(self):
+        count = [0]
+        def content():
+            count[0] += 1
+            yield '%2i.' % count[0]
+        self.assertEqual(
+            self._apply('foo', content),
+            [(None, 'START', u'root'),
+             (None, 'TEXT', u'ROOT'),
+             (ENTER, 'START', u'foo'),
+             (None, 'TEXT', u' 1.'),
+             (INSIDE, 'TEXT', u'FOO'),
+             (EXIT, 'END', u'foo'),
+             (None, 'START', u'bar'),
+             (None, 'TEXT', u'BAR'),
+             (None, 'END', u'bar'),
+             (None, 'END', u'root')]
+            )
+
+
+class AppendTest(unittest.TestCase, ContentTestMixin):
+    operation = 'append'
+
+    def test_append_element(self):
+        self.assertEqual(
+            self._apply('foo'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (None, TEXT, u'CONTENT 1'),
+             (EXIT, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_append_text(self):
+        self.assertEqual(
+            self._apply('text()'),
+            [(None, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (None, START, u'foo'),
+             (None, TEXT, u'FOO'),
+             (None, END, u'foo'),
+             (None, START, u'bar'),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_append_context(self):
+        self.assertEqual(
+            self._apply('.'),
+            [(ENTER, START, u'root'),
+             (INSIDE, TEXT, u'ROOT'),
+             (INSIDE, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (INSIDE, END, u'foo'),
+             (INSIDE, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (INSIDE, END, u'bar'),
+             (None, TEXT, u'CONTENT 1'),
+             (EXIT, END, u'root')],
+            )
+
+    def test_append_text_context(self):
+        self.assertEqual(
+            self._apply('.', html='foo'),
+            [(OUTSIDE, TEXT, u'foo')]
+            )
+
+    def test_append_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('*'),
+            [(None, START, u'root'),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (None, TEXT, u'CONTENT 1'),
+             (EXIT, END, u'foo'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (None, TEXT, u'CONTENT 2'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+
+            )
+
+    def test_append_all(self):
+        self.assertEqual(
+            self._apply('*|text()'),
+            [(None, START, u'root'),
+             (OUTSIDE, TEXT, u'ROOT'),
+             (ENTER, START, u'foo'),
+             (INSIDE, TEXT, u'FOO'),
+             (None, TEXT, u'CONTENT 1'),
+             (EXIT, END, u'foo'),
+             (ENTER, START, u'bar'),
+             (INSIDE, TEXT, u'BAR'),
+             (None, TEXT, u'CONTENT 2'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_append_with_callback(self):
+        count = [0]
+        def content():
+            count[0] += 1
+            yield '%2i.' % count[0]
+        self.assertEqual(
+            self._apply('foo', content),
+            [(None, 'START', u'root'),
+             (None, 'TEXT', u'ROOT'),
+             (ENTER, 'START', u'foo'),
+             (INSIDE, 'TEXT', u'FOO'),
+             (None, 'TEXT', u' 1.'),
+             (EXIT, 'END', u'foo'),
+             (None, 'START', u'bar'),
+             (None, 'TEXT', u'BAR'),
+             (None, 'END', u'bar'),
+             (None, 'END', u'root')]
+            )
+
+
+
+class AttrTest(unittest.TestCase):
+    def _attr(self, select, name, value):
+        return _transform(FOOBAR, Transformer(select).attr(name, value),
+                          with_attrs=True)
+
+    def test_set_existing_attr(self):
+        self.assertEqual(
+            self._attr('foo', 'name', 'FOO'),
+            [(None, START, (u'root', {})),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, (u'foo', {u'name': 'FOO', u'size': '100'})),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, (u'bar', {u'name': u'bar'})),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_set_new_attr(self):
+        self.assertEqual(
+            self._attr('foo', 'title', 'FOO'),
+            [(None, START, (u'root', {})),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, (u'foo', {u'name': u'foo', u'title': 'FOO', u'size': '100'})),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, (u'bar', {u'name': u'bar'})),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_attr_from_function(self):
+        def set(name, event):
+            self.assertEqual(name, 'name')
+            return event[1][1].get('name').upper()
+
+        self.assertEqual(
+            self._attr('foo|bar', 'name', set),
+            [(None, START, (u'root', {})),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, (u'foo', {u'name': 'FOO', u'size': '100'})),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (ENTER, START, (u'bar', {u'name': 'BAR'})),
+             (INSIDE, TEXT, u'BAR'),
+             (EXIT, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_remove_attr(self):
+        self.assertEqual(
+            self._attr('foo', 'name', None),
+            [(None, START, (u'root', {})),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, (u'foo', {u'size': '100'})),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, (u'bar', {u'name': u'bar'})),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+    def test_remove_attr_with_function(self):
+        def set(name, event):
+            return None
+
+        self.assertEqual(
+            self._attr('foo', 'name', set),
+            [(None, START, (u'root', {})),
+             (None, TEXT, u'ROOT'),
+             (ENTER, START, (u'foo', {u'size': '100'})),
+             (INSIDE, TEXT, u'FOO'),
+             (EXIT, END, u'foo'),
+             (None, START, (u'bar', {u'name': u'bar'})),
+             (None, TEXT, u'BAR'),
+             (None, END, u'bar'),
+             (None, END, u'root')]
+            )
+
+
+class BufferTestMixin(object):
+    def _apply(self, select, with_attrs=False):
+        buffer = StreamBuffer()
+        events = buffer.events
+
+        class Trace(object):
+            last = None
+            trace = []
+
+            def __call__(self, stream):
+                for event in stream:
+                    if events and hash(tuple(events)) != self.last:
+                        self.last = hash(tuple(events))
+                        self.trace.append(list(events))
+                    yield event
+
+        trace = Trace()
+        output = _transform(FOOBAR, getattr(Transformer(select), self.operation)
+                                    (buffer).apply(trace), with_attrs=with_attrs)
+        simplified = []
+        for interval in trace.trace:
+            simplified.append(_simplify([(None, e) for e in interval],
+                                         with_attrs=with_attrs))
+        return output, simplified
+
+
+class CopyTest(unittest.TestCase, BufferTestMixin):
+    operation = 'copy'
+
+    def test_copy_element(self):
+        self.assertEqual(
+            self._apply('foo')[1],
+            [[(None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo')]]
+            )
+
+    def test_copy_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('foo|bar')[1],
+            [[(None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo')],
+             [(None, START, u'bar'),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar')]]
+            )
+
+    def test_copy_all(self):
+        self.assertEqual(
+            self._apply('*|text()')[1],
+            [[(None, TEXT, u'ROOT')],
+             [(None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo')],
+             [(None, START, u'bar'),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar')]]
+            )
+
+    def test_copy_text(self):
+        self.assertEqual(
+            self._apply('*/text()')[1],
+            [[(None, TEXT, u'FOO')],
+             [(None, TEXT, u'BAR')]]
+            )
+
+    def test_copy_context(self):
+        self.assertEqual(
+            self._apply('.')[1],
+            [[(None, START, u'root'),
+              (None, TEXT, u'ROOT'),
+              (None, START, u'foo'),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo'),
+              (None, START, u'bar'),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar'),
+              (None, END, u'root')]]
+            )
+
+    def test_copy_attribute(self):
+        self.assertEqual(
+            self._apply('foo/@name', with_attrs=True)[1],
+            [[(None, ATTR, {'name': u'foo'})]]
+            )
+
+    def test_copy_attributes(self):
+        self.assertEqual(
+            self._apply('foo/@*', with_attrs=True)[1],
+            [[(None, ATTR, {u'name': u'foo', u'size': u'100'})]]
+            )
+
+
+class CutTest(unittest.TestCase, BufferTestMixin):
+    operation = 'cut'
+
+    def test_cut_element(self):
+        self.assertEqual(
+            self._apply('foo'),
+            ([(None, START, u'root'),
+              (None, TEXT, u'ROOT'),
+              (None, START, u'bar'),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar'),
+              (None, END, u'root')],
+             [[(None, START, u'foo'),
+               (None, TEXT, u'FOO'),
+               (None, END, u'foo')]])
+            )
+
+    def test_cut_adjacent_elements(self):
+        self.assertEqual(
+            self._apply('foo|bar'),
+            ([(None, START, u'root'), 
+              (None, TEXT, u'ROOT'),
+              (BREAK, BREAK, None),
+              (None, END, u'root')],
+             [[(None, START, u'foo'),
+               (None, TEXT, u'FOO'),
+               (None, END, u'foo')],
+              [(None, START, u'bar'),
+               (None, TEXT, u'BAR'),
+               (None, END, u'bar')]])
+            )
+
+    def test_cut_all(self):
+        self.assertEqual(
+            self._apply('*|text()'),
+            ([(None, 'START', u'root'),
+              ('BREAK', 'BREAK', None),
+              ('BREAK', 'BREAK', None),
+              (None, 'END', u'root')],
+             [[(None, 'TEXT', u'ROOT')],
+              [(None, 'START', u'foo'),
+               (None, 'TEXT', u'FOO'),
+               (None, 'END', u'foo')],
+              [(None, 'START', u'bar'),
+               (None, 'TEXT', u'BAR'),
+               (None, 'END', u'bar')]])
+            )
+
+    def test_cut_text(self):
+        self.assertEqual(
+            self._apply('*/text()'),
+            ([(None, 'START', u'root'),
+              (None, 'TEXT', u'ROOT'),
+              (None, 'START', u'foo'),
+              (None, 'END', u'foo'),
+              (None, 'START', u'bar'),
+              (None, 'END', u'bar'),
+              (None, 'END', u'root')],
+             [[(None, 'TEXT', u'FOO')],
+              [(None, 'TEXT', u'BAR')]])
+            )
+
+    def test_cut_context(self):
+        self.assertEqual(
+            self._apply('.')[1],
+            [[(None, 'START', u'root'),
+              (None, 'TEXT', u'ROOT'),
+              (None, 'START', u'foo'),
+              (None, 'TEXT', u'FOO'),
+              (None, 'END', u'foo'),
+              (None, 'START', u'bar'),
+              (None, 'TEXT', u'BAR'),
+              (None, 'END', u'bar'),
+              (None, 'END', u'root')]]
+            )
+
+    def test_cut_attribute(self):
+        self.assertEqual(
+            self._apply('foo/@name', with_attrs=True),
+            ([(None, START, (u'root', {})),
+              (None, TEXT, u'ROOT'),
+              (None, START, (u'foo', {u'size': u'100'})),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo'),
+              (None, START, (u'bar', {u'name': u'bar'})),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar'),
+              (None, END, u'root')],
+             [[(None, ATTR, {u'name': u'foo'})]])
+            )
+
+    def test_cut_attributes(self):
+        self.assertEqual(
+            self._apply('foo/@*', with_attrs=True),
+            ([(None, START, (u'root', {})),
+              (None, TEXT, u'ROOT'),
+              (None, START, (u'foo', {})),
+              (None, TEXT, u'FOO'),
+              (None, END, u'foo'),
+              (None, START, (u'bar', {u'name': u'bar'})),
+              (None, TEXT, u'BAR'),
+              (None, END, u'bar'),
+              (None, END, u'root')],
+             [[(None, ATTR, {u'name': u'foo', u'size': u'100'})]])
+            )
+
+# XXX Test this when the XPath implementation is fixed (#233).
+#    def test_cut_attribute_or_attribute(self):
+#        self.assertEqual(
+#            self._apply('foo/@name | foo/@size', with_attrs=True),
+#            ([(None, START, (u'root', {})),
+#              (None, TEXT, u'ROOT'),
+#              (None, START, (u'foo', {})),
+#              (None, TEXT, u'FOO'),
+#              (None, END, u'foo'),
+#              (None, START, (u'bar', {u'name': u'bar'})),
+#              (None, TEXT, u'BAR'),
+#              (None, END, u'bar'),
+#              (None, END, u'root')],
+#             [[(None, ATTR, {u'name': u'foo', u'size': u'100'})]])
+#            )
+
+
+
+
 def suite():
     from genshi.input import HTML
     from genshi.core import Markup
     from genshi.builder import tag
-    suite = doctest.DocTestSuite(genshi.filters.transform,
-                                 optionflags=doctest.NORMALIZE_WHITESPACE,
-                                 extraglobs={'HTML': HTML, 'tag': tag,
-                                     'Markup': Markup})
+    suite = unittest.TestSuite()
+    for test in (SelectTest, InvertTest, EndTest,
+                 EmptyTest, RemoveTest, UnwrapText, WrapTest, FilterTest,
+                 MapTest, SubstituteTest, RenameTest, ReplaceTest, BeforeTest,
+                 AfterTest, PrependTest, AppendTest, AttrTest, CopyTest, CutTest):
+        suite.addTest(unittest.makeSuite(test, 'test'))
+    suite.addTest(doctest.DocTestSuite(
+        genshi.filters.transform, optionflags=doctest.NORMALIZE_WHITESPACE,
+        extraglobs={'HTML': HTML, 'tag': tag, 'Markup': Markup}))
     return suite
 
+
 if __name__ == '__main__':
     unittest.main(defaultTest='suite')
diff --git a/genshi/filters/transform.py b/genshi/filters/transform.py
--- a/genshi/filters/transform.py
+++ b/genshi/filters/transform.py
@@ -55,7 +55,7 @@
 from genshi.path import Path
 
 __all__ = ['Transformer', 'StreamBuffer', 'InjectorTransformation', 'ENTER',
-           'EXIT', 'INSIDE', 'OUTSIDE']
+           'EXIT', 'INSIDE', 'OUTSIDE', 'BREAK']
 
 
 class TransformMark(str):
@@ -86,6 +86,40 @@
 """Stream augmentation mark indicating that a selected element is being
 exited."""
 
+BREAK = TransformMark('BREAK')
+"""Stream augmentation mark indicating a break between two otherwise contiguous
+blocks of marked events.
+
+This is used primarily by the cut() transform to provide later transforms with
+an opportunity to operate on the cut buffer.
+"""
+
+
+class PushBackStream(object):
+    """Allows a single event to be pushed back onto the stream and re-consumed.
+    """
+    def __init__(self, stream):
+        self.stream = iter(stream)
+        self.peek = None
+
+    def push(self, event):
+        assert self.peek is None
+        self.peek = event
+
+    def __iter__(self):
+        while True:
+            if self.peek is not None:
+                peek = self.peek
+                self.peek = None
+                yield peek
+            else:
+                try:
+                    event = self.stream.next()
+                    yield event
+                except StopIteration:
+                    if self.peek is None:
+                        raise
+
 
 class Transformer(object):
     """Stream filter that can apply a variety of different transformations to
@@ -150,17 +184,21 @@
         """
         self.transforms = [SelectTransformation(path)]
 
-    def __call__(self, stream):
+    def __call__(self, stream, keep_marks=False):
         """Apply the transform filter to the marked stream.
 
         :param stream: the marked event stream to filter
+        :param keep_marks: Do not strip transformer selection marks from the
+                           stream. Useful for testing.
         :return: the transformed stream
         :rtype: `Stream`
         """
         transforms = self._mark(stream)
         for link in self.transforms:
             transforms = link(transforms)
-        return Stream(self._unmark(transforms),
+        if not keep_marks:
+            transforms = self._unmark(transforms)
+        return Stream(transforms,
                       serializer=getattr(stream, 'serializer', None))
 
     def apply(self, function):
@@ -329,7 +367,8 @@
         <html><head><title>New Title</title></head><body>Some <em>body</em>
         text.</body></html>
 
-        :param content: Either an iterable of events or a string to insert.
+        :param content: Either a callable, an iterable of events, or a string
+                        to insert.
         :rtype: `Transformer`
         """
         return self.apply(ReplaceTransformation(content))
@@ -346,7 +385,8 @@
         <html><head><title>Some Title</title></head><body>Some emphasised
         <em>body</em> text.</body></html>
 
-        :param content: Either an iterable of events or a string to insert.
+        :param content: Either a callable, an iterable of events, or a string
+                        to insert.
         :rtype: `Transformer`
         """
         return self.apply(BeforeTransformation(content))
@@ -362,7 +402,8 @@
         <html><head><title>Some Title</title></head><body>Some <em>body</em>
         rock text.</body></html>
 
-        :param content: Either an iterable of events or a string to insert.
+        :param content: Either a callable, an iterable of events, or a string
+                        to insert.
         :rtype: `Transformer`
         """
         return self.apply(AfterTransformation(content))
@@ -378,7 +419,8 @@
         <html><head><title>Some Title</title></head><body>Some new body text.
         Some <em>body</em> text.</body></html>
 
-        :param content: Either an iterable of events or a string to insert.
+        :param content: Either a callable, an iterable of events, or a string
+                        to insert.
         :rtype: `Transformer`
         """
         return self.apply(PrependTransformation(content))
@@ -392,7 +434,8 @@
         <html><head><title>Some Title</title></head><body>Some <em>body</em>
         text. Some new body text.</body></html>
 
-        :param content: Either an iterable of events or a string to insert.
+        :param content: Either a callable, an iterable of events, or a string
+                        to insert.
         :rtype: `Transformer`
         """
         return self.apply(AppendTransformation(content))
@@ -440,9 +483,13 @@
 
     #{ Buffer operations
 
-    def copy(self, buffer):
+    def copy(self, buffer, accumulate=False):
         """Copy selection into buffer.
 
+        The buffer is replaced by each *contiguous* selection before being passed
+        to the next transformation. If accumulate=True, further selections will
+        be appended to the buffer rather than replacing it.
+
         >>> from genshi.builder import tag
         >>> buffer = StreamBuffer()
         >>> html = HTML('<html><head><title>Some Title</title></head>'
@@ -452,17 +499,14 @@
         <html><head><title>Some Title</title></head><body><h1>Some
         Title</h1>Some <em>body</em> text.</body></html>
 
-        To ensure that a transformation can be reused deterministically, the
-        contents of ``buffer`` is replaced by the ``copy()`` operation:
+        This example illustrates that only a single contiguous selection will
+        be buffered:
 
-        >>> print buffer
-        Some Title
         >>> print html | Transformer('head/title/text()').copy(buffer) \\
         ...     .end().select('body/em').copy(buffer).end().select('body') \\
         ...     .prepend(tag.h1(buffer))
-        <html><head><title>Some
-        Title</title></head><body><h1><em>body</em></h1>Some <em>body</em>
-        text.</body></html>
+        <html><head><title>Some Title</title></head><body><h1>Some
+        Title</h1>Some <em>body</em> text.</body></html>
         >>> print buffer
         <em>body</em>
 
@@ -475,7 +519,8 @@
         >>> def apply_attr(name, entry):
         ...     return list(buffer)[0][1][1].get('class')
         >>> print html | Transformer('body/em[@class]/@class').copy(buffer) \\
-        ...     .end().select('body/em[not(@class)]').attr('class', apply_attr)
+        ...     .end().buffer().select('body/em[not(@class)]') \\
+        ...     .attr('class', apply_attr)
         <html><head><title>Some Title</title></head><body><em
         class="before">Some</em> <em class="before">body</em><em
         class="before">text</em>.</body></html>
@@ -484,11 +529,18 @@
         :param buffer: the `StreamBuffer` in which the selection should be
                        stored
         :rtype: `Transformer`
-        :note: this transformation will buffer the entire input stream
+        :note: Copy (and cut) copy each individual selected object into the
+               buffer before passing to the next transform. For example, the
+               XPath ``*|text()`` will select all elements and text, each
+               instance of which will be copied to the buffer individually
+               before passing to the next transform. This has implications for
+               how ``StreamBuffer`` objects can be used, so some
+               experimentation may be required.
+
         """
-        return self.apply(CopyTransformation(buffer))
+        return self.apply(CopyTransformation(buffer, accumulate))
 
-    def cut(self, buffer):
+    def cut(self, buffer, accumulate=False):
         """Copy selection into buffer and remove the selection from the stream.
 
         >>> from genshi.builder import tag
@@ -500,12 +552,40 @@
         <html><head><title>Some Title</title></head><body>Some
         <em/><h1>body</h1> text.</body></html>
 
+        Specifying accumulate=True, appends all selected intervals onto the
+        buffer. Combining this with the .buffer() operation allows us operate
+        on all copied events rather than per-segment. See the documentation on
+        buffer() for more information.
+
         :param buffer: the `StreamBuffer` in which the selection should be
                        stored
         :rtype: `Transformer`
         :note: this transformation will buffer the entire input stream
         """
-        return self.apply(CutTransformation(buffer))
+        return self.apply(CutTransformation(buffer, accumulate))
+
+    def buffer(self):
+        """Buffer the entire stream (can consume a considerable amount of
+        memory).
+
+        Useful in conjunction with copy(accumulate=True) and
+        cut(accumulate=True) to ensure that all marked events in the entire
+        stream are copied to the buffer before further transformations are
+        applied.
+
+        For example, to move all <note> elements inside a <notes> tag at the
+        top of the document:
+
+        >>> doc = HTML('<doc><notes></notes><body>Some <note>one</note> '
+        ...            'text <note>two</note>.</body></doc>')
+        >>> buffer = StreamBuffer()
+        >>> print doc | Transformer('body/note').cut(buffer, accumulate=True) \\
+        ...     .end().buffer().select('notes').prepend(buffer)
+        <doc><notes><note>one</note><note>two</note></notes><body>Some  text
+        .</body></doc>
+
+        """
+        return self.apply(list)
 
     #{ Miscellaneous operations
 
@@ -546,13 +626,17 @@
         Refer to the documentation for ``re.sub()`` for details.
 
         >>> html = HTML('<html><body>Some text, some more text and '
-        ...             '<b>some bold text</b></body></html>')
-        >>> print html | Transformer('body').substitute('(?i)some', 'SOME')
-        <html><body>SOME text, some more text and <b>SOME bold text</b></body></html>
-        >>> tags = tag.html(tag.body('Some text, some more text and ',
+        ...             '<b>some bold text</b>\\n'
+        ...             '<i>some italicised text</i></body></html>')
+        >>> print html | Transformer('body/b').substitute('(?i)some', 'SOME')
+        <html><body>Some text, some more text and <b>SOME bold text</b>
+        <i>some italicised text</i></body></html>
+        >>> tags = tag.html(tag.body('Some text, some more text and\\n',
         ...      Markup('<b>some bold text</b>')))
-        >>> print tags.generate() | Transformer('body').substitute('(?i)some', 'SOME')
-        <html><body>SOME text, some more text and <b>SOME bold text</b></body></html>
+        >>> print tags.generate() | Transformer('body').substitute(
+        ...     '(?i)some', 'SOME')
+        <html><body>SOME text, some more text and
+        <b>SOME bold text</b></body></html>
 
         :param pattern: A regular expression object or string.
         :param replace: Replacement pattern.
@@ -600,7 +684,8 @@
 
     def _unmark(self, stream):
         for mark, event in stream:
-            if event[0] is not None:
+            kind = event[0]
+            if not (kind is None or kind is ATTR or kind is BREAK):
                 yield event
 
 
@@ -652,9 +737,12 @@
             elif isinstance(result, Attrs):
                 # XXX  Selected *attributes* are given a "kind" of None to
                 # indicate they are not really part of the stream.
-                yield ATTR, (None, (QName(event[1][0] + '@*'), result), event[2])
+                yield ATTR, (ATTR, (QName(event[1][0] + '@*'), result), event[2])
                 yield None, event
+            elif isinstance(result, tuple):
+                yield OUTSIDE, result
             elif result:
+                # XXX Assume everything else is "text"?
                 yield None, (TEXT, unicode(result), (None, -1, -1))
             else:
                 yield None, event
@@ -700,8 +788,12 @@
         :param stream: the marked event stream to filter
         """
         for mark, event in stream:
-            if mark not in (INSIDE, OUTSIDE):
-                yield mark, event
+            yield mark, event
+            if mark is ENTER:
+                for mark, event in stream:
+                    if mark is EXIT:
+                        yield mark, event
+                        break
 
 
 class RemoveTransformation(object):
@@ -746,16 +838,21 @@
                 for prefix in element[:-1]:
                     yield None, prefix
                 yield mark, event
-                while True:
-                    try:
-                        mark, event = stream.next()
-                    except StopIteration:
-                        yield None, element[-1]
+                start = mark
+                stopped = False
+                for mark, event in stream:
+                    if start is ENTER and mark is EXIT:
+                        yield mark, event
+                        stopped = True
+                        break
                     if not mark:
                         break
                     yield mark, event
+                else:
+                    stopped = True
                 yield None, element[-1]
-                yield mark, event
+                if not stopped:
+                    yield mark, event
             else:
                 yield mark, event
 
@@ -784,7 +881,7 @@
 
 class FilterTransformation(object):
     """Apply a normal stream filter to the selection. The filter is called once
-    for each contiguous block of marked events."""
+    for each selection."""
 
     def __init__(self, filter):
         """Create the transform.
@@ -806,14 +903,31 @@
 
         queue = []
         for mark, event in stream:
-            if mark:
+            if mark is ENTER:
                 queue.append(event)
-            else:
+                for mark, event in stream:
+                    queue.append(event)
+                    if mark is EXIT:
+                        break
                 for queue_event in flush(queue):
                     yield queue_event
-                yield None, event
-        for event in flush(queue):
-            yield event
+            elif mark is OUTSIDE:
+                stopped = True
+                queue.append(event)
+                for mark, event in stream:
+                    if mark is not OUTSIDE:
+                        break
+                    queue.append(event)
+                else:
+                    stopped = True
+                for queue_event in flush(queue):
+                    yield queue_event
+                if not stopped:
+                    yield None, event
+            else:
+                yield mark, event
+        for queue_event in flush(queue):
+            yield queue_event
 
 
 class MapTransformation(object):
@@ -848,7 +962,7 @@
 
     Refer to the documentation for ``re.sub()`` for details.
     """
-    def __init__(self, pattern, replace, count=1):
+    def __init__(self, pattern, replace, count=0):
         """Create the transform.
 
         :param pattern: A regular expression object, or string.
@@ -868,7 +982,7 @@
         :param stream: The marked event stream to filter
         """
         for mark, (kind, data, pos) in stream:
-            if kind is TEXT:
+            if mark is not None and kind is TEXT:
                 new_data = self.pattern.sub(self.replace, data, self.count)
                 if isinstance(data, Markup):
                     data = Markup(new_data)
@@ -922,7 +1036,10 @@
         self.content = content
 
     def _inject(self):
-        for event in _ensure(self.content):
+        content = self.content
+        if callable(content):
+            content = content()
+        for event in _ensure(content):
             yield None, event
 
 
@@ -934,14 +1051,18 @@
 
         :param stream: The marked event stream to filter
         """
+        stream = PushBackStream(stream)
         for mark, event in stream:
             if mark is not None:
+                start = mark
                 for subevent in self._inject():
                     yield subevent
-                while True:
-                    mark, event = stream.next()
-                    if mark is None:
-                        yield mark, event
+                for mark, event in stream:
+                    if start is ENTER:
+                        if mark is EXIT:
+                            break
+                    elif mark != start:
+                        stream.push((mark, event))
                         break
             else:
                 yield mark, event
@@ -955,17 +1076,22 @@
 
         :param stream: The marked event stream to filter
         """
+        stream = PushBackStream(stream)
         for mark, event in stream:
             if mark is not None:
+                start = mark
                 for subevent in self._inject():
                     yield subevent
                 yield mark, event
-                while True:
-                    mark, event = stream.next()
-                    if not mark:
+                for mark, event in stream:
+                    if mark != start and start is not ENTER:
+                        stream.push((mark, event))
                         break
                     yield mark, event
-            yield mark, event
+                    if start is ENTER and mark is EXIT:
+                        break
+            else:
+                yield mark, event
 
 
 class AfterTransformation(InjectorTransformation):
@@ -976,20 +1102,20 @@
 
         :param stream: The marked event stream to filter
         """
+        stream = PushBackStream(stream)
         for mark, event in stream:
             yield mark, event
             if mark:
-                while True:
-                    try:
-                        mark, event = stream.next()
-                    except StopIteration:
-                        break
-                    if not mark:
+                start = mark
+                for mark, event in stream:
+                    if start is not ENTER and mark != start:
+                        stream.push((mark, event))
                         break
                     yield mark, event
+                    if start is ENTER and mark is EXIT:
+                        break
                 for subevent in self._inject():
                     yield subevent
-                yield mark, event
 
 
 class PrependTransformation(InjectorTransformation):
@@ -1002,7 +1128,7 @@
         """
         for mark, event in stream:
             yield mark, event
-            if mark in (ENTER, OUTSIDE):
+            if mark is ENTER:
                 for subevent in self._inject():
                     yield subevent
 
@@ -1018,8 +1144,7 @@
         for mark, event in stream:
             yield mark, event
             if mark is ENTER:
-                while True:
-                    mark, event = stream.next()
+                for mark, event in stream:
                     if mark is EXIT:
                         break
                     yield mark, event
@@ -1076,32 +1201,50 @@
         self.events.append(event)
 
     def reset(self):
-        """Reset the buffer so that it's empty."""
+        """Empty the buffer of events."""
         del self.events[:]
 
 
 class CopyTransformation(object):
     """Copy selected events into a buffer for later insertion."""
 
-    def __init__(self, buffer):
+    def __init__(self, buffer, accumulate=False):
         """Create the copy transformation.
 
         :param buffer: the `StreamBuffer` in which the selection should be
                        stored
         """
+        if not accumulate:
+            buffer.reset()
         self.buffer = buffer
+        self.accumulate = accumulate
 
     def __call__(self, stream):
         """Apply the transformation to the marked stream.
 
         :param stream: the marked event stream to filter
         """
-        self.buffer.reset()
-        stream = list(stream)
+        stream = PushBackStream(stream)
+
         for mark, event in stream:
             if mark:
+                if not self.accumulate:
+                    self.buffer.reset()
+                events = [(mark, event)]
                 self.buffer.append(event)
-        return stream
+                start = mark
+                for mark, event in stream:
+                    if start is not ENTER and mark != start:
+                        stream.push((mark, event))
+                        break
+                    events.append((mark, event))
+                    self.buffer.append(event)
+                    if start is ENTER and mark is EXIT:
+                        break
+                for i in events:
+                    yield i
+            else:
+                yield mark, event
 
 
 class CutTransformation(object):
@@ -1109,36 +1252,58 @@
     selection.
     """
 
-    def __init__(self, buffer):
+    def __init__(self, buffer, accumulate=False):
         """Create the cut transformation.
 
         :param buffer: the `StreamBuffer` in which the selection should be
                        stored
         """
         self.buffer = buffer
+        self.accumulate = accumulate
+
 
     def __call__(self, stream):
         """Apply the transform filter to the marked stream.
 
         :param stream: the marked event stream to filter
         """
-        out_stream = []
-        attributes = None
-        for mark, (kind, data, pos) in stream:
-            if attributes:
-                assert kind is START
-                data = (data[0], data[1] - attributes)
-                attributes = None
+        attributes = []
+        stream = PushBackStream(stream)
+        broken = False
+        if not self.accumulate:
+            self.buffer.reset()
+        for mark, event in stream:
             if mark:
-                # There is some magic here. ATTR marked events are pushed into
-                # the stream *before* the START event they originated from.
-                # This allows cut() to strip out the attributes from START
-                # event as would be expected.
+                # Send a BREAK event if there was no other event sent between 
+                if not self.accumulate:
+                    if not broken and self.buffer:
+                        yield BREAK, (BREAK, None, None)
+                    self.buffer.reset()
+                self.buffer.append(event)
+                start = mark
                 if mark is ATTR:
-                    self.buffer.append((kind, data, pos))
-                    attributes = [name for name, _ in data[1]]
-                else:
-                    self.buffer.append((kind, data, pos))
+                    attributes.extend([name for name, _ in event[1][1]])
+                for mark, event in stream:
+                    if start is mark is ATTR:
+                        attributes.extend([name for name, _ in event[1][1]])
+                    # Handle non-element contiguous selection
+                    if start is not ENTER and mark != start:
+                        # Operating on the attributes of a START event
+                        if start is ATTR:
+                            kind, data, pos = event
+                            assert kind is START
+                            data = (data[0], data[1] - attributes)
+                            attributes = None
+                            stream.push((mark, (kind, data, pos)))
+                        else:
+                            stream.push((mark, event))
+                        break
+                    self.buffer.append(event)
+                    if start is ENTER and mark is EXIT:
+                        break
+                broken = False
             else:
-                out_stream.append((mark, (kind, data, pos)))
-        return out_stream
+                broken = True
+                yield mark, event
+        if not broken and self.buffer:
+            yield BREAK, (BREAK, None, None)
diff --git a/genshi/output.py b/genshi/output.py
--- a/genshi/output.py
+++ b/genshi/output.py
@@ -114,6 +114,11 @@
     )
     XHTML = XHTML_STRICT
 
+    XHTML11 = (
+        'html', '-//W3C//DTD XHTML 1.1//EN',
+        'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd'
+    )
+
     SVG_FULL = (
         'svg', '-//W3C//DTD SVG 1.1//EN',
         'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd'
@@ -135,11 +140,12 @@
         The following names are recognized in this version:
          * "html" or "html-strict" for the HTML 4.01 strict DTD
          * "html-transitional" for the HTML 4.01 transitional DTD
-         * "html-transitional" for the HTML 4.01 frameset DTD
+         * "html-frameset" for the HTML 4.01 frameset DTD
          * "html5" for the ``DOCTYPE`` proposed for HTML5
          * "xhtml" or "xhtml-strict" for the XHTML 1.0 strict DTD
          * "xhtml-transitional" for the XHTML 1.0 transitional DTD
          * "xhtml-frameset" for the XHTML 1.0 frameset DTD
+         * "xhtml11" for the XHTML 1.1 DTD
          * "svg" or "svg-full" for the SVG 1.1 DTD
          * "svg-basic" for the SVG Basic 1.1 DTD
          * "svg-tiny" for the SVG Tiny 1.1 DTD
@@ -157,6 +163,7 @@
             'xhtml': cls.XHTML, 'xhtml-strict': cls.XHTML_STRICT,
             'xhtml-transitional': cls.XHTML_TRANSITIONAL,
             'xhtml-frameset': cls.XHTML_FRAMESET,
+            'xhtml11': cls.XHTML11,
             'svg': cls.SVG, 'svg-full': cls.SVG_FULL,
             'svg-basic': cls.SVG_BASIC,
             'svg-tiny': cls.SVG_TINY
@@ -280,7 +287,7 @@
     ])
 
     def __init__(self, doctype=None, strip_whitespace=True,
-                 namespace_prefixes=None):
+                 namespace_prefixes=None, drop_xml_decl=True):
         super(XHTMLSerializer, self).__init__(doctype, False)
         self.filters = [EmptyTagFilter()]
         if strip_whitespace:
@@ -290,11 +297,13 @@
         self.filters.append(NamespaceFlattener(prefixes=namespace_prefixes))
         if doctype:
             self.filters.append(DocTypeInserter(doctype))
+        self.drop_xml_decl = drop_xml_decl
 
     def __call__(self, stream):
         boolean_attrs = self._BOOLEAN_ATTRS
         empty_elems = self._EMPTY_ELEMS
-        have_doctype = False
+        drop_xml_decl = self.drop_xml_decl
+        have_decl = have_doctype = False
         in_cdata = False
 
         for filter_ in self.filters:
@@ -346,6 +355,18 @@
                 yield Markup(u''.join(buf)) % filter(None, data)
                 have_doctype = True
 
+            elif kind is XML_DECL and not have_decl and not drop_xml_decl:
+                version, encoding, standalone = data
+                buf = ['<?xml version="%s"' % version]
+                if encoding:
+                    buf.append(' encoding="%s"' % encoding)
+                if standalone != -1:
+                    standalone = standalone and 'yes' or 'no'
+                    buf.append(' standalone="%s"' % standalone)
+                buf.append('?>\n')
+                yield Markup(u''.join(buf))
+                have_decl = True
+
             elif kind is START_CDATA:
                 yield Markup('<![CDATA[')
                 in_cdata = True
@@ -473,7 +494,7 @@
     >>> print elem.generate().render(TextSerializer)
     <a href="foo">Hello &amp; Bye!</a><br/>
     
-    You can use the `strip_markup` to change this behavior, so that tags and
+    You can use the ``strip_markup`` to change this behavior, so that tags and
     entities are stripped from the output (or in the case of entities,
     replaced with the equivalent character):
 
@@ -482,6 +503,11 @@
     """
 
     def __init__(self, strip_markup=False):
+        """Create the serializer.
+        
+        :param strip_markup: whether markup (tags and encoded characters) found
+                             in the text should be removed
+        """
         self.strip_markup = strip_markup
 
     def __call__(self, stream):
diff --git a/genshi/template/base.py b/genshi/template/base.py
--- a/genshi/template/base.py
+++ b/genshi/template/base.py
@@ -244,6 +244,10 @@
         """
         return [(key, self.get(key)) for key in self.keys()]
 
+    def update(self, mapping):
+        """Update the context from the mapping provided."""
+        self.frames[0].update(mapping)
+
     def push(self, data):
         """Push a new scope on the stack.
         
diff --git a/genshi/template/eval.py b/genshi/template/eval.py
--- a/genshi/template/eval.py
+++ b/genshi/template/eval.py
@@ -22,7 +22,6 @@
 except NameError:
     from sets import ImmutableSet as frozenset
     from sets import Set as set
-import sys
 from textwrap import dedent
 
 from genshi.core import Markup
@@ -33,6 +32,29 @@
            'Undefined', 'UndefinedError']
 __docformat__ = 'restructuredtext en'
 
+# Check for a Python 2.4 bug in the eval loop
+has_star_import_bug = False
+try:
+    class _FakeMapping(object):
+        __getitem__ = __setitem__ = lambda *a: None
+    exec 'from sys import *' in {}, _FakeMapping()
+except SystemError:
+    has_star_import_bug = True
+except TypeError:
+    pass # Python 2.3
+del _FakeMapping
+
+def _star_import_patch(mapping, modname):
+    """This function is used as helper if a Python version with a broken
+    star-import opcode is in use.
+    """
+    module = __import__(modname, None, None, ['__all__'])
+    if hasattr(module, '__all__'):
+        members = module.__all__
+    else:
+        members = [x for x in module.__dict__ if not x.startswith('_')]
+    mapping.update([(name, getattr(module, name)) for name in members])
+
 
 class Code(object):
     """Abstract base class for the `Expression` and `Suite` classes."""
@@ -270,6 +292,7 @@
             '_lookup_name': cls.lookup_name,
             '_lookup_attr': cls.lookup_attr,
             '_lookup_item': cls.lookup_item,
+            '_star_import_patch': _star_import_patch,
             'UndefinedError': UndefinedError,
         }
     globals = classmethod(globals)
@@ -462,7 +485,7 @@
         if lineno is not None:
             node.lineno = lineno
         if isinstance(node, (ast.Class, ast.Function, ast.Lambda)) or \
-                sys.version_info > (2, 4) and isinstance(node, ast.GenExpr):
+                hasattr(ast, 'GenExpr') and isinstance(node, ast.GenExpr):
             node.filename = '<string>' # workaround for bug in pycodegen
         return node
 
@@ -492,6 +515,19 @@
             node.doc, self.visit(node.code)
         )
 
+    def visitFrom(self, node):
+        if not has_star_import_bug or node.names != [('*', None)]:
+            # This is a Python 2.4 bug. Only if we have a broken Python
+            # version we have to apply the hack
+            return node
+        new_node = ast.Discard(ast.CallFunc(
+            ast.Name('_star_import_patch'),
+            [ast.Name('__data__'), ast.Const(node.modname)], None, None
+        ))
+        if hasattr(node, 'lineno'): # No lineno in Python 2.3
+            new_node.lineno = node.lineno
+        return new_node
+
     def visitFunction(self, node):
         args = []
         if hasattr(node, 'decorators'):
diff --git a/genshi/template/interpolation.py b/genshi/template/interpolation.py
--- a/genshi/template/interpolation.py
+++ b/genshi/template/interpolation.py
@@ -17,7 +17,8 @@
 
 from itertools import chain
 import os
-from tokenize import tokenprog
+import re
+from tokenize import PseudoToken
 
 from genshi.core import TEXT
 from genshi.template.base import TemplateSyntaxError, EXPR
@@ -30,6 +31,11 @@
 NAMECHARS = NAMESTART + '.0123456789'
 PREFIX = '$'
 
+token_re = re.compile('%s|%s(?s)' % (
+    r'[uU]?[rR]?("""|\'\'\')((?<!\\)\\\1|.)*?\1',
+    PseudoToken
+))
+
 def interpolate(text, filepath=None, lineno=-1, offset=0, lookup='strict'):
     """Parse the given string and extract expressions.
     
@@ -106,7 +112,7 @@
             pos = offset + 2
             level = 1
             while level:
-                match = tokenprog.match(text, pos)
+                match = token_re.match(text, pos)
                 if match is None:
                     raise TemplateSyntaxError('invalid syntax',  filepath,
                                               *textpos[1:])
diff --git a/genshi/template/loader.py b/genshi/template/loader.py
--- a/genshi/template/loader.py
+++ b/genshi/template/loader.py
@@ -22,7 +22,8 @@
 from genshi.template.base import TemplateError
 from genshi.util import LRUCache
 
-__all__ = ['TemplateLoader', 'TemplateNotFound']
+__all__ = ['TemplateLoader', 'TemplateNotFound', 'directory', 'package',
+           'prefixed']
 __docformat__ = 'restructuredtext en'
 
 
@@ -163,8 +164,14 @@
         """
         if cls is None:
             cls = self.default_class
-        if relative_to and not os.path.isabs(relative_to):
+        search_path = self.search_path
+
+        # Make the filename relative to the template file its being loaded
+        # from, but only if that file is specified as a relative path, or no
+        # search path has been set up
+        if relative_to and (not search_path or not os.path.isabs(relative_to)):
             filename = os.path.join(os.path.dirname(relative_to), filename)
+
         filename = os.path.normpath(filename)
         cachekey = filename
 
@@ -181,7 +188,6 @@
             except (KeyError, OSError):
                 pass
 
-            search_path = self.search_path
             isabs = False
 
             if os.path.isabs(filename):
diff --git a/genshi/template/markup.py b/genshi/template/markup.py
--- a/genshi/template/markup.py
+++ b/genshi/template/markup.py
@@ -303,7 +303,9 @@
 
                     # Make the select() function available in the body of the
                     # match template
+                    selected = [False]
                     def select(path):
+                        selected[0] = True
                         return Stream(content).select(path, namespaces, ctxt)
                     vars = dict(select=select)
 
@@ -320,6 +322,13 @@
                             **vars):
                         yield event
 
+                    # If the match template did not actually call select to
+                    # consume the matched stream, the original events need to
+                    # be consumed here or they'll get appended to the output
+                    if not selected[0]:
+                        for event in content:
+                            pass
+
                     break
 
             else: # no matches
diff --git a/genshi/template/plugin.py b/genshi/template/plugin.py
--- a/genshi/template/plugin.py
+++ b/genshi/template/plugin.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2006-2007 Edgewall Software
+# Copyright (C) 2006-2008 Edgewall Software
 # Copyright (C) 2006 Matthew Good
 # All rights reserved.
 #
@@ -16,8 +16,6 @@
 CherryPy/Buffet.
 """
 
-from pkg_resources import resource_filename
-
 from genshi.input import ET, HTML, XML
 from genshi.output import DocType
 from genshi.template.base import Template
@@ -91,6 +89,7 @@
         if self.use_package_naming:
             divider = templatename.rfind('.')
             if divider >= 0:
+                from pkg_resources import resource_filename
                 package = templatename[:divider]
                 basename = templatename[divider + 1:] + self.extension
                 templatename = resource_filename(package, basename)
diff --git a/genshi/template/tests/eval.py b/genshi/template/tests/eval.py
--- a/genshi/template/tests/eval.py
+++ b/genshi/template/tests/eval.py
@@ -18,6 +18,7 @@
 import unittest
 
 from genshi.core import Markup
+from genshi.template.base import Context, _ctxt2dict
 from genshi.template.eval import Expression, Suite, Undefined, UndefinedError, \
                                  UNDEFINED
 
@@ -571,6 +572,12 @@
         suite.execute(data)
         assert 'ifilter' in data
 
+    def test_import_star(self):
+        suite = Suite("from itertools import *")
+        data = Context()
+        suite.execute(_ctxt2dict(data))
+        assert 'ifilter' in data
+
     def test_for(self):
         suite = Suite("""x = []
 for i in range(3):
diff --git a/genshi/template/tests/interpolation.py b/genshi/template/tests/interpolation.py
--- a/genshi/template/tests/interpolation.py
+++ b/genshi/template/tests/interpolation.py
@@ -186,6 +186,11 @@
         self.assertEqual(TEXT, parts[2][0])
         self.assertEqual(' baz', parts[2][1])
 
+    def test_interpolate_triplequoted(self):
+        parts = list(interpolate('${"""foo\nbar"""}'))
+        self.assertEqual(1, len(parts))
+        self.assertEqual('"""foo\nbar"""', parts[0][1].source)
+
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/template/tests/loader.py b/genshi/template/tests/loader.py
--- a/genshi/template/tests/loader.py
+++ b/genshi/template/tests/loader.py
@@ -261,6 +261,47 @@
             </html>""", tmpl1.generate().render())
         assert 'tmpl2.html' in loader._cache
 
+    def test_abspath_include_caching_without_search_path(self):
+        file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w')
+        try:
+            file1.write("""<html xmlns:xi="http://www.w3.org/2001/XInclude">
+              <xi:include href="tmpl2.html" />
+            </html>""")
+        finally:
+            file1.close()
+
+        file2 = open(os.path.join(self.dirname, 'tmpl2.html'), 'w')
+        try:
+            file2.write("""<div>Included</div>""")
+        finally:
+            file2.close()
+
+        os.mkdir(os.path.join(self.dirname, 'sub'))
+        file3 = open(os.path.join(self.dirname, 'sub', 'tmpl1.html'), 'w')
+        try:
+            file3.write("""<html xmlns:xi="http://www.w3.org/2001/XInclude">
+              <xi:include href="tmpl2.html" />
+            </html>""")
+        finally:
+            file3.close()
+
+        file4 = open(os.path.join(self.dirname, 'sub', 'tmpl2.html'), 'w')
+        try:
+            file4.write("""<div>Included from sub</div>""")
+        finally:
+            file4.close()
+
+        loader = TemplateLoader()
+        tmpl1 = loader.load(os.path.join(self.dirname, 'tmpl1.html'))
+        self.assertEqual("""<html>
+              <div>Included</div>
+            </html>""", tmpl1.generate().render())
+        tmpl2 = loader.load(os.path.join(self.dirname, 'sub', 'tmpl1.html'))
+        self.assertEqual("""<html>
+              <div>Included from sub</div>
+            </html>""", tmpl2.generate().render())
+        assert 'tmpl2.html' not in loader._cache
+
     def test_load_with_default_encoding(self):
         f = open(os.path.join(self.dirname, 'tmpl.html'), 'w')
         try:
diff --git a/genshi/template/tests/markup.py b/genshi/template/tests/markup.py
--- a/genshi/template/tests/markup.py
+++ b/genshi/template/tests/markup.py
@@ -691,6 +691,46 @@
         finally:
             shutil.rmtree(dirname)
 
+    def test_nested_matches_without_buffering(self):
+        xml = ("""<html xmlns:py="http://genshi.edgewall.org/">
+          <py:match path="body" once="true" buffer="false">
+            <body>
+              ${select('*|text')}
+              And some other stuff...
+            </body>
+          </py:match>
+          <body>
+            <span py:match="span">Foo</span>
+            <span>Bar</span>
+          </body>
+        </html>""")
+        tmpl = MarkupTemplate(xml, filename='test.html')
+        self.assertEqual("""<html>
+            <body>
+              <span>Foo</span>
+              And some other stuff...
+            </body>
+        </html>""", tmpl.generate().render())
+
+    def test_match_without_select(self):
+        # See <http://genshi.edgewall.org/ticket/243>
+        xml = ("""<html xmlns:py="http://genshi.edgewall.org/">
+          <py:match path="body" buffer="false">
+            <body>
+              This replaces the other text.
+            </body>
+          </py:match>
+          <body>
+            This gets replaced.
+          </body>
+        </html>""")
+        tmpl = MarkupTemplate(xml, filename='test.html')
+        self.assertEqual("""<html>
+            <body>
+              This replaces the other text.
+            </body>
+        </html>""", tmpl.generate().render())
+
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/tests/builder.py b/genshi/tests/builder.py
--- a/genshi/tests/builder.py
+++ b/genshi/tests/builder.py
@@ -16,7 +16,7 @@
 import unittest
 
 from genshi.builder import Element, tag
-from genshi.core import Attrs, Stream
+from genshi.core import Attrs, Markup, Stream
 from genshi.input import XML
 
 
@@ -42,6 +42,15 @@
                           (None, -1, -1)),
                          event)
 
+    def test_duplicate_attributes(self):
+        link = tag.a(href='#1', href_='#2')('Bar')
+        bits = iter(link.generate())
+        self.assertEqual((Stream.START,
+                          ('a', Attrs([('href', "#1")])),
+                          (None, -1, -1)), bits.next())
+        self.assertEqual((Stream.TEXT, u'Bar', (None, -1, -1)), bits.next())
+        self.assertEqual((Stream.END, 'a', (None, -1, -1)), bits.next())
+
     def test_stream_as_child(self):
         xml = list(tag.span(XML('<b>Foo</b>')).generate())
         self.assertEqual(5, len(xml))
@@ -51,6 +60,12 @@
         self.assertEqual((Stream.END, 'b'), xml[3][:2])
         self.assertEqual((Stream.END, 'span'), xml[4][:2])
 
+    def test_markup_escape(self):
+        from genshi.core import Markup
+        m = Markup('See %s') % tag.a('genshi',
+                                     href='http://genshi.edgwall.org')
+        self.assertEqual(m, Markup('See <a href="http://genshi.edgwall.org">'
+                                   'genshi</a>'))
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/tests/output.py b/genshi/tests/output.py
--- a/genshi/tests/output.py
+++ b/genshi/tests/output.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2006-2007 Edgewall Software
+# Copyright (C) 2006-2008 Edgewall Software
 # All rights reserved.
 #
 # This software is licensed as described in the file COPYING, which
@@ -23,7 +23,7 @@
 
 class XMLSerializerTestCase(unittest.TestCase):
 
-    def test_xml_serialiser_with_decl(self):
+    def test_with_xml_decl(self):
         stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))])
         output = stream.render(XMLSerializer, doctype='xhtml')
         self.assertEqual('<?xml version="1.0"?>\n'
@@ -203,6 +203,24 @@
 
 class XHTMLSerializerTestCase(unittest.TestCase):
 
+    def test_xml_decl_dropped(self):
+        stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))])
+        output = stream.render(XHTMLSerializer, doctype='xhtml')
+        self.assertEqual('<!DOCTYPE html PUBLIC '
+                         '"-//W3C//DTD XHTML 1.0 Strict//EN" '
+                         '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n',
+                         output)
+
+    def test_xml_decl_included(self):
+        stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))])
+        output = stream.render(XHTMLSerializer, doctype='xhtml',
+                               drop_xml_decl=False)
+        self.assertEqual('<?xml version="1.0"?>\n'
+                         '<!DOCTYPE html PUBLIC '
+                         '"-//W3C//DTD XHTML 1.0 Strict//EN" '
+                         '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n',
+                         output)
+
     def test_xml_lang(self):
         text = '<p xml:lang="en">English text</p>'
         output = XML(text).render(XHTMLSerializer)
@@ -372,7 +390,7 @@
 
     def test_empty_script(self):
         text = '<script src="foo.js" />'
-        output = XML(text).render(XHTMLSerializer)
+        output = XML(text).render(HTMLSerializer)
         self.assertEqual('<script src="foo.js"></script>', output)
 
     def test_script_escaping(self):
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@
 
 setup(
     name = 'Genshi',
-    version = '0.5',
+    version = '0.5.2',
     description = 'A toolkit for generation of output for the web',
     long_description = \
 """Genshi is a Python library that provides an integrated set of