# HG changeset patch
# User cmlenz
# Date 1154616562 0
# Node ID 10279d2eeec99cd32595f884c96e8c2d2625f14d
# Parent  6c5c6f67d3e8a03a57d07ac24d209d42ec8b5e98
Fix for #18: whitespace in space-sensitive elements such as `<pre>` and `<textarea>` is now preserved.

diff --git a/examples/bench/bigtable.py b/examples/bench/bigtable.py
--- a/examples/bench/bigtable.py
+++ b/examples/bench/bigtable.py
@@ -50,7 +50,7 @@
     """Markup template"""
     ctxt = Context(table=table)
     stream = markup_tmpl.generate(ctxt)
-    stream.render('html')
+    stream.render('html', strip_whitespace=False)
 
 def test_markup_builder():
     """Markup template + tag builder"""
@@ -60,7 +60,7 @@
     ]).generate()
     ctxt = Context(table=stream)
     stream = markup_tmpl2.generate(ctxt)
-    stream.render('html')
+    stream.render('html', strip_whitespace=False)
 
 def test_builder():
     """Markup tag builder"""
@@ -70,7 +70,7 @@
         ])
         for row in table
     ]).generate()
-    stream.render('html')
+    stream.render('html', strip_whitespace=False)
 
 def test_kid():
     """Kid template"""
diff --git a/markup/core.py b/markup/core.py
--- a/markup/core.py
+++ b/markup/core.py
@@ -64,16 +64,19 @@
     def __iter__(self):
         return iter(self.events)
 
-    def filter(self, func):
-        """Apply a filter to the stream.
+    def filter(self, *filters):
+        """Apply filters to the stream.
         
-        This method returns a new stream with the given filter applied. The
-        filter must be a callable that accepts the stream object as parameter,
-        and returns the filtered stream.
+        This method returns a new stream with the given filters applied. The
+        filters must be callables that accept the stream object as parameter,
+        and return the filtered stream.
         """
-        return Stream(func(self))
+        stream = self
+        for filter_ in filters:
+            stream = filter_(iter(stream))
+        return Stream(stream)
 
-    def render(self, method='xml', encoding='utf-8', filters=None, **kwargs):
+    def render(self, method='xml', encoding='utf-8', **kwargs):
         """Return a string representation of the stream.
         
         @param method: determines how the stream is serialized; can be either
@@ -85,7 +88,7 @@
         Any additional keyword arguments are passed to the serializer, and thus
         depend on the `method` parameter value.
         """
-        generator = self.serialize(method=method, filters=filters, **kwargs)
+        generator = self.serialize(method=method, **kwargs)
         output = u''.join(list(generator))
         if encoding is not None:
             return output.encode(encoding)
@@ -100,7 +103,7 @@
         from markup.path import Path
         return Path(path).select(self)
 
-    def serialize(self, method='xml', filters=None, **kwargs):
+    def serialize(self, method='xml', **kwargs):
         """Generate strings corresponding to a specific serialization of the
         stream.
         
@@ -109,30 +112,16 @@
         string.
         
         @param method: determines how the stream is serialized; can be either
-                       "xml", "xhtml", or "html", or a custom `Serializer`
-                       subclass
-        @param filters: list of filters to apply to the stream before
-                        serialization. The default is to apply whitespace
-                        reduction using `markup.filters.WhitespaceFilter`.
+                       "xml", "xhtml", or "html", or a custom serializer class
         """
-        from markup.filters import WhitespaceFilter
         from markup import output
         cls = method
         if isinstance(method, basestring):
             cls = {'xml':   output.XMLSerializer,
                    'xhtml': output.XHTMLSerializer,
                    'html':  output.HTMLSerializer}[method]
-        else:
-            assert issubclass(cls, output.Serializer)
-        serializer = cls(**kwargs)
-
-        stream = _ensure(self)
-        if filters is None:
-            filters = [WhitespaceFilter()]
-        for filter_ in filters:
-            stream = filter_(iter(stream))
-
-        return serializer.serialize(stream)
+        serialize = cls(**kwargs)
+        return serialize(_ensure(self))
 
     def __str__(self):
         return self.render()
diff --git a/markup/filters.py b/markup/filters.py
--- a/markup/filters.py
+++ b/markup/filters.py
@@ -24,7 +24,97 @@
 from markup.core import END, END_NS, START, START_NS, TEXT
 from markup.path import Path
 
-__all__ = ['IncludeFilter', 'WhitespaceFilter', 'HTMLSanitizer']
+__all__ = ['HTMLSanitizer', 'IncludeFilter']
+
+
+class HTMLSanitizer(object):
+    """A filter that removes potentially dangerous HTML tags and attributes
+    from the stream.
+    """
+
+    _SAFE_TAGS = frozenset(['a', 'abbr', 'acronym', 'address', 'area', 'b',
+        'big', 'blockquote', 'br', 'button', 'caption', 'center', 'cite',
+        'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt',
+        'em', 'fieldset', 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+        'hr', 'i', 'img', 'input', 'ins', 'kbd', 'label', 'legend', 'li', 'map',
+        'menu', 'ol', 'optgroup', 'option', 'p', 'pre', 'q', 's', 'samp',
+        'select', 'small', 'span', 'strike', 'strong', 'sub', 'sup', 'table',
+        'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u',
+        'ul', 'var'])
+
+    _SAFE_ATTRS = frozenset(['abbr', 'accept', 'accept-charset', 'accesskey',
+        'action', 'align', 'alt', 'axis', 'bgcolor', 'border', 'cellpadding',
+        'cellspacing', 'char', 'charoff', 'charset', 'checked', 'cite', 'class',
+        'clear', 'cols', 'colspan', 'color', 'compact', 'coords', 'datetime',
+        'dir', 'disabled', 'enctype', 'for', 'frame', 'headers', 'height',
+        'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang',
+        'longdesc', 'maxlength', 'media', 'method', 'multiple', 'name',
+        'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', 'rel', 'rev',
+        'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
+        'span', 'src', 'start', 'style', 'summary', 'tabindex', 'target',
+        'title', 'type', 'usemap', 'valign', 'value', 'vspace', 'width'])
+    _URI_ATTRS = frozenset(['action', 'background', 'dynsrc', 'href', 'lowsrc',
+        'src'])
+    _SAFE_SCHEMES = frozenset(['file', 'ftp', 'http', 'https', 'mailto', None])
+
+    def __call__(self, stream, ctxt=None):
+        waiting_for = None
+
+        for kind, data, pos in stream:
+            if kind is START:
+                if waiting_for:
+                    continue
+                tag, attrib = data
+                if tag not in self._SAFE_TAGS:
+                    waiting_for = tag
+                    continue
+
+                new_attrib = Attributes()
+                for attr, value in attrib:
+                    value = stripentities(value)
+                    if attr not in self._SAFE_ATTRS:
+                        continue
+                    elif attr in self._URI_ATTRS:
+                        # Don't allow URI schemes such as "javascript:"
+                        if self._get_scheme(value) not in self._SAFE_SCHEMES:
+                            continue
+                    elif attr == 'style':
+                        # Remove dangerous CSS declarations from inline styles
+                        decls = []
+                        for decl in filter(None, value.split(';')):
+                            is_evil = False
+                            if 'expression' in decl:
+                                is_evil = True
+                            for m in re.finditer(r'url\s*\(([^)]+)', decl):
+                                if self._get_scheme(m.group(1)) not in self._SAFE_SCHEMES:
+                                    is_evil = True
+                                    break
+                            if not is_evil:
+                                decls.append(decl.strip())
+                        if not decls:
+                            continue
+                        value = '; '.join(decls)
+                    new_attrib.append((attr, value))
+
+                yield kind, (tag, new_attrib), pos
+
+            elif kind is END:
+                tag = data
+                if waiting_for:
+                    if waiting_for == tag:
+                        waiting_for = None
+                else:
+                    yield kind, data, pos
+
+            else:
+                if not waiting_for:
+                    yield kind, data, pos
+
+    def _get_scheme(self, text):
+        if ':' not in text:
+            return None
+        chars = [char for char in text.split(':', 1)[0] if char.isalnum()]
+        return ''.join(chars).lower()
 
 
 class IncludeFilter(object):
@@ -101,128 +191,3 @@
 
             else:
                 yield kind, data, pos
-
-
-class WhitespaceFilter(object):
-    """A filter that removes extraneous white space from the stream.
-
-    TODO:
-     * Support for xml:space
-    """
-    _TRAILING_SPACE = re.compile('[ \t]+(?=\n)')
-    _LINE_COLLAPSE = re.compile('\n{2,}')
-
-    def __call__(self, stream, ctxt=None):
-        trim_trailing_space = self._TRAILING_SPACE.sub
-        collapse_lines = self._LINE_COLLAPSE.sub
-        mjoin = Markup('').join
-
-        textbuf = []
-        for kind, data, pos in chain(stream, [(None, None, None)]):
-            if kind is TEXT:
-                textbuf.append(data)
-            else:
-                if textbuf:
-                    if len(textbuf) > 1:
-                        output = Markup(collapse_lines('\n',
-                            trim_trailing_space('',
-                                mjoin(textbuf, escape_quotes=False))))
-                        del textbuf[:]
-                        yield TEXT, output, pos
-                    else:
-                        output = Markup(collapse_lines('\n',
-                            trim_trailing_space('',
-                                escape(textbuf.pop(), quotes=False))))
-                        yield TEXT, output, pos
-                if kind is not None:
-                    yield kind, data, pos
-
-
-class HTMLSanitizer(object):
-    """A filter that removes potentially dangerous HTML tags and attributes
-    from the stream.
-    """
-
-    _SAFE_TAGS = frozenset(['a', 'abbr', 'acronym', 'address', 'area', 'b',
-        'big', 'blockquote', 'br', 'button', 'caption', 'center', 'cite',
-        'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt',
-        'em', 'fieldset', 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
-        'hr', 'i', 'img', 'input', 'ins', 'kbd', 'label', 'legend', 'li', 'map',
-        'menu', 'ol', 'optgroup', 'option', 'p', 'pre', 'q', 's', 'samp',
-        'select', 'small', 'span', 'strike', 'strong', 'sub', 'sup', 'table',
-        'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u',
-        'ul', 'var'])
-
-    _SAFE_ATTRS = frozenset(['abbr', 'accept', 'accept-charset', 'accesskey',
-        'action', 'align', 'alt', 'axis', 'bgcolor', 'border', 'cellpadding',
-        'cellspacing', 'char', 'charoff', 'charset', 'checked', 'cite', 'class',
-        'clear', 'cols', 'colspan', 'color', 'compact', 'coords', 'datetime',
-        'dir', 'disabled', 'enctype', 'for', 'frame', 'headers', 'height',
-        'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang',
-        'longdesc', 'maxlength', 'media', 'method', 'multiple', 'name',
-        'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', 'rel', 'rev',
-        'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
-        'span', 'src', 'start', 'style', 'summary', 'tabindex', 'target',
-        'title', 'type', 'usemap', 'valign', 'value', 'vspace', 'width'])
-    _URI_ATTRS = frozenset(['action', 'background', 'dynsrc', 'href', 'lowsrc',
-        'src'])
-    _SAFE_SCHEMES = frozenset(['file', 'ftp', 'http', 'https', 'mailto', None])
-
-    def __call__(self, stream, ctxt=None):
-        waiting_for = None
-
-        for kind, data, pos in stream:
-            if kind is START:
-                if waiting_for:
-                    continue
-                tag, attrib = data
-                if tag not in self._SAFE_TAGS:
-                    waiting_for = tag
-                    continue
-
-                new_attrib = []
-                for attr, value in attrib:
-                    value = stripentities(value)
-                    if attr not in self._SAFE_ATTRS:
-                        continue
-                    elif attr in self._URI_ATTRS:
-                        # Don't allow URI schemes such as "javascript:"
-                        if self._get_scheme(value) not in self._SAFE_SCHEMES:
-                            continue
-                    elif attr == 'style':
-                        # Remove dangerous CSS declarations from inline styles
-                        decls = []
-                        for decl in filter(None, value.split(';')):
-                            is_evil = False
-                            if 'expression' in decl:
-                                is_evil = True
-                            for m in re.finditer(r'url\s*\(([^)]+)', decl):
-                                if self._get_scheme(m.group(1)) not in self._SAFE_SCHEMES:
-                                    is_evil = True
-                                    break
-                            if not is_evil:
-                                decls.append(decl.strip())
-                        if not decls:
-                            continue
-                        value = '; '.join(decls)
-                    new_attrib.append((attr, value))
-
-                yield kind, (tag, new_attrib), pos
-
-            elif kind is END:
-                tag = data
-                if waiting_for:
-                    if waiting_for == tag:
-                        waiting_for = None
-                else:
-                    yield kind, data, pos
-
-            else:
-                if not waiting_for:
-                    yield kind, data, pos
-
-    def _get_scheme(self, text):
-        if ':' not in text:
-            return None
-        chars = [char for char in text.split(':', 1)[0] if char.isalnum()]
-        return ''.join(chars).lower()
diff --git a/markup/output.py b/markup/output.py
--- a/markup/output.py
+++ b/markup/output.py
@@ -15,11 +15,12 @@
 streams.
 """
 
+from itertools import chain
 try:
     frozenset
 except NameError:
     from sets import ImmutableSet as frozenset
-from itertools import chain
+import re
 
 from markup.core import escape, Markup, Namespace, QName
 from markup.core import DOCTYPE, START, END, START_NS, END_NS, TEXT, COMMENT, PI
@@ -27,19 +28,6 @@
 __all__ = ['Serializer', 'XMLSerializer', 'HTMLSerializer']
 
 
-class Serializer(object):
-    """Base class for serializers."""
-
-    def serialize(self, stream):
-        """Must be implemented by concrete subclasses to serialize the given
-        stream.
-        
-        This method must be implemented as a generator, producing the
-        serialized output incrementally as unicode strings.
-        """
-        raise NotImplementedError
-
-
 class DocType(object):
     """Defines a number of commonly used DOCTYPE declarations as constants."""
 
@@ -56,56 +44,66 @@
     XHTML = XHTML_STRICT
 
 
-class XMLSerializer(Serializer):
+class XMLSerializer(object):
     """Produces XML text from an event stream.
     
     >>> from markup.builder import tag
     >>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True))
-    >>> print ''.join(XMLSerializer().serialize(elem.generate()))
+    >>> print ''.join(XMLSerializer()(elem.generate()))
     <div><a href="foo"/><br/><hr noshade="True"/></div>
     """
-    def __init__(self, doctype=None):
+
+    _PRESERVE_SPACE = frozenset()
+
+    def __init__(self, doctype=None, strip_whitespace=True):
         """Initialize the XML serializer.
         
         @param doctype: a `(name, pubid, sysid)` tuple that represents the
             DOCTYPE declaration that should be included at the top of the
             generated output
+        @param strip_whitespace: whether extraneous whitespace should be
+            stripped from the output
         """
         self.preamble = []
         if doctype:
             self.preamble.append((DOCTYPE, doctype, (None, -1, -1)))
+        self.filters = []
+        if strip_whitespace:
+            self.filters.append(WhitespaceFilter(self._PRESERVE_SPACE))
 
-    def serialize(self, stream):
+    def __call__(self, stream):
         have_doctype = False
         ns_attrib = []
         ns_mapping = {}
 
-        stream = _PushbackIterator(chain(self.preamble, stream))
+        stream = chain(self.preamble, stream)
+        for filter_ in self.filters:
+            stream = filter_(stream)
+        stream = _PushbackIterator(stream)
         for kind, data, pos in stream:
 
             if kind is START:
                 tag, attrib = data
 
                 tagname = tag.localname
-                if tag.namespace:
-                    try:
-                        prefix = ns_mapping[tag.namespace]
+                namespace = tag.namespace
+                if namespace:
+                    if namespace in ns_mapping:
+                        prefix = ns_mapping[namespace]
                         if prefix:
-                            tagname = '%s:%s' % (prefix, tag.localname)
-                    except KeyError:
-                        ns_attrib.append((QName('xmlns'), tag.namespace))
+                            tagname = '%s:%s' % (prefix, tagname)
+                    else:
+                        ns_attrib.append((QName('xmlns'), namespace))
                 buf = ['<%s' % tagname]
 
-                if ns_attrib:
-                    attrib.extend(ns_attrib)
-                    ns_attrib = []
-                for attr, value in attrib:
+                for attr, value in attrib + ns_attrib:
                     attrname = attr.localname
                     if attr.namespace:
                         prefix = ns_mapping.get(attr.namespace)
                         if prefix:
                             attrname = '%s:%s' % (prefix, attrname)
                     buf.append(' %s="%s"' % (attrname, escape(value)))
+                ns_attrib = []
 
                 kind, data, pos = stream.next()
                 if kind is END:
@@ -163,7 +161,7 @@
     
     >>> from markup.builder import tag
     >>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True))
-    >>> print ''.join(XHTMLSerializer().serialize(elem.generate()))
+    >>> print ''.join(XHTMLSerializer()(elem.generate()))
     <div><a href="foo"></a><br /><hr noshade="noshade" /></div>
     """
 
@@ -175,12 +173,16 @@
     _BOOLEAN_ATTRS = frozenset(['selected', 'checked', 'compact', 'declare',
                                 'defer', 'disabled', 'ismap', 'multiple',
                                 'nohref', 'noresize', 'noshade', 'nowrap'])
+    _PRESERVE_SPACE = frozenset([QName('pre'), QName('textarea')])
 
-    def serialize(self, stream):
+    def __call__(self, stream):
         have_doctype = False
         ns_mapping = {}
 
-        stream = _PushbackIterator(chain(self.preamble, stream))
+        stream = chain(self.preamble, stream)
+        for filter_ in self.filters:
+            stream = filter_(stream)
+        stream = _PushbackIterator(stream)
         for kind, data, pos in stream:
 
             if kind is START:
@@ -250,15 +252,18 @@
     
     >>> from markup.builder import tag
     >>> elem = tag.div(tag.a(href='foo'), tag.br, tag.hr(noshade=True))
-    >>> print ''.join(HTMLSerializer().serialize(elem.generate()))
+    >>> print ''.join(HTMLSerializer()(elem.generate()))
     <div><a href="foo"></a><br><hr noshade></div>
     """
 
-    def serialize(self, stream):
+    def __call__(self, stream):
         have_doctype = False
         ns_mapping = {}
 
-        stream = _PushbackIterator(chain(self.preamble, stream))
+        stream = chain(self.preamble, stream)
+        for filter_ in self.filters:
+            stream = filter_(stream)
+        stream = _PushbackIterator(stream)
         for kind, data, pos in stream:
 
             if kind is START:
@@ -268,7 +273,8 @@
                 buf = ['<', tag.localname]
 
                 for attr, value in attrib:
-                    if attr.namespace and attr not in self.NAMESPACE:
+                    if attr.namespace and attr not in self.NAMESPACE \
+                            or attr.localname.startswith('xml:'):
                         continue # not in the HTML namespace, so don't emit
                     if attr.localname in self._BOOLEAN_ATTRS:
                         if value:
@@ -318,6 +324,52 @@
                 yield Markup('<?%s %s?>' % data)
 
 
+class WhitespaceFilter(object):
+    """A filter that removes extraneous ignorable white space from the
+    stream."""
+
+    _TRAILING_SPACE = re.compile('[ \t]+(?=\n)')
+    _LINE_COLLAPSE = re.compile('\n{2,}')
+
+    def __init__(self, preserve=None):
+        """Initialize the filter.
+        
+        @param preserve: a sequence of tag names for which white-space should
+            be ignored.
+        """
+        if preserve is None:
+            preserve = []
+        self.preserve = frozenset(preserve)
+
+    def __call__(self, stream, ctxt=None):
+        trim_trailing_space = self._TRAILING_SPACE.sub
+        collapse_lines = self._LINE_COLLAPSE.sub
+        mjoin = Markup('').join
+        preserve = [False]
+
+        textbuf = []
+        for kind, data, pos in chain(stream, [(None, None, None)]):
+            if kind is TEXT:
+                textbuf.append(data)
+            else:
+                if kind is START:
+                    preserve.append(data[0] in self.preserve or 
+                                    data[1].get('xml:space') == 'preserve')
+                if textbuf:
+                    if len(textbuf) > 1:
+                        text = mjoin(textbuf, escape_quotes=False)
+                        del textbuf[:]
+                    else:
+                        text = escape(textbuf.pop(), quotes=False)
+                    if not preserve[-1]:
+                        text = collapse_lines('\n', trim_trailing_space('', text))
+                    yield TEXT, Markup(text), pos
+                if kind is END:
+                    preserve.pop()
+                if kind is not None:
+                    yield kind, data, pos
+
+
 class _PushbackIterator(object):
     """A simple wrapper for iterators that allows pushing items back on the
     queue via the `pushback()` method.
diff --git a/markup/tests/output.py b/markup/tests/output.py
--- a/markup/tests/output.py
+++ b/markup/tests/output.py
@@ -16,7 +16,9 @@
 import sys
 
 from markup.core import Stream
-from markup.output import DocType, XMLSerializer
+from markup.input import HTML
+from markup.output import DocType, XMLSerializer, XHTMLSerializer, \
+                          HTMLSerializer
 
 
 class XMLSerializerTestCase(unittest.TestCase):
@@ -79,9 +81,33 @@
         self.assertEqual('<?python x = 2?>', output)
 
 
+class XHTMLSerializerTestCase(unittest.TestCase):
+
+    def test_textarea_whitespace(self):
+        content = '\nHey there.  \n\n    I am indented.\n'
+        stream = HTML('<textarea name="foo">%s</textarea>' % content)
+        output = stream.render(XHTMLSerializer)
+        self.assertEqual('<textarea name="foo">%s</textarea>' % content, output)
+
+    def test_xml_space(self):
+        text = '<foo xml:space="preserve"> Do not mess  \n\n with me </foo>'
+        output = HTML(text).render(XHTMLSerializer)
+        self.assertEqual(text, output)
+
+
+class HTMLSerializerTestCase(unittest.TestCase):
+
+    def test_xml_space(self):
+        text = '<foo xml:space="preserve"> Do not mess  \n\n with me </foo>'
+        output = HTML(text).render(HTMLSerializer)
+        self.assertEqual('<foo> Do not mess  \n\n with me </foo>', output)
+
+
 def suite():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(XMLSerializerTestCase, 'test'))
+    suite.addTest(unittest.makeSuite(XHTMLSerializerTestCase, 'test'))
+    suite.addTest(unittest.makeSuite(HTMLSerializerTestCase, 'test'))
     suite.addTest(doctest.DocTestSuite(XMLSerializer.__module__))
     return suite