# HG changeset patch
# User aronacher
# Date 1196359652 0
# Node ID 7aebde54c10f7d10201a44d67c24734cc353e334
# Parent  4ad264337a500e43b11eeaafe1b8f2be47f301c4
merged trunk into sandbox branch

diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -29,10 +29,21 @@
  * Text templates now default to rendering as plain text; it is no longer
    necessary to explicitly specify the "text" method to the `render()` or
    `serialize()` method of the generated markup stream.
- * XInclude elements in markup templates now support the `parse` attribute; when
-   set to "xml" (the default), the include is processed as before, but when set
-   to "text", the included template is parsed as a text template using the new
-   syntax (ticket #101).
+ * XInclude elements in markup templates now support the `parse` attribute;
+   when set to "xml" (the default), the include is processed as before, but
+   when set to "text", the included template is parsed as a text template using
+   the new syntax (ticket #101).
+ * Python code blocks inside match templates are now executed (ticket #155).
+ * The template engine plugin no longer adds the `default_doctype` when the
+   `fragment` parameter is `True`.
+ * The `striptags` function now also removes HTML/XML-style comments (ticket
+   #150).
+ * The `py:replace` directive can now also be used as an element, with an
+   attribute named `value` (ticket #144).
+ * The `TextSerializer` class no longer strips all markup in text by default,
+   so that it is still possible to use the Genshi `escape` function even with
+   text templates. The old behavior is available via the `strip_markup` option
+   of the serializer (ticket #146).
 
 
 Version 0.4.4
diff --git a/doc/templates.txt b/doc/templates.txt
--- a/doc/templates.txt
+++ b/doc/templates.txt
@@ -217,7 +217,7 @@
     <?python
         from genshi.builder import tag
         def greeting(name):
-            return tag.b('Hello, %s!' % name') ?>
+            return tag.b('Hello, %s!' % name) ?>
     ${greeting('world')}
   </div>
 
@@ -237,7 +237,7 @@
   {% python
       from genshi.builder import tag
       def greeting(name):
-          return tag.b('Hello, %s!' % name')
+          return 'Hello, %s!' % name
   %}
   ${greeting('world')}
 
diff --git a/doc/xml-templates.txt b/doc/xml-templates.txt
--- a/doc/xml-templates.txt
+++ b/doc/xml-templates.txt
@@ -499,7 +499,14 @@
     Bye
   </div>
 
-This directive can only be used as an attribute.
+This directive can also be used as an element (since version 0.5):
+
+.. code-block:: genshi
+
+  <div>
+    <py:replace value="title">Placeholder</py:replace>
+  </div>
+
 
 
 .. _`py:strip`:
diff --git a/examples/bench/basic.py b/examples/bench/basic.py
--- a/examples/bench/basic.py
+++ b/examples/bench/basic.py
@@ -122,7 +122,7 @@
     try:
         import kid
     except ImportError:
-        print>>sys.stderr, "SimpleTAL not installed, skipping"
+        print>>sys.stderr, "Kid not installed, skipping"
         return lambda: None
     kid.path = kid.TemplatePath([dirname])
     template = kid.load_template('template.kid').Template
diff --git a/genshi/output.py b/genshi/output.py
--- a/genshi/output.py
+++ b/genshi/output.py
@@ -436,20 +436,29 @@
     <Hello!>
 
     If text events contain literal markup (instances of the `Markup` class),
-    tags or entities are stripped from the output:
+    that markup is by default passed through unchanged:
     
-    >>> elem = tag.div(Markup('<a href="foo">Hello!</a><br/>'))
-    >>> print elem
-    <div><a href="foo">Hello!</a><br/></div>
-    >>> print ''.join(TextSerializer()(elem.generate()))
-    Hello!
+    >>> elem = tag.div(Markup('<a href="foo">Hello &amp; Bye!</a><br/>'))
+    >>> print elem.generate().render(TextSerializer)
+    <a href="foo">Hello &amp; Bye!</a><br/>
+    
+    You can use the `strip_markup` to change this behavior, so that tags and
+    entities are stripped from the output (or in the case of entities,
+    replaced with the equivalent character):
+
+    >>> print elem.generate().render(TextSerializer, strip_markup=True)
+    Hello & Bye!
     """
 
+    def __init__(self, strip_markup=False):
+        self.strip_markup = strip_markup
+
     def __call__(self, stream):
+        strip_markup = self.strip_markup
         for event in stream:
             if event[0] is TEXT:
                 data = event[1]
-                if type(data) is Markup:
+                if strip_markup and type(data) is Markup:
                     data = data.striptags().stripentities()
                 yield unicode(data)
 
diff --git a/genshi/template/directives.py b/genshi/template/directives.py
--- a/genshi/template/directives.py
+++ b/genshi/template/directives.py
@@ -208,6 +208,10 @@
     __slots__ = []
 
     def attach(cls, template, stream, value, namespaces, pos):
+        if type(value) is dict:
+            raise TemplateSyntaxError('The content directive can not be used '
+                                      'as an element', template.filepath,
+                                      *pos[1:])
         expr = cls._parse_expr(value, template, *pos[1:])
         return None, [stream[0], (EXPR, expr, pos),  stream[-1]]
     attach = classmethod(attach)
@@ -485,6 +489,8 @@
     __slots__ = []
 
     def attach(cls, template, stream, value, namespaces, pos):
+        if type(value) is dict:
+            value = value.get('value')
         if not value:
             raise TemplateSyntaxError('missing value for "replace" directive',
                                       template.filepath, *pos[1:])
diff --git a/genshi/template/eval.py b/genshi/template/eval.py
--- a/genshi/template/eval.py
+++ b/genshi/template/eval.py
@@ -82,7 +82,7 @@
             if restricted:
                 lookup = RestrictedLookupWrapper(lookup)
         self.restricted = restricted
-        self._globals = lookup.globals()
+        self._globals = lookup.globals
 
     def __eq__(self, other):
         return (type(other) == type(self)) and (self.code == other.code)
@@ -148,7 +148,7 @@
         :return: the result of the evaluation
         """
         __traceback_hide__ = 'before_and_this'
-        _globals = self._globals
+        _globals = self._globals()
         _globals['data'] = data
         return eval(self.code, _globals, {'data': data})
 
@@ -170,7 +170,7 @@
         :param data: a mapping containing the data to execute in
         """
         __traceback_hide__ = 'before_and_this'
-        _globals = self._globals
+        _globals = self._globals()
         _globals['data'] = data
         exec self.code in _globals, data
 
diff --git a/genshi/template/markup.py b/genshi/template/markup.py
--- a/genshi/template/markup.py
+++ b/genshi/template/markup.py
@@ -295,9 +295,9 @@
                     remaining = match_templates
                     if 'match_once' not in hints:
                         remaining = remaining[:idx] + remaining[idx + 1:]
-                    for event in self._match(self._eval(self._flatten(template,
-                                                                      ctxt),
-                                                        ctxt), ctxt, remaining):
+                    for event in self._match(self._exec(
+                                    self._eval(self._flatten(template, ctxt),
+                                    ctxt), ctxt), ctxt, remaining):
                         yield event
 
                     ctxt.pop()
diff --git a/genshi/template/plugin.py b/genshi/template/plugin.py
--- a/genshi/template/plugin.py
+++ b/genshi/template/plugin.py
@@ -97,7 +97,7 @@
 
         return self.loader.load(templatename)
 
-    def _get_render_options(self, format=None):
+    def _get_render_options(self, format=None, fragment=False):
         if format is None:
             format = self.default_format
         kwargs = {'method': format}
@@ -107,7 +107,7 @@
 
     def render(self, info, format=None, fragment=False, template=None):
         """Render the template to a string using the provided info."""
-        kwargs = self._get_render_options(format=format)
+        kwargs = self._get_render_options(format=format, fragment=fragment)
         return self.transform(info, template).render(**kwargs)
 
     def transform(self, info, template):
@@ -140,10 +140,10 @@
             raise ConfigurationError('Unknown output format %r' % format)
         self.default_format = format
 
-    def _get_render_options(self, format=None):
+    def _get_render_options(self, format=None, fragment=False):
         kwargs = super(MarkupTemplateEnginePlugin,
-                       self)._get_render_options(format)
-        if self.default_doctype:
+                       self)._get_render_options(format, fragment)
+        if self.default_doctype and not fragment:
             kwargs['doctype'] = self.default_doctype
         return kwargs
 
diff --git a/genshi/template/tests/directives.py b/genshi/template/tests/directives.py
--- a/genshi/template/tests/directives.py
+++ b/genshi/template/tests/directives.py
@@ -484,7 +484,8 @@
             list(tmpl.generate(foo=12))
             self.fail('Expected TemplateRuntimeError')
         except TypeError, e:
-            self.assertEqual('iteration over non-sequence', str(e))
+            assert (str(e) == "iteration over non-sequence" or
+                    str(e) == "'int' object is not iterable")
             exc_type, exc_value, exc_traceback = sys.exc_info()
             frame = exc_traceback.tb_next
             frames = []
@@ -915,6 +916,21 @@
     #    </div>""", str(tmpl.generate()))
 
 
+class ContentDirectiveTestCase(unittest.TestCase):
+    """Tests for the `py:content` template directive."""
+
+    def test_as_element(self):
+        try:
+            tmpl = MarkupTemplate("""<doc xmlns:py="http://genshi.edgewall.org/">
+              <py:content foo="">Foo</py:content>
+            </doc>""", filename='test.html')
+            self.fail('Expected TemplateSyntaxError')
+        except TemplateSyntaxError, e:
+            self.assertEqual('test.html', e.filename)
+            if sys.version_info[:2] >= (2, 4):
+                self.assertEqual(2, e.lineno)
+
+
 class ReplaceDirectiveTestCase(unittest.TestCase):
     """Tests for the `py:replace` template directive."""
 
@@ -933,6 +949,14 @@
             if sys.version_info[:2] >= (2, 4):
                 self.assertEqual(2, e.lineno)
 
+    def test_as_element(self):
+        tmpl = MarkupTemplate("""<div xmlns:py="http://genshi.edgewall.org/">
+          <py:replace value="title" />
+        </div>""", filename='test.html')
+        self.assertEqual("""<div>
+          Test
+        </div>""", str(tmpl.generate(title='Test')))
+
 
 class StripDirectiveTestCase(unittest.TestCase):
     """Tests for the `py:strip` template directive."""
@@ -1064,6 +1088,7 @@
     suite.addTest(unittest.makeSuite(ForDirectiveTestCase, 'test'))
     suite.addTest(unittest.makeSuite(IfDirectiveTestCase, 'test'))
     suite.addTest(unittest.makeSuite(MatchDirectiveTestCase, 'test'))
+    suite.addTest(unittest.makeSuite(ContentDirectiveTestCase, 'test'))
     suite.addTest(unittest.makeSuite(ReplaceDirectiveTestCase, 'test'))
     suite.addTest(unittest.makeSuite(StripDirectiveTestCase, 'test'))
     suite.addTest(unittest.makeSuite(WithDirectiveTestCase, 'test'))
diff --git a/genshi/template/tests/markup.py b/genshi/template/tests/markup.py
--- a/genshi/template/tests/markup.py
+++ b/genshi/template/tests/markup.py
@@ -270,7 +270,7 @@
         finally:
             shutil.rmtree(dirname)
 
-    def test_dynamic_inlude_href(self):
+    def test_dynamic_include_href(self):
         dirname = tempfile.mkdtemp(suffix='genshi_test')
         try:
             file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
@@ -296,7 +296,7 @@
         finally:
             shutil.rmtree(dirname)
 
-    def test_select_inluded_elements(self):
+    def test_select_included_elements(self):
         dirname = tempfile.mkdtemp(suffix='genshi_test')
         try:
             file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
@@ -596,6 +596,75 @@
         </html>""")
         tmpl = MarkupTemplate(xml, filename='test.html', allow_exec=True)
 
+    def test_exec_in_match(self): 
+        xml = ("""<html xmlns:py="http://genshi.edgewall.org/">
+          <py:match path="body/p">
+            <?python title="wakka wakka wakka" ?>
+            ${title}
+          </py:match>
+          <body><p>moot text</p></body>
+        </html>""")
+        tmpl = MarkupTemplate(xml, filename='test.html', allow_exec=True)
+        self.assertEqual("""<html>
+          <body>
+            wakka wakka wakka
+          </body>
+        </html>""", tmpl.generate().render())
+
+    def test_nested_include_matches(self):
+        # See ticket #157
+        dirname = tempfile.mkdtemp(suffix='genshi_test')
+        try:
+            file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w')
+            try:
+                file1.write("""<html xmlns:py="http://genshi.edgewall.org/" py:strip="">
+   <div class="target">Some content.</div>
+</html>""")
+            finally:
+                file1.close()
+
+            file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w')
+            try:
+                file2.write("""<html xmlns:py="http://genshi.edgewall.org/"
+    xmlns:xi="http://www.w3.org/2001/XInclude">
+  <body>
+    <h1>Some full html document that includes file1.html</h1>
+    <xi:include href="tmpl1.html" />
+  </body>
+</html>""")
+            finally:
+                file2.close()
+
+            file3 = open(os.path.join(dirname, 'tmpl3.html'), 'w')
+            try:
+                file3.write("""<html xmlns:py="http://genshi.edgewall.org/"
+    xmlns:xi="http://www.w3.org/2001/XInclude" py:strip="">
+  <div py:match="div[@class='target']" py:attrs="select('@*')">
+    Some added stuff.
+    ${select('*|text()')}
+  </div>
+  <xi:include href="tmpl2.html" />
+</html>
+""")
+            finally:
+                file3.close()
+
+            loader = TemplateLoader([dirname])
+            tmpl = loader.load('tmpl3.html')
+            self.assertEqual("""
+  <html>
+  <body>
+    <h1>Some full html document that includes file1.html</h1>
+   <div class="target">
+    Some added stuff.
+    Some content.
+  </div>
+  </body>
+</html>
+""", tmpl.generate().render())
+        finally:
+            shutil.rmtree(dirname)
+
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/genshi/template/tests/plugin.py b/genshi/template/tests/plugin.py
--- a/genshi/template/tests/plugin.py
+++ b/genshi/template/tests/plugin.py
@@ -145,6 +145,23 @@
   </body>
 </html>""", output)
 
+    def test_render_fragment_with_doctype(self):
+        plugin = MarkupTemplateEnginePlugin(options={
+            'genshi.default_doctype': 'html-strict',
+        })
+        tmpl = plugin.load_template(PACKAGE + '.templates.test_no_doctype')
+        output = plugin.render({'message': 'Hello'}, template=tmpl,
+                               fragment=True)
+        self.assertEqual("""<html lang="en">
+  <head>
+    <title>Test</title>
+  </head>
+  <body>
+    <h1>Test</h1>
+    <p>Hello</p>
+  </body>
+</html>""", output)
+
     def test_helper_functions(self):
         plugin = MarkupTemplateEnginePlugin()
         tmpl = plugin.load_template(PACKAGE + '.templates.functions')
diff --git a/genshi/template/tests/templates/test_no_doctype.html b/genshi/template/tests/templates/test_no_doctype.html
new file mode 100644
--- /dev/null
+++ b/genshi/template/tests/templates/test_no_doctype.html
@@ -0,0 +1,13 @@
+<html xmlns="http://www.w3.org/1999/xhtml"
+      lang="en">
+
+  <head>
+    <title>Test</title>
+  </head>
+
+  <body>
+    <h1>Test</h1>
+    <p>$message</p>
+  </body>
+
+</html>
diff --git a/genshi/util.py b/genshi/util.py
--- a/genshi/util.py
+++ b/genshi/util.py
@@ -228,7 +228,7 @@
                     return ref
     return _STRIPENTITIES_RE.sub(_replace_entity, text)
 
-_STRIPTAGS_RE = re.compile(r'<[^>]*?>')
+_STRIPTAGS_RE = re.compile(r'(<!--.*?-->|<[^>]*>)')
 def striptags(text):
     """Return a copy of the text with any XML/HTML tags removed.
     
@@ -239,6 +239,11 @@
     >>> striptags('Foo<br />')
     'Foo'
     
+    HTML/XML comments are stripped, too:
+    
+    >>> striptags('<!-- <blub>hehe</blah> -->test')
+    'test'
+    
     :param text: the string to remove tags from
     :return: the text with tags removed
     """