# HG changeset patch
# User cmlenz
# Date 1169131962 0
# Node ID f0b785d3d407e07637740e515350c20ca994998a
# Parent  31742fe6d47e78feefe5d44299ba6f5d963ae0fa
Rework parsing of expressions in template text, to be able to:
 * handle dict literals (as well as strings containing the character ?}?) inside the expression (#37), and
 * allow escaped dollar signs in front of full expressions (#92)

diff --git a/ChangeLog b/ChangeLog
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,7 +30,10 @@
  * `MarkupTemplate`s can now be instantiated from markup streams, in addition
    to strings and file-like objects (ticket #69).
  * Improve handling of incorrectly nested tags in the HTML parser.
- * Template includes can you be nested inside fallback content.
+ * Template includes can now be nested inside fallback content.
+ * Expressions can now contain dict literals (ticket #37).
+ * It is now possible to have one or more escaped dollar signs in front of a 
+   full expression (ticket #92).
 
 
 Version 0.3.6
diff --git a/genshi/template/core.py b/genshi/template/core.py
--- a/genshi/template/core.py
+++ b/genshi/template/core.py
@@ -17,9 +17,11 @@
     class deque(list):
         def appendleft(self, x): self.insert(0, x)
         def popleft(self): return self.pop(0)
+from itertools import chain
 import os
 import re
 from StringIO import StringIO
+from tokenize import tokenprog
 
 from genshi.core import Attrs, Stream, StreamEventKind, START, TEXT, _ensure
 from genshi.template.eval import Expression
@@ -198,9 +200,6 @@
         """
         raise NotImplementedError
 
-    _FULL_EXPR_RE = re.compile(r'(?<!\$)\$\{(.+?)\}', re.DOTALL)
-    _SHORT_EXPR_RE = re.compile(r'(?<!\$)\$([a-zA-Z_][a-zA-Z0-9_\.]*)')
-
     def _interpolate(cls, text, basedir=None, filename=None, lineno=-1,
                      offset=0):
         """Parse the given string and extract expressions.
@@ -216,29 +215,94 @@
         filepath = filename
         if filepath and basedir:
             filepath = os.path.join(basedir, filepath)
-        def _interpolate(text, patterns, lineno=lineno, offset=offset):
-            for idx, grp in enumerate(patterns.pop(0).split(text)):
-                if idx % 2:
+
+        namestart = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'
+        namechars = namestart + '.0123456789'
+
+        def _split():
+            pos = 0
+            end = len(text)
+            escaped = False
+
+            while 1:
+                if escaped:
+                    offset = text.find('$', offset + 2)
+                    escaped = False
+                else:
+                    offset = text.find('$', pos)
+                if offset < 0 or offset == end - 1:
+                    break
+                next = text[offset + 1]
+
+                if next == '{':
+                    if offset > pos:
+                        yield False, text[pos:offset]
+                    pos = offset + 2
+                    level = 1
+                    while level:
+                        match = tokenprog.match(text, pos)
+                        if match is None:
+                            raise TemplateSyntaxError('invalid syntax',
+                                                      filename, lineno, offset)
+                        pos = match.end()
+                        tstart, tend = match.regs[3]
+                        token = text[tstart:tend]
+                        if token == '{':
+                            level += 1
+                        elif token == '}':
+                            level -= 1
+                    yield True, text[offset + 2:pos - 1]
+
+                elif next in namestart:
+                    if offset > pos:
+                        yield False, text[pos:offset]
+                        pos = offset
+                    pos += 1
+                    while pos < end:
+                        char = text[pos]
+                        if char not in namechars:
+                            break
+                        pos += 1
+                    yield True, text[offset + 1:pos].strip()
+
+                elif not escaped and next == '$':
+                    escaped = True
+                    pos = offset + 1
+
+                else:
+                    yield False, text[pos:offset + 1]
+                    pos = offset + 1
+
+            if pos < end:
+                yield False, text[pos:]
+
+        textbuf = []
+        textpos = None
+        for is_expr, chunk in chain(_split(), [(True, '')]):
+            if is_expr:
+                if textbuf:
+                    yield TEXT, u''.join(textbuf), textpos
+                    del textbuf[:]
+                    textpos = None
+                if chunk:
                     try:
-                        yield EXPR, Expression(grp.strip(), filepath, lineno), \
-                              (filename, lineno, offset)
+                        expr = Expression(chunk.strip(), filename, lineno)
+                        yield EXPR, expr, (filename, lineno, offset)
                     except SyntaxError, err:
-                        raise TemplateSyntaxError(err, filepath, lineno,
+                        raise TemplateSyntaxError(err, filename, lineno,
                                                   offset + (err.offset or 0))
-                elif grp:
-                    if patterns:
-                        for result in _interpolate(grp, patterns[:]):
-                            yield result
-                    else:
-                        yield TEXT, grp.replace('$$', '$'), \
-                              (filename, lineno, offset)
-                if '\n' in grp:
-                    lines = grp.splitlines()
-                    lineno += len(lines) - 1
-                    offset += len(lines[-1])
-                else:
-                    offset += len(grp)
-        return _interpolate(text, [cls._FULL_EXPR_RE, cls._SHORT_EXPR_RE])
+            else:
+                textbuf.append(chunk)
+                if textpos is None:
+                    textpos = (filename, lineno, offset)
+
+            if '\n' in chunk:
+                lines = chunk.splitlines()
+                lineno += len(lines) - 1
+                offset += len(lines[-1])
+            else:
+                offset += len(chunk)
+
     _interpolate = classmethod(_interpolate)
 
     def _prepare(self, stream):
diff --git a/genshi/template/tests/core.py b/genshi/template/tests/core.py
--- a/genshi/template/tests/core.py
+++ b/genshi/template/tests/core.py
@@ -15,7 +15,7 @@
 import unittest
 
 from genshi.core import Stream
-from genshi.template.core import Template
+from genshi.template.core import Template, TemplateSyntaxError
 
 
 class TemplateTestCase(unittest.TestCase):
@@ -41,12 +41,34 @@
         self.assertEqual(Stream.TEXT, parts[0][0])
         self.assertEqual('${bla}', parts[0][1])
 
+    def test_interpolate_dobuleescaped(self):
+        parts = list(Template._interpolate('$$${bla}'))
+        self.assertEqual(2, len(parts))
+        self.assertEqual(Stream.TEXT, parts[0][0])
+        self.assertEqual('$', parts[0][1])
+        self.assertEqual(Template.EXPR, parts[1][0])
+        self.assertEqual('bla', parts[1][1].source)
+
     def test_interpolate_short(self):
         parts = list(Template._interpolate('$bla'))
         self.assertEqual(1, len(parts))
         self.assertEqual(Template.EXPR, parts[0][0])
         self.assertEqual('bla', parts[0][1].source)
 
+    def test_interpolate_short_escaped(self):
+        parts = list(Template._interpolate('$$bla'))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Stream.TEXT, parts[0][0])
+        self.assertEqual('$bla', parts[0][1])
+
+    def test_interpolate_short_doubleescaped(self):
+        parts = list(Template._interpolate('$$$bla'))
+        self.assertEqual(2, len(parts))
+        self.assertEqual(Stream.TEXT, parts[0][0])
+        self.assertEqual('$', parts[0][1])
+        self.assertEqual(Template.EXPR, parts[1][0])
+        self.assertEqual('bla', parts[1][1].source)
+
     def test_interpolate_short_starting_with_underscore(self):
         parts = list(Template._interpolate('$_bla'))
         self.assertEqual(1, len(parts))
@@ -83,6 +105,62 @@
         self.assertEqual(Template.EXPR, parts[0][0])
         self.assertEqual('foo0', parts[0][1].source)
 
+    def test_interpolate_short_starting_with_digit(self):
+        parts = list(Template._interpolate('$0bla'))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Stream.TEXT, parts[0][0])
+        self.assertEqual('$0bla', parts[0][1])
+
+    def test_interpolate_short_containing_digit(self):
+        parts = list(Template._interpolate('$foo0'))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual('foo0', parts[0][1].source)
+
+    def test_interpolate_full_nested_brackets(self):
+        parts = list(Template._interpolate('${{1:2}}'))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual('{1:2}', parts[0][1].source)
+
+    def test_interpolate_full_mismatched_brackets(self):
+        try:
+            list(Template._interpolate('${{1:2}'))
+        except TemplateSyntaxError, e:
+            pass
+        else:
+            self.fail('Expected TemplateSyntaxError')
+
+    def test_interpolate_quoted_brackets_1(self):
+        parts = list(Template._interpolate('${"}"}'))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual('"}"', parts[0][1].source)
+
+    def test_interpolate_quoted_brackets_2(self):
+        parts = list(Template._interpolate("${'}'}"))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual("'}'", parts[0][1].source)
+
+    def test_interpolate_quoted_brackets_3(self):
+        parts = list(Template._interpolate("${'''}'''}"))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual("'''}'''", parts[0][1].source)
+
+    def test_interpolate_quoted_brackets_4(self):
+        parts = list(Template._interpolate("${'''}\"\"\"'''}"))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual("'''}\"\"\"'''", parts[0][1].source)
+
+    def test_interpolate_quoted_brackets_5(self):
+        parts = list(Template._interpolate(r"${'\'}'}"))
+        self.assertEqual(1, len(parts))
+        self.assertEqual(Template.EXPR, parts[0][0])
+        self.assertEqual(r"'\'}'", parts[0][1].source)
+
     def test_interpolate_mixed1(self):
         parts = list(Template._interpolate('$foo bar $baz'))
         self.assertEqual(3, len(parts))