# HG changeset patch
# User cmlenz
# Date 1184344307 0
# Node ID aa8e85a4085eb2c27451c72fb8bee6f66b9adc25
# Parent  b3b07279b86c10e3fe8ee0e2335dfdfa4e33fa8b
 * The I18n extractor now handles gettext function calls that use non-string parameters as well as keyword arguments.
 * Expression and Suite objects now retain the AST information so that it can be used by the I18n message extraction.
 * The AST transformation no longer modifies the AST in place, but instead returns a transformed copy of the AST node.

diff --git a/genshi/filters/i18n.py b/genshi/filters/i18n.py
--- a/genshi/filters/i18n.py
+++ b/genshi/filters/i18n.py
@@ -13,12 +13,12 @@
 
 """Utilities for internationalization and localization of templates."""
 
+from compiler import ast
 try:
     frozenset
 except NameError:
     from sets import ImmutableSet as frozenset
 from gettext import gettext
-from opcode import opmap
 import re
 
 from genshi.core import Attrs, Namespace, QName, START, END, TEXT, START_NS, \
@@ -29,11 +29,6 @@
 __all__ = ['Translator', 'extract']
 __docformat__ = 'restructuredtext en'
 
-_LOAD_NAME = chr(opmap['LOAD_NAME'])
-_LOAD_CONST = chr(opmap['LOAD_CONST'])
-_CALL_FUNCTION = chr(opmap['CALL_FUNCTION'])
-_BINARY_ADD = chr(opmap['BINARY_ADD'])
-
 I18N_NAMESPACE = Namespace('http://genshi.edgewall.org/i18n')
 
 
@@ -246,7 +241,7 @@
         3, None, u'Example'
         6, None, u'Example'
         7, '_', u'Hello, %(name)s'
-        8, 'ngettext', (u'You have %d item', u'You have %d items')
+        8, 'ngettext', (u'You have %d item', u'You have %d items', None)
         
         :param stream: the event stream to extract strings from; can be a
                        regular stream or a template stream
@@ -312,7 +307,7 @@
                     msgbuf = None
 
             elif kind is EXPR or kind is EXEC:
-                for funcname, strings in extract_from_code(data.code,
+                for funcname, strings in extract_from_code(data,
                                                            gettext_functions):
                     yield pos[1], funcname, strings
 
@@ -379,43 +374,38 @@
     >>> from genshi.template.eval import Expression
     
     >>> expr = Expression('_("Hello")')
-    >>> list(extract_from_code(expr.code, Translator.GETTEXT_FUNCTIONS))
+    >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS))
     [('_', u'Hello')]
 
     >>> expr = Expression('ngettext("You have %(num)s item", '
     ...                            '"You have %(num)s items", num)')
-    >>> list(extract_from_code(expr.code, Translator.GETTEXT_FUNCTIONS))
-    [('ngettext', (u'You have %(num)s item', u'You have %(num)s items'))]
+    >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS))
+    [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None))]
     
-    :param code: the code object
+    :param code: the `Code` object
+    :type code: `genshi.template.eval.Code`
     :param gettext_functions: a sequence of function names
     """
-    consts = dict([(n, chr(i) + '\x00') for i, n in enumerate(code.co_consts)])
-    gettext_locs = [consts[n] for n in gettext_functions if n in consts]
-    ops = [
-        _LOAD_CONST, '(', '|'.join(gettext_locs), ')',
-        _CALL_FUNCTION, '.\x00',
-        '((?:', _BINARY_ADD, '|', _LOAD_CONST, '.\x00)+)'
-    ]
-    for loc, opcodes in re.findall(''.join(ops), code.co_code):
-        funcname = code.co_consts[ord(loc[0])]
-        strings = []
-        opcodes = iter(opcodes)
-        for opcode in opcodes:
-            if opcode == _BINARY_ADD:
-                arg = strings.pop()
-                strings[-1] += arg
+    def _walk(node):
+        if isinstance(node, ast.CallFunc) and isinstance(node.node, ast.Name) \
+                and node.node.name in gettext_functions:
+            strings = []
+            for arg in node.args:
+                if isinstance(arg, ast.Const) \
+                        and isinstance(arg.value, basestring):
+                    strings.append(unicode(arg.value))
+                elif not isinstance(arg, ast.Keyword):
+                    strings.append(None)
+            if len(strings) == 1:
+                strings = strings[0]
             else:
-                arg = code.co_consts[ord(opcodes.next())]
-                opcodes.next() # skip second byte
-                if not isinstance(arg, basestring):
-                    break
-                strings.append(unicode(arg))
-        if len(strings) == 1:
-            strings = strings[0]
+                strings = tuple(strings)
+            yield node.node.name, strings
         else:
-            strings = tuple(strings)
-        yield funcname, strings
+            for child in node.getChildNodes():
+                for funcname, strings in _walk(child):
+                    yield funcname, strings
+    return _walk(code.ast)
 
 def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|\]')):
     """Parse a message using Genshi compound message formatting.
diff --git a/genshi/filters/tests/i18n.py b/genshi/filters/tests/i18n.py
--- a/genshi/filters/tests/i18n.py
+++ b/genshi/filters/tests/i18n.py
@@ -28,7 +28,8 @@
         translator = Translator()
         messages = list(translator.extract(tmpl.stream))
         self.assertEqual(1, len(messages))
-        self.assertEqual((2, 'ngettext', (u'Singular', u'Plural')), messages[0])
+        self.assertEqual((2, 'ngettext', (u'Singular', u'Plural', None)),
+                         messages[0])
 
     def test_extract_included_attribute_text(self):
         tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/">
@@ -263,7 +264,8 @@
             (3, None, u'Example', []),
             (6, None, u'Example', []),
             (7, '_', u'Hello, %(name)s', []),
-            (8, 'ngettext', (u'You have %d item', u'You have %d items'), []),
+            (8, 'ngettext', (u'You have %d item', u'You have %d items', None),
+                             []),
         ], results)
 
     def test_text_template_extraction(self):
@@ -281,10 +283,28 @@
         }))
         self.assertEqual([
             (1, '_', u'Dear %(name)s', []),
-            (3, 'ngettext', (u'Your item:', u'Your items'), []),
+            (3, 'ngettext', (u'Your item:', u'Your items', None), []),
             (7, None, u'All the best,\n        Foobar', [])
         ], results)
 
+    def test_extraction_with_keyword_arg(self):
+        buf = StringIO("""<html xmlns:py="http://genshi.edgewall.org/">
+          ${gettext('Foobar', foo='bar')}
+        </html>""")
+        results = list(extract(buf, ['gettext'], [], {}))
+        self.assertEqual([
+            (2, 'gettext', (u'Foobar'), []),
+        ], results)
+
+    def test_extraction_with_nonstring_arg(self):
+        buf = StringIO("""<html xmlns:py="http://genshi.edgewall.org/">
+          ${dgettext(curdomain, 'Foobar')}
+        </html>""")
+        results = list(extract(buf, ['dgettext'], [], {}))
+        self.assertEqual([
+            (2, 'dgettext', (None, u'Foobar'), []),
+        ], results)
+
     def test_extraction_inside_ignored_tags(self):
         buf = StringIO("""<html xmlns:py="http://genshi.edgewall.org/">
           <script type="text/javascript">
diff --git a/genshi/template/eval.py b/genshi/template/eval.py
--- a/genshi/template/eval.py
+++ b/genshi/template/eval.py
@@ -20,6 +20,7 @@
 try:
     set
 except NameError:
+    from sets import ImmutableSet as frozenset
     from sets import Set as set
 import sys
 
@@ -34,7 +35,7 @@
 
 class Code(object):
     """Abstract base class for the `Expression` and `Suite` classes."""
-    __slots__ = ['source', 'code', '_globals']
+    __slots__ = ['source', 'code', 'ast', '_globals']
 
     def __init__(self, source, filename=None, lineno=-1, lookup='lenient'):
         """Create the code object, either from a string, or from an AST node.
@@ -59,6 +60,7 @@
             else:
                 node = ast.Module(None, source)
 
+        self.ast = node
         self.code = _compile(node, self.source, mode=self.mode,
                              filename=filename, lineno=lineno)
         if lookup is None:
@@ -409,244 +411,223 @@
                           self._visitDefault)
         return visitor(node)
 
+    def _clone(self, node, *args):
+        lineno = getattr(node, 'lineno', None)
+        node = node.__class__(*args)
+        if lineno is not None:
+            node.lineno = lineno
+        if isinstance(node, (ast.Class, ast.Function, ast.GenExpr, ast.Lambda)):
+            node.filename = '<string>' # workaround for bug in pycodegen
+        return node
+
     def _visitDefault(self, node):
         return node
 
     def visitExpression(self, node):
-        node.node = self.visit(node.node)
-        return node
+        return self._clone(node, self.visit(node.node))
 
     def visitModule(self, node):
-        node.node = self.visit(node.node)
-        return node
+        return self._clone(node, node.doc, self.visit(node.node))
 
     def visitStmt(self, node):
-        node.nodes = [self.visit(x) for x in node.nodes]
-        return node
+        return self._clone(node, [self.visit(x) for x in node.nodes])
 
     # Classes, Functions & Accessors
 
     def visitCallFunc(self, node):
-        node.node = self.visit(node.node)
-        node.args = [self.visit(x) for x in node.args]
-        if node.star_args:
-            node.star_args = self.visit(node.star_args)
-        if node.dstar_args:
-            node.dstar_args = self.visit(node.dstar_args)
-        return node
+        return self._clone(node, self.visit(node.node),
+            [self.visit(x) for x in node.args],
+            node.star_args and self.visit(node.star_args) or None,
+            node.dstar_args and self.visit(node.dstar_args) or None
+        )
 
     def visitClass(self, node):
-        node.bases = [self.visit(x) for x in node.bases]
-        node.code = self.visit(node.code)
-        node.filename = '<string>' # workaround for bug in pycodegen
-        return node
+        return self._clone(node, node.name, [self.visit(x) for x in node.bases],
+            node.doc, node.code
+        )
 
     def visitFunction(self, node):
+        args = []
         if hasattr(node, 'decorators'):
-            node.decorators = self.visit(node.decorators)
-        node.defaults = [self.visit(x) for x in node.defaults]
-        node.code = self.visit(node.code)
-        node.filename = '<string>' # workaround for bug in pycodegen
-        return node
+            args.append(self.visit(node.decorators))
+        return self._clone(node, *args + [
+            node.name,
+            node.argnames,
+            [self.visit(x) for x in node.defaults],
+            node.flags,
+            node.doc,
+            self.visit(node.code)
+        ])
 
     def visitGetattr(self, node):
-        node.expr = self.visit(node.expr)
-        return node
+        return self._clone(node, self.visit(node.expr), node.attrname)
 
     def visitLambda(self, node):
-        node.code = self.visit(node.code)
-        node.filename = '<string>' # workaround for bug in pycodegen
+        node = self._clone(node, node.argnames,
+            [self.visit(x) for x in node.defaults], node.flags,
+            self.visit(node.code)
+        )
         return node
 
     def visitSubscript(self, node):
-        node.expr = self.visit(node.expr)
-        node.subs = [self.visit(x) for x in node.subs]
-        return node
+        return self._clone(node, self.visit(node.expr), node.flags,
+            [self.visit(x) for x in node.subs]
+        )
 
     # Statements
 
     def visitAssert(self, node):
-        node.test = self.visit(node.test)
-        node.fail = self.visit(node.fail)
-        return node
+        return self._clone(node, self.visit(node.test), self.visit(node.fail))
 
     def visitAssign(self, node):
-        node.nodes = [self.visit(x) for x in node.nodes]
-        node.expr = self.visit(node.expr)
-        return node
+        return self._clone(node, [self.visit(x) for x in node.nodes],
+            self.visit(node.expr)
+        )
 
     def visitAssAttr(self, node):
-        node.expr = self.visit(node.expr)
-        return node
+        return self._clone(node, self.visit(node.expr), node.attrname,
+            node.flags
+        )
 
     def visitAugAssign(self, node):
-        node.node = self.visit(node.node)
-        node.expr = self.visit(node.expr)
-        return node
+        return self._clone(node, self.visit(node.node), node.op,
+            self.visit(node.expr)
+        )
 
     def visitDecorators(self, node):
-        node.nodes = [self.visit(x) for x in node.nodes]
-        return node
+        return self._clone(node, [self.visit(x) for x in node.nodes])
 
     def visitExec(self, node):
-        node.expr = self.visit(node.expr)
-        node.locals = self.visit(node.locals)
-        node.globals = self.visit(node.globals)
-        return node
+        return self._clone(node, self.visit(node.expr), self.visit(node.locals),
+            self.visit(node.globals)
+        )
 
     def visitFor(self, node):
-        node.assign = self.visit(node.assign)
-        node.list = self.visit(node.list)
-        node.body = self.visit(node.body)
-        node.else_ = self.visit(node.else_)
-        return node
+        return self._clone(node, self.visit(node.assign), self.visit(node.list),
+            self.visit(node.body), self.visit(node.else_)
+        )
 
     def visitIf(self, node):
-        node.tests = [self.visit(x) for x in node.tests]
-        node.else_ = self.visit(node.else_)
-        return node
+        return self._clone(node, [self.visit(x) for x in node.tests],
+            self.visit(node.else_)
+        )
 
     def _visitPrint(self, node):
-        node.nodes = [self.visit(x) for x in node.nodes]
-        node.dest = self.visit(node.dest)
-        return node
+        return self._clone(node, [self.visit(x) for x in node.nodes],
+            self.visit(node.dest)
+        )
     visitPrint = visitPrintnl = _visitPrint
 
     def visitRaise(self, node):
-        node.expr1 = self.visit(node.expr1)
-        node.expr2 = self.visit(node.expr2)
-        node.expr3 = self.visit(node.expr3)
-        return node
+        return self._clone(node, self.visit(node.expr1), self.visit(node.expr2),
+            self.visit(node.expr3)
+        )
 
     def visitReturn(self, node):
-        node.value = self.visit(node.value)
-        return node
+        return self._clone(node, self.visit(node.value))
 
     def visitTryExcept(self, node):
-        node.body = self.visit(node.body)
-        node.handlers = self.visit(node.handlers)
-        node.else_ = self.visit(node.else_)
-        return node
+        return self._clone(node, self.visit(node.body), self.visit(node.handlers),
+            self.visit(node.else_)
+        )
 
     def visitTryFinally(self, node):
-        node.body = self.visit(node.body)
-        node.final = self.visit(node.final)
-        return node
+        return self._clone(node, self.visit(node.body), self.visit(node.final))
 
     def visitWhile(self, node):
-        node.test = self.visit(node.test)
-        node.body = self.visit(node.body)
-        node.else_ = self.visit(node.else_)
-        return node
+        return self._clone(node, self.visit(node.test), self.visit(node.body),
+            self.visit(node.else_)
+        )
 
     def visitWith(self, node):
-        node.expr = self.visit(node.expr)
-        node.vars = [self.visit(x) for x in node.vars]
-        node.body = self.visit(node.body)
-        return node
+        return self._clone(node, self.visit(node.expr),
+            [self.visit(x) for x in node.vars], self.visit(node.body)
+        )
 
     def visitYield(self, node):
-        node.value = self.visit(node.value)
-        return node
+        return self._clone(node, self.visit(node.value))
 
     # Operators
 
     def _visitBoolOp(self, node):
-        node.nodes = [self.visit(x) for x in node.nodes]
-        return node
+        return self._clone(node, [self.visit(x) for x in node.nodes])
     visitAnd = visitOr = visitBitand = visitBitor = visitBitxor = _visitBoolOp
     visitAssTuple = visitAssList = _visitBoolOp
 
     def _visitBinOp(self, node):
-        node.left = self.visit(node.left)
-        node.right = self.visit(node.right)
-        return node
+        return self._clone(node,
+            (self.visit(node.left), self.visit(node.right))
+        )
     visitAdd = visitSub = _visitBinOp
     visitDiv = visitFloorDiv = visitMod = visitMul = visitPower = _visitBinOp
     visitLeftShift = visitRightShift = _visitBinOp
 
     def visitCompare(self, node):
-        node.expr = self.visit(node.expr)
-        node.ops = [(op, self.visit(n)) for op, n in  node.ops]
-        return node
+        return self._clone(node, self.visit(node.expr),
+            [(op, self.visit(n)) for op, n in  node.ops]
+        )
 
     def _visitUnaryOp(self, node):
-        node.expr = self.visit(node.expr)
-        return node
+        return self._clone(node, self.visit(node.expr))
     visitUnaryAdd = visitUnarySub = visitNot = visitInvert = _visitUnaryOp
     visitBackquote = visitDiscard = _visitUnaryOp
 
     def visitIfExp(self, node):
-        node.test = self.visit(node.test)
-        node.then = self.visit(node.then)
-        node.else_ = self.visit(node.else_)
-        return node
+        return self._clone(node, self.visit(node.test), self.visit(node.then),
+            self.visit(node.else_)
+        )
 
     # Identifiers, Literals and Comprehensions
 
     def visitDict(self, node):
-        node.items = [(self.visit(k),
-                       self.visit(v)) for k, v in node.items]
-        return node
+        return self._clone(node, 
+            [(self.visit(k), self.visit(v)) for k, v in node.items]
+        )
 
     def visitGenExpr(self, node):
-        node.code = self.visit(node.code)
-        node.filename = '<string>' # workaround for bug in pycodegen
-        return node
+        return self._clone(node, self.visit(node.code))
 
     def visitGenExprFor(self, node):
-        node.assign = self.visit(node.assign)
-        node.iter = self.visit(node.iter)
-        node.ifs = [self.visit(x) for x in node.ifs]
-        return node
+        return self._clone(node, self.visit(node.assign), self.visit(node.iter),
+            [self.visit(x) for x in node.ifs]
+        )
 
     def visitGenExprIf(self, node):
-        node.test = self.visit(node.test)
-        return node
+        return self._clone(node, self.visit(node.test))
 
     def visitGenExprInner(self, node):
-        node.quals = [self.visit(x) for x in node.quals]
-        node.expr = self.visit(node.expr)
-        return node
+        quals = [self.visit(x) for x in node.quals]
+        return self._clone(node, self.visit(node.expr), quals)
 
     def visitKeyword(self, node):
-        node.expr = self.visit(node.expr)
-        return node
+        return self._clone(node, node.name, self.visit(node.expr))
 
     def visitList(self, node):
-        node.nodes = [self.visit(n) for n in node.nodes]
-        return node
+        return self._clone(node, [self.visit(n) for n in node.nodes])
 
     def visitListComp(self, node):
-        node.quals = [self.visit(x) for x in node.quals]
-        node.expr = self.visit(node.expr)
-        return node
+        quals = [self.visit(x) for x in node.quals]
+        return self._clone(node, self.visit(node.expr), quals)
 
     def visitListCompFor(self, node):
-        node.assign = self.visit(node.assign)
-        node.list = self.visit(node.list)
-        node.ifs = [self.visit(x) for x in node.ifs]
-        return node
+        return self._clone(node, self.visit(node.assign), self.visit(node.list),
+            [self.visit(x) for x in node.ifs]
+        )
 
     def visitListCompIf(self, node):
-        node.test = self.visit(node.test)
-        return node
+        return self._clone(node, self.visit(node.test))
 
     def visitSlice(self, node):
-        node.expr = self.visit(node.expr)
-        if node.lower is not None:
-            node.lower = self.visit(node.lower)
-        if node.upper is not None:
-            node.upper = self.visit(node.upper)
-        return node
+        return self._clone(node, self.visit(node.expr), node.flags,
+            node.lower and self.visit(node.lower) or None,
+            node.upper and self.visit(node.upper) or None
+        )
 
     def visitSliceobj(self, node):
-        node.nodes = [self.visit(x) for x in node.nodes]
-        return node
+        return self._clone(node, [self.visit(x) for x in node.nodes])
 
     def visitTuple(self, node):
-        node.nodes = [self.visit(n) for n in node.nodes]
-        return node
+        return self._clone(node, [self.visit(n) for n in node.nodes])
 
 
 class TemplateASTTransformer(ASTTransformer):
@@ -687,39 +668,45 @@
 
     def visitClass(self, node):
         self.locals.append(set())
-        node = ASTTransformer.visitClass(self, node)
-        self.locals.pop()
-        return node
+        try:
+            return ASTTransformer.visitClass(self, node)
+        finally:
+            self.locals.pop()
 
     def visitFor(self, node):
         self.locals.append(set())
-        node = ASTTransformer.visitFor(self, node)
-        self.locals.pop()
-        return node
+        try:
+            return ASTTransformer.visitFor(self, node)
+        finally:
+            self.locals.pop()
 
     def visitFunction(self, node):
         self.locals.append(set(node.argnames))
-        node = ASTTransformer.visitFunction(self, node)
-        self.locals.pop()
-        return node
+        try:
+            return ASTTransformer.visitFunction(self, node)
+        finally:
+            self.locals.pop()
 
     def visitGenExpr(self, node):
         self.locals.append(set())
-        node = ASTTransformer.visitGenExpr(self, node)
-        self.locals.pop()
-        return node
+        try:
+            return ASTTransformer.visitGenExpr(self, node)
+        finally:
+            self.locals.pop()
 
     def visitLambda(self, node):
         self.locals.append(set(flatten(node.argnames)))
-        node = ASTTransformer.visitLambda(self, node)
-        self.locals.pop()
-        return node
+        try:
+            return ASTTransformer.visitLambda(self, node)
+        finally:
+            self.locals.pop()
 
     def visitListComp(self, node):
         self.locals.append(set())
-        node = ASTTransformer.visitListComp(self, node)
-        self.locals.pop()
-        return node
+        try:
+            return ASTTransformer.visitListComp(self, node)
+        finally:
+            self.locals.pop()
 
     def visitName(self, node):
         # If the name refers to a local inside a lambda, list comprehension, or