Mercurial > genshi > genshi-test
changeset 565:aa8e85a4085e
* The I18n extractor now handles gettext function calls that use non-string parameters as well as keyword arguments.
* Expression and Suite objects now retain the AST information so that it can be used by the I18n message extraction.
* The AST transformation no longer modifies the AST in place, but instead returns a transformed copy of the AST node.
author | cmlenz |
---|---|
date | Fri, 13 Jul 2007 16:31:47 +0000 |
parents | b3b07279b86c |
children | 8f0d601afc0c |
files | genshi/filters/i18n.py genshi/filters/tests/i18n.py genshi/template/eval.py |
diffstat | 3 files changed, 187 insertions(+), 190 deletions(-) [+] |
line wrap: on
line diff
--- a/genshi/filters/i18n.py +++ b/genshi/filters/i18n.py @@ -13,12 +13,12 @@ """Utilities for internationalization and localization of templates.""" +from compiler import ast try: frozenset except NameError: from sets import ImmutableSet as frozenset from gettext import gettext -from opcode import opmap import re from genshi.core import Attrs, Namespace, QName, START, END, TEXT, START_NS, \ @@ -29,11 +29,6 @@ __all__ = ['Translator', 'extract'] __docformat__ = 'restructuredtext en' -_LOAD_NAME = chr(opmap['LOAD_NAME']) -_LOAD_CONST = chr(opmap['LOAD_CONST']) -_CALL_FUNCTION = chr(opmap['CALL_FUNCTION']) -_BINARY_ADD = chr(opmap['BINARY_ADD']) - I18N_NAMESPACE = Namespace('http://genshi.edgewall.org/i18n') @@ -246,7 +241,7 @@ 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' - 8, 'ngettext', (u'You have %d item', u'You have %d items') + 8, 'ngettext', (u'You have %d item', u'You have %d items', None) :param stream: the event stream to extract strings from; can be a regular stream or a template stream @@ -312,7 +307,7 @@ msgbuf = None elif kind is EXPR or kind is EXEC: - for funcname, strings in extract_from_code(data.code, + for funcname, strings in extract_from_code(data, gettext_functions): yield pos[1], funcname, strings @@ -379,43 +374,38 @@ >>> from genshi.template.eval import Expression >>> expr = Expression('_("Hello")') - >>> list(extract_from_code(expr.code, Translator.GETTEXT_FUNCTIONS)) + >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS)) [('_', u'Hello')] >>> expr = Expression('ngettext("You have %(num)s item", ' ... '"You have %(num)s items", num)') - >>> list(extract_from_code(expr.code, Translator.GETTEXT_FUNCTIONS)) - [('ngettext', (u'You have %(num)s item', u'You have %(num)s items'))] + >>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS)) + [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None))] - :param code: the code object + :param code: the `Code` object + :type code: `genshi.template.eval.Code` :param gettext_functions: a sequence of function names """ - consts = dict([(n, chr(i) + '\x00') for i, n in enumerate(code.co_consts)]) - gettext_locs = [consts[n] for n in gettext_functions if n in consts] - ops = [ - _LOAD_CONST, '(', '|'.join(gettext_locs), ')', - _CALL_FUNCTION, '.\x00', - '((?:', _BINARY_ADD, '|', _LOAD_CONST, '.\x00)+)' - ] - for loc, opcodes in re.findall(''.join(ops), code.co_code): - funcname = code.co_consts[ord(loc[0])] - strings = [] - opcodes = iter(opcodes) - for opcode in opcodes: - if opcode == _BINARY_ADD: - arg = strings.pop() - strings[-1] += arg + def _walk(node): + if isinstance(node, ast.CallFunc) and isinstance(node.node, ast.Name) \ + and node.node.name in gettext_functions: + strings = [] + for arg in node.args: + if isinstance(arg, ast.Const) \ + and isinstance(arg.value, basestring): + strings.append(unicode(arg.value)) + elif not isinstance(arg, ast.Keyword): + strings.append(None) + if len(strings) == 1: + strings = strings[0] else: - arg = code.co_consts[ord(opcodes.next())] - opcodes.next() # skip second byte - if not isinstance(arg, basestring): - break - strings.append(unicode(arg)) - if len(strings) == 1: - strings = strings[0] + strings = tuple(strings) + yield node.node.name, strings else: - strings = tuple(strings) - yield funcname, strings + for child in node.getChildNodes(): + for funcname, strings in _walk(child): + yield funcname, strings + return _walk(code.ast) def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|\]')): """Parse a message using Genshi compound message formatting.
--- a/genshi/filters/tests/i18n.py +++ b/genshi/filters/tests/i18n.py @@ -28,7 +28,8 @@ translator = Translator() messages = list(translator.extract(tmpl.stream)) self.assertEqual(1, len(messages)) - self.assertEqual((2, 'ngettext', (u'Singular', u'Plural')), messages[0]) + self.assertEqual((2, 'ngettext', (u'Singular', u'Plural', None)), + messages[0]) def test_extract_included_attribute_text(self): tmpl = MarkupTemplate("""<html xmlns:py="http://genshi.edgewall.org/"> @@ -263,7 +264,8 @@ (3, None, u'Example', []), (6, None, u'Example', []), (7, '_', u'Hello, %(name)s', []), - (8, 'ngettext', (u'You have %d item', u'You have %d items'), []), + (8, 'ngettext', (u'You have %d item', u'You have %d items', None), + []), ], results) def test_text_template_extraction(self): @@ -281,10 +283,28 @@ })) self.assertEqual([ (1, '_', u'Dear %(name)s', []), - (3, 'ngettext', (u'Your item:', u'Your items'), []), + (3, 'ngettext', (u'Your item:', u'Your items', None), []), (7, None, u'All the best,\n Foobar', []) ], results) + def test_extraction_with_keyword_arg(self): + buf = StringIO("""<html xmlns:py="http://genshi.edgewall.org/"> + ${gettext('Foobar', foo='bar')} + </html>""") + results = list(extract(buf, ['gettext'], [], {})) + self.assertEqual([ + (2, 'gettext', (u'Foobar'), []), + ], results) + + def test_extraction_with_nonstring_arg(self): + buf = StringIO("""<html xmlns:py="http://genshi.edgewall.org/"> + ${dgettext(curdomain, 'Foobar')} + </html>""") + results = list(extract(buf, ['dgettext'], [], {})) + self.assertEqual([ + (2, 'dgettext', (None, u'Foobar'), []), + ], results) + def test_extraction_inside_ignored_tags(self): buf = StringIO("""<html xmlns:py="http://genshi.edgewall.org/"> <script type="text/javascript">
--- a/genshi/template/eval.py +++ b/genshi/template/eval.py @@ -20,6 +20,7 @@ try: set except NameError: + from sets import ImmutableSet as frozenset from sets import Set as set import sys @@ -34,7 +35,7 @@ class Code(object): """Abstract base class for the `Expression` and `Suite` classes.""" - __slots__ = ['source', 'code', '_globals'] + __slots__ = ['source', 'code', 'ast', '_globals'] def __init__(self, source, filename=None, lineno=-1, lookup='lenient'): """Create the code object, either from a string, or from an AST node. @@ -59,6 +60,7 @@ else: node = ast.Module(None, source) + self.ast = node self.code = _compile(node, self.source, mode=self.mode, filename=filename, lineno=lineno) if lookup is None: @@ -409,244 +411,223 @@ self._visitDefault) return visitor(node) + def _clone(self, node, *args): + lineno = getattr(node, 'lineno', None) + node = node.__class__(*args) + if lineno is not None: + node.lineno = lineno + if isinstance(node, (ast.Class, ast.Function, ast.GenExpr, ast.Lambda)): + node.filename = '<string>' # workaround for bug in pycodegen + return node + def _visitDefault(self, node): return node def visitExpression(self, node): - node.node = self.visit(node.node) - return node + return self._clone(node, self.visit(node.node)) def visitModule(self, node): - node.node = self.visit(node.node) - return node + return self._clone(node, node.doc, self.visit(node.node)) def visitStmt(self, node): - node.nodes = [self.visit(x) for x in node.nodes] - return node + return self._clone(node, [self.visit(x) for x in node.nodes]) # Classes, Functions & Accessors def visitCallFunc(self, node): - node.node = self.visit(node.node) - node.args = [self.visit(x) for x in node.args] - if node.star_args: - node.star_args = self.visit(node.star_args) - if node.dstar_args: - node.dstar_args = self.visit(node.dstar_args) - return node + return self._clone(node, self.visit(node.node), + [self.visit(x) for x in node.args], + node.star_args and self.visit(node.star_args) or None, + node.dstar_args and self.visit(node.dstar_args) or None + ) def visitClass(self, node): - node.bases = [self.visit(x) for x in node.bases] - node.code = self.visit(node.code) - node.filename = '<string>' # workaround for bug in pycodegen - return node + return self._clone(node, node.name, [self.visit(x) for x in node.bases], + node.doc, node.code + ) def visitFunction(self, node): + args = [] if hasattr(node, 'decorators'): - node.decorators = self.visit(node.decorators) - node.defaults = [self.visit(x) for x in node.defaults] - node.code = self.visit(node.code) - node.filename = '<string>' # workaround for bug in pycodegen - return node + args.append(self.visit(node.decorators)) + return self._clone(node, *args + [ + node.name, + node.argnames, + [self.visit(x) for x in node.defaults], + node.flags, + node.doc, + self.visit(node.code) + ]) def visitGetattr(self, node): - node.expr = self.visit(node.expr) - return node + return self._clone(node, self.visit(node.expr), node.attrname) def visitLambda(self, node): - node.code = self.visit(node.code) - node.filename = '<string>' # workaround for bug in pycodegen + node = self._clone(node, node.argnames, + [self.visit(x) for x in node.defaults], node.flags, + self.visit(node.code) + ) return node def visitSubscript(self, node): - node.expr = self.visit(node.expr) - node.subs = [self.visit(x) for x in node.subs] - return node + return self._clone(node, self.visit(node.expr), node.flags, + [self.visit(x) for x in node.subs] + ) # Statements def visitAssert(self, node): - node.test = self.visit(node.test) - node.fail = self.visit(node.fail) - return node + return self._clone(node, self.visit(node.test), self.visit(node.fail)) def visitAssign(self, node): - node.nodes = [self.visit(x) for x in node.nodes] - node.expr = self.visit(node.expr) - return node + return self._clone(node, [self.visit(x) for x in node.nodes], + self.visit(node.expr) + ) def visitAssAttr(self, node): - node.expr = self.visit(node.expr) - return node + return self._clone(node, self.visit(node.expr), node.attrname, + node.flags + ) def visitAugAssign(self, node): - node.node = self.visit(node.node) - node.expr = self.visit(node.expr) - return node + return self._clone(node, self.visit(node.node), node.op, + self.visit(node.expr) + ) def visitDecorators(self, node): - node.nodes = [self.visit(x) for x in node.nodes] - return node + return self._clone(node, [self.visit(x) for x in node.nodes]) def visitExec(self, node): - node.expr = self.visit(node.expr) - node.locals = self.visit(node.locals) - node.globals = self.visit(node.globals) - return node + return self._clone(node, self.visit(node.expr), self.visit(node.locals), + self.visit(node.globals) + ) def visitFor(self, node): - node.assign = self.visit(node.assign) - node.list = self.visit(node.list) - node.body = self.visit(node.body) - node.else_ = self.visit(node.else_) - return node + return self._clone(node, self.visit(node.assign), self.visit(node.list), + self.visit(node.body), self.visit(node.else_) + ) def visitIf(self, node): - node.tests = [self.visit(x) for x in node.tests] - node.else_ = self.visit(node.else_) - return node + return self._clone(node, [self.visit(x) for x in node.tests], + self.visit(node.else_) + ) def _visitPrint(self, node): - node.nodes = [self.visit(x) for x in node.nodes] - node.dest = self.visit(node.dest) - return node + return self._clone(node, [self.visit(x) for x in node.nodes], + self.visit(node.dest) + ) visitPrint = visitPrintnl = _visitPrint def visitRaise(self, node): - node.expr1 = self.visit(node.expr1) - node.expr2 = self.visit(node.expr2) - node.expr3 = self.visit(node.expr3) - return node + return self._clone(node, self.visit(node.expr1), self.visit(node.expr2), + self.visit(node.expr3) + ) def visitReturn(self, node): - node.value = self.visit(node.value) - return node + return self._clone(node, self.visit(node.value)) def visitTryExcept(self, node): - node.body = self.visit(node.body) - node.handlers = self.visit(node.handlers) - node.else_ = self.visit(node.else_) - return node + return self._clone(node, self.visit(node.body), self.visit(node.handlers), + self.visit(node.else_) + ) def visitTryFinally(self, node): - node.body = self.visit(node.body) - node.final = self.visit(node.final) - return node + return self._clone(node, self.visit(node.body), self.visit(node.final)) def visitWhile(self, node): - node.test = self.visit(node.test) - node.body = self.visit(node.body) - node.else_ = self.visit(node.else_) - return node + return self._clone(node, self.visit(node.test), self.visit(node.body), + self.visit(node.else_) + ) def visitWith(self, node): - node.expr = self.visit(node.expr) - node.vars = [self.visit(x) for x in node.vars] - node.body = self.visit(node.body) - return node + return self._clone(node, self.visit(node.expr), + [self.visit(x) for x in node.vars], self.visit(node.body) + ) def visitYield(self, node): - node.value = self.visit(node.value) - return node + return self._clone(node, self.visit(node.value)) # Operators def _visitBoolOp(self, node): - node.nodes = [self.visit(x) for x in node.nodes] - return node + return self._clone(node, [self.visit(x) for x in node.nodes]) visitAnd = visitOr = visitBitand = visitBitor = visitBitxor = _visitBoolOp visitAssTuple = visitAssList = _visitBoolOp def _visitBinOp(self, node): - node.left = self.visit(node.left) - node.right = self.visit(node.right) - return node + return self._clone(node, + (self.visit(node.left), self.visit(node.right)) + ) visitAdd = visitSub = _visitBinOp visitDiv = visitFloorDiv = visitMod = visitMul = visitPower = _visitBinOp visitLeftShift = visitRightShift = _visitBinOp def visitCompare(self, node): - node.expr = self.visit(node.expr) - node.ops = [(op, self.visit(n)) for op, n in node.ops] - return node + return self._clone(node, self.visit(node.expr), + [(op, self.visit(n)) for op, n in node.ops] + ) def _visitUnaryOp(self, node): - node.expr = self.visit(node.expr) - return node + return self._clone(node, self.visit(node.expr)) visitUnaryAdd = visitUnarySub = visitNot = visitInvert = _visitUnaryOp visitBackquote = visitDiscard = _visitUnaryOp def visitIfExp(self, node): - node.test = self.visit(node.test) - node.then = self.visit(node.then) - node.else_ = self.visit(node.else_) - return node + return self._clone(node, self.visit(node.test), self.visit(node.then), + self.visit(node.else_) + ) # Identifiers, Literals and Comprehensions def visitDict(self, node): - node.items = [(self.visit(k), - self.visit(v)) for k, v in node.items] - return node + return self._clone(node, + [(self.visit(k), self.visit(v)) for k, v in node.items] + ) def visitGenExpr(self, node): - node.code = self.visit(node.code) - node.filename = '<string>' # workaround for bug in pycodegen - return node + return self._clone(node, self.visit(node.code)) def visitGenExprFor(self, node): - node.assign = self.visit(node.assign) - node.iter = self.visit(node.iter) - node.ifs = [self.visit(x) for x in node.ifs] - return node + return self._clone(node, self.visit(node.assign), self.visit(node.iter), + [self.visit(x) for x in node.ifs] + ) def visitGenExprIf(self, node): - node.test = self.visit(node.test) - return node + return self._clone(node, self.visit(node.test)) def visitGenExprInner(self, node): - node.quals = [self.visit(x) for x in node.quals] - node.expr = self.visit(node.expr) - return node + quals = [self.visit(x) for x in node.quals] + return self._clone(node, self.visit(node.expr), quals) def visitKeyword(self, node): - node.expr = self.visit(node.expr) - return node + return self._clone(node, node.name, self.visit(node.expr)) def visitList(self, node): - node.nodes = [self.visit(n) for n in node.nodes] - return node + return self._clone(node, [self.visit(n) for n in node.nodes]) def visitListComp(self, node): - node.quals = [self.visit(x) for x in node.quals] - node.expr = self.visit(node.expr) - return node + quals = [self.visit(x) for x in node.quals] + return self._clone(node, self.visit(node.expr), quals) def visitListCompFor(self, node): - node.assign = self.visit(node.assign) - node.list = self.visit(node.list) - node.ifs = [self.visit(x) for x in node.ifs] - return node + return self._clone(node, self.visit(node.assign), self.visit(node.list), + [self.visit(x) for x in node.ifs] + ) def visitListCompIf(self, node): - node.test = self.visit(node.test) - return node + return self._clone(node, self.visit(node.test)) def visitSlice(self, node): - node.expr = self.visit(node.expr) - if node.lower is not None: - node.lower = self.visit(node.lower) - if node.upper is not None: - node.upper = self.visit(node.upper) - return node + return self._clone(node, self.visit(node.expr), node.flags, + node.lower and self.visit(node.lower) or None, + node.upper and self.visit(node.upper) or None + ) def visitSliceobj(self, node): - node.nodes = [self.visit(x) for x in node.nodes] - return node + return self._clone(node, [self.visit(x) for x in node.nodes]) def visitTuple(self, node): - node.nodes = [self.visit(n) for n in node.nodes] - return node + return self._clone(node, [self.visit(n) for n in node.nodes]) class TemplateASTTransformer(ASTTransformer): @@ -687,39 +668,45 @@ def visitClass(self, node): self.locals.append(set()) - node = ASTTransformer.visitClass(self, node) - self.locals.pop() - return node + try: + return ASTTransformer.visitClass(self, node) + finally: + self.locals.pop() def visitFor(self, node): self.locals.append(set()) - node = ASTTransformer.visitFor(self, node) - self.locals.pop() - return node + try: + return ASTTransformer.visitFor(self, node) + finally: + self.locals.pop() def visitFunction(self, node): self.locals.append(set(node.argnames)) - node = ASTTransformer.visitFunction(self, node) - self.locals.pop() - return node + try: + return ASTTransformer.visitFunction(self, node) + finally: + self.locals.pop() def visitGenExpr(self, node): self.locals.append(set()) - node = ASTTransformer.visitGenExpr(self, node) - self.locals.pop() - return node + try: + return ASTTransformer.visitGenExpr(self, node) + finally: + self.locals.pop() def visitLambda(self, node): self.locals.append(set(flatten(node.argnames))) - node = ASTTransformer.visitLambda(self, node) - self.locals.pop() - return node + try: + return ASTTransformer.visitLambda(self, node) + finally: + self.locals.pop() def visitListComp(self, node): self.locals.append(set()) - node = ASTTransformer.visitListComp(self, node) - self.locals.pop() - return node + try: + return ASTTransformer.visitListComp(self, node) + finally: + self.locals.pop() def visitName(self, node): # If the name refers to a local inside a lambda, list comprehension, or