genshi/genshi-test: genshi/filters/i18n.py comparison

comparison genshi/filters/i18n.py @ 892:1de952fd479e

i18n: Support extraction of attributes in markup embedded in ``i18n:msg`` and ``i18n:choose`` directives. See also #380.

author	cmlenz
date	Wed, 21 Apr 2010 10:42:41 +0000
parents	b40dbfee9ba6
children	f30c9fb10272

comparison

equal deleted inserted replaced

-:b40dbfee9ba6
+:1de952fd479e
 MSGBUF = StreamEventKind('MSGBUF')
 SUB_START = StreamEventKind('SUB_START')
 SUB_END = StreamEventKind('SUB_END')
+GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext', 'dgettext', 'dngettext',
+'ugettext', 'ungettext')
 class I18NDirective(Directive):
 """Simple interface for i18n directives to support messages extraction."""
 def __call__(self, stream, directives, ctxt, **vars):
 class ExtractableI18NDirective(I18NDirective):
 """Simple interface for directives to support messages extraction."""
-def extract(self, stream, comment_stack):
+def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS,
+search_text=True, comment_stack=None):
 raise NotImplementedError
 class CommentDirective(I18NDirective):
 """Implementation of the ``i18n:comment`` template directive which adds
 >>> list(translator.extract(tmpl.stream))
 [(2, None, u'Foo', [u'As in Foo Bar'])]
 """
 __slots__ = ['comment']
-def __init__(self, value, template, hints=None, namespaces=None,
+def __init__(self, value, template=None, namespaces=None, lineno=-1,
-lineno=-1, offset=-1):
+offset=-1):
 Directive.__init__(self, None, template, namespaces, lineno, offset)
 self.comment = value
 class MsgDirective(ExtractableI18NDirective):
 Starting and ending white-space is stripped of to make it simpler for
 translators. Stripping it is not that important since it's on the html
 source, the rendered output will remain the same.
 """
-__slots__ = ['params']
+__slots__ = ['params', 'lineno']
-def __init__(self, value, template, hints=None, namespaces=None,
+def __init__(self, value, template=None, namespaces=None, lineno=-1,
-lineno=-1, offset=-1):
+offset=-1):
 Directive.__init__(self, None, template, namespaces, lineno, offset)
 self.params = [param.strip() for param in value.split(',') if param]
+self.lineno = lineno
 @classmethod
 def attach(cls, template, stream, value, namespaces, pos):
 if type(value) is dict:
 value = value.get('params', '').strip()
 if previous:
 yield previous
 return _apply_directives(_generate(), directives, ctxt, vars)
-def extract(self, stream, comment_stack):
+def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS,
+search_text=True, comment_stack=None):
 msgbuf = MessageBuffer(self)
+strip = False
 stream = iter(stream)
 previous = stream.next()
 if previous[0] is START:
+for message in translator._extract_attrs(previous,
+gettext_functions,
+search_text=search_text):
+yield message
 previous = stream.next()
+strip = True
 for event in stream:
+if event[0] is START:
+for message in translator._extract_attrs(event,
+gettext_functions,
+search_text=search_text):
+yield message
 msgbuf.append(*previous)
 previous = event
-msgbuf.append(*previous)
+if not strip:
+msgbuf.append(*previous)
-yield None, msgbuf.format(), comment_stack[-1:]
+yield self.lineno, None, msgbuf.format(), comment_stack[-1:]
 class ChooseBranchDirective(I18NDirective):
 __slots__ = ['params']
 def __call__(self, stream, directives, ctxt, **vars):
 self.params = ctxt.get('_i18n.choose.params', [])[:]
 msgbuf = MessageBuffer(self)
 stream = iter(_apply_directives(stream, directives, ctxt, vars))
 else:
 msgbuf.append(*previous)
 ctxt['_i18n.choose.%s' % type(self).__name__] = msgbuf
-def extract(self, stream, comment_stack, msgbuf):
+def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS,
+search_text=True, comment_stack=None, msgbuf=None):
 stream = iter(stream)
 previous = stream.next()
 if previous[0] is START:
+# skip the enclosing element
+for message in translator._extract_attrs(previous,
+gettext_functions,
+search_text=search_text):
+yield message
 previous = stream.next()
 for event in stream:
+if previous[0] is START:
+for message in translator._extract_attrs(previous,
+gettext_functions,
+search_text=search_text):
+yield message
 msgbuf.append(*previous)
 previous = event
 if previous[0] is not END:
 msgbuf.append(*previous)
-return msgbuf
 class SingularDirective(ChooseBranchDirective):
 """Implementation of the ``i18n:singular`` directive to be used with the
 ``i18n:choose`` directive."""
 >>> translator.setup(tmpl)
 >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE
 [(2, 'ngettext', (u'There is %(num)s coin',
 u'There are %(num)s coins'), [])]
 """
-__slots__ = ['numeral', 'params']
+__slots__ = ['numeral', 'params', 'lineno']
-def __init__(self, value, template, hints=None, namespaces=None,
+def __init__(self, value, template=None, namespaces=None, lineno=-1,
-lineno=-1, offset=-1):
+offset=-1):
 Directive.__init__(self, None, template, namespaces, lineno, offset)
 params = [v.strip() for v in value.split(';')]
 self.numeral = self._parse_expr(params.pop(0), template, lineno, offset)
 self.params = params and [name.strip() for name in
 params[0].split(',') if name] or []
+self.lineno = lineno
 @classmethod
 def attach(cls, template, stream, value, namespaces, pos):
 if type(value) is dict:
 numeral = value.get('numeral', '').strip()
 ngettext = ctxt.get('_i18n.ungettext')
 assert hasattr(ngettext, '__call__'), 'No ngettext function available'
 dngettext = ctxt.get('_i18n.dngettext')
 if not dngettext:
 dngettext = lambda d, s, p, n: ngettext(s, p, n)
 for kind, event, pos in stream:
 if kind is SUB:
 subdirectives, substream = event
 if isinstance(subdirectives[0],
 SingularDirective) and not singular_stream:
 # plural test strings
 singular_test = u'O\x85\xbe\xa9\xa8az\xc3?\xe6\xa1\x02n\x84\x93'
 plural_test = u'\xcc\xfb+\xd3Pn\x9d\tT\xec\x1d\xda\x1a\x88\x00'
 translation = ngettext(singular_test, plural_test,
 self.numeral.evaluate(ctxt))
-if translation==singular_test:
+if translation == singular_test:
 chosen_msgbuf = singular_msgbuf
 chosen_stream = singular_stream
 else:
 chosen_msgbuf = plural_msgbuf
 chosen_stream = plural_stream
 else:
 yield kind, data, pos
 ctxt.pop()
-def extract(self, stream, comment_stack):
+def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS,
+search_text=True, comment_stack=None):
+strip = False
 stream = iter(stream)
 previous = stream.next()
-if previous is START:
-stream.next()
+if previous[0] is START:
+# skip the enclosing element
+for message in translator._extract_attrs(previous,
+gettext_functions,
+search_text=search_text):
+yield message
+previous = stream.next()
+strip = True
 singular_msgbuf = MessageBuffer(self)
 plural_msgbuf = MessageBuffer(self)
-for kind, event, pos in stream:
+for event in stream:
-if kind is SUB:
+if previous[0] is SUB:
-subdirectives, substream = event
+directives, substream = previous[1]
-for subdirective in subdirectives:
+for directive in directives:
-if isinstance(subdirective, SingularDirective):
+if isinstance(directive, SingularDirective):
-singular_msgbuf = subdirective.extract(substream, comment_stack,
+for message in directive.extract(translator,
-singular_msgbuf)
+substream, gettext_functions, search_text,
-elif isinstance(subdirective, PluralDirective):
+comment_stack, msgbuf=singular_msgbuf):
-plural_msgbuf = subdirective.extract(substream, comment_stack,
+yield message
-plural_msgbuf)
+elif isinstance(directive, PluralDirective):
-elif not isinstance(subdirective, StripDirective):
+for message in directive.extract(translator,
-singular_msgbuf.append(kind, event, pos)
+substream, gettext_functions, search_text,
-plural_msgbuf.append(kind, event, pos)
+comment_stack, msgbuf=plural_msgbuf):
+yield message
+elif not isinstance(directive, StripDirective):
+singular_msgbuf.append(*previous)
+plural_msgbuf.append(*previous)
 else:
-singular_msgbuf.append(kind, event, pos)
+if previous[0] is START:
-plural_msgbuf.append(kind, event, pos)
+for message in translator._extract_attrs(previous,
+gettext_functions,
-yield 'ngettext', \
+search_text):
+yield message
+singular_msgbuf.append(*previous)
+plural_msgbuf.append(*previous)
+previous = event
+if not strip:
+singular_msgbuf.append(*previous)
+plural_msgbuf.append(*previous)
+yield self.lineno, 'ngettext', \
 (singular_msgbuf.format(), plural_msgbuf.format()), \
 comment_stack[-1:]
 class DomainDirective(I18NDirective):
 <p>Voh</p>
 </html>
 """
 __slots__ = ['domain']
-def __init__(self, value, template, hints=None, namespaces=None,
+def __init__(self, value, template=None, namespaces=None, lineno=-1,
-lineno=-1, offset=-1):
+offset=-1):
 Directive.__init__(self, None, template, namespaces, lineno, offset)
 self.domain = value and value.strip() or '__DEFAULT__'
 @classmethod
 def attach(cls, template, stream, value, namespaces, pos):
 if current_domain:
 ctxt.pop()
 else:
 yield kind, data, pos
-GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext', 'dgettext', 'dngettext',
-'ugettext', 'ungettext')
 def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS,
 search_text=True, comment_stack=None):
 """Extract localizable strings from the given template stream.
 if kind is END:
 skip -= 1
 if kind is START and not skip:
 tag, attrs = data
 if tag in self.ignore_tags or \
 isinstance(attrs.get(xml_lang), basestring):
 skip += 1
 continue
-for name, value in attrs:
+for message in self._extract_attrs((kind, data, pos),
-if search_text and isinstance(value, basestring):
+gettext_functions,
-if name in self.include_attrs:
+search_text=search_text):
-text = value.strip()
+yield message
-if text:
-# XXX: Do we need to grab i18n:comment from comment_stack ???
-yield pos[1], None, text, []
-else:
-for lineno, funcname, text, comments in self.extract(
-_ensure(value), gettext_functions,
-search_text=False):
-yield lineno, funcname, text, comments
 elif not skip and search_text and kind is TEXT:
 text = data.strip()
 if text and [ch for ch in text if ch.isalpha()]:
 yield pos[1], None, text, comment_stack[-1:]
 in_comment = True
 comment_stack.append(directive.comment)
 if len(directives) == 1:
 # in case we're in the presence of something like:
 # <p i18n:comment="foo">Foo</p>
-messages = self.extract(
+for message in self.extract(
 substream, gettext_functions,
 search_text=search_text and not skip,
-comment_stack=comment_stack)
+comment_stack=comment_stack):
-for lineno, funcname, text, comments in messages:
+yield message
-yield lineno, funcname, text, comments
 directives.pop(idx)
 elif not isinstance(directive, I18NDirective):
 # Remove all other non i18n directives from the process
 directives.pop(idx)
 if not directives and not in_comment:
 # Extract content if there's no directives because
 # strip was pop'ed and not because comment was pop'ed.
 # Extraction in this case has been taken care of.
-messages = self.extract(
+for message in self.extract(
 substream, gettext_functions,
-search_text=search_text and not skip)
+search_text=search_text and not skip):
-for lineno, funcname, text, comments in messages:
+yield message
-yield lineno, funcname, text, comments
 for directive in directives:
 if isinstance(directive, ExtractableI18NDirective):
-messages = directive.extract(substream, comment_stack)
+for message in directive.extract(self,
-for funcname, text, comments in messages:
+substream, gettext_functions,
-yield pos[1], funcname, text, comments
+search_text=search_text and not skip,
+comment_stack=comment_stack):
+yield message
 else:
-messages = self.extract(
+for message in self.extract(
 substream, gettext_functions,
-search_text=search_text and not skip)
+search_text=search_text and not skip,
-for lineno, funcname, text, comments in messages:
+comment_stack=comment_stack):
-yield lineno, funcname, text, comments
+yield message
 if in_comment:
 comment_stack.pop()
 def get_directive_index(self, dir_cls):
 :param template: a `Template` instance
 """
 template.filters.insert(0, self)
 if hasattr(template, 'add_directives'):
 template.add_directives(Translator.NAMESPACE, self)
+def _extract_attrs(self, event, gettext_functions, search_text):
+for name, value in event[1][1]:
+if search_text and isinstance(value, basestring):
+if name in self.include_attrs:
+text = value.strip()
+if text:
+yield event[2][1], None, text, []
+else:
+for message in self.extract(_ensure(value), gettext_functions,
+search_text=False):
+yield message
 class MessageBuffer(object):
 """Helper class for managing internationalized mixed content.
 params = "(%s)" % params
 raise IndexError("%d parameters%s given to 'i18n:%s' but "
 "%d or more expressions used in '%s', line %s"
 % (len(self.orig_params), params,
 self.directive.tagname,
-len(self.orig_params)+1,
+len(self.orig_params) + 1,
 os.path.basename(pos[0] or
 'In-memory Template'),
 pos[1]))
 self.string.append('%%(%s)s' % param)
 self.events.setdefault(self.stack[-1], []).append((kind, data, pos))
 if substream is not None:
 substream.append(event)
 else:
 yield event
 def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|(?<!\\)\]')):
 """Parse a translated message using Genshi mixed content message
 formatting.
 >>> parse_msg("See [1:Help].")
 def extract_from_code(code, gettext_functions):
 """Extract strings from Python bytecode.
 >>> from genshi.template.eval import Expression
 >>> expr = Expression('_("Hello")')
->>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS))
+>>> list(extract_from_code(expr, GETTEXT_FUNCTIONS))
 [('_', u'Hello')]
 >>> expr = Expression('ngettext("You have %(num)s item", '
 ...                            '"You have %(num)s items", num)')
->>> list(extract_from_code(expr, Translator.GETTEXT_FUNCTIONS))
+>>> list(extract_from_code(expr, GETTEXT_FUNCTIONS))
 [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None))]
 :param code: the `Code` object
 :type code: `genshi.template.eval.Code`
 :param gettext_functions: a sequence of function names

Mercurial > genshi > genshi-test

comparison genshi/filters/i18n.py @ 892:1de952fd479e