Foo
cmlenz@902: ... ''') cmlenz@902: >>> translator = Translator() cmlenz@902: >>> translator.setup(tmpl) cmlenz@902: >>> list(translator.extract(tmpl.stream)) cmlenz@902: [(2, None, u'Foo', [u'As in Foo Bar'])] cmlenz@902: """ cmlenz@902: __slots__ = ['comment'] cmlenz@902: cmlenz@902: def __init__(self, value, template=None, namespaces=None, lineno=-1, cmlenz@902: offset=-1): cmlenz@902: Directive.__init__(self, None, template, namespaces, lineno, offset) cmlenz@902: self.comment = value cmlenz@902: cmlenz@902: cmlenz@902: class MsgDirective(ExtractableI18NDirective): cmlenz@902: r"""Implementation of the ``i18n:msg`` directive which marks inner content cmlenz@902: as translatable. Consider the following examples: cmlenz@902: cmlenz@902: >>> tmpl = MarkupTemplate(''' cmlenz@902: ...Foo
cmlenz@902: ...Bar
cmlenz@902: ...Foo bar!
cmlenz@902: ... ''') cmlenz@902: cmlenz@902: >>> translator = Translator() cmlenz@902: >>> translator.setup(tmpl) cmlenz@902: >>> list(translator.extract(tmpl.stream)) cmlenz@902: [(2, None, u'[1:Foo]\n [2:Bar]', []), (6, None, u'Foo [1:bar]!', [])] cmlenz@902: >>> print(tmpl.generate().render()) cmlenz@902: cmlenz@902:Foo
cmlenz@902:Bar
Foo bar!
cmlenz@902: cmlenz@902: cmlenz@902: >>> tmpl = MarkupTemplate(''' cmlenz@902: ...First Name: ${fname}
cmlenz@902: ...Last Name: ${lname}
cmlenz@902: ...Foo bar!
cmlenz@902: ... ''') cmlenz@902: >>> translator.setup(tmpl) cmlenz@902: >>> list(translator.extract(tmpl.stream)) #doctest: +NORMALIZE_WHITESPACE cmlenz@902: [(2, None, u'[1:First Name: %(fname)s]\n [2:Last Name: %(lname)s]', []), cmlenz@902: (6, None, u'Foo [1:bar]!', [])] cmlenz@902: cmlenz@902: >>> tmpl = MarkupTemplate(''' cmlenz@902: ...First Name: ${fname}
cmlenz@902: ...Last Name: ${lname}
cmlenz@902: ...Foo bar!
cmlenz@902: ... ''') cmlenz@902: >>> translator.setup(tmpl) cmlenz@902: >>> print(tmpl.generate(fname='John', lname='Doe').render()) cmlenz@902: cmlenz@902:First Name: John
cmlenz@902:Last Name: Doe
Foo bar!
cmlenz@902: cmlenz@902: cmlenz@902: Starting and ending white-space is stripped of to make it simpler for cmlenz@902: translators. Stripping it is not that important since it's on the html cmlenz@902: source, the rendered output will remain the same. cmlenz@902: """ cmlenz@902: __slots__ = ['params', 'lineno'] cmlenz@902: cmlenz@902: def __init__(self, value, template=None, namespaces=None, lineno=-1, cmlenz@902: offset=-1): cmlenz@902: Directive.__init__(self, None, template, namespaces, lineno, offset) cmlenz@902: self.params = [param.strip() for param in value.split(',') if param] cmlenz@902: self.lineno = lineno cmlenz@820: cmlenz@820: @classmethod cmlenz@820: def attach(cls, template, stream, value, namespaces, pos): cmlenz@902: if type(value) is dict: cmlenz@902: value = value.get('params', '').strip() cmlenz@902: return super(MsgDirective, cls).attach(template, stream, value.strip(), cmlenz@902: namespaces, pos) cmlenz@902: cmlenz@902: def __call__(self, stream, directives, ctxt, **vars): cmlenz@902: gettext = ctxt.get('_i18n.gettext') cmlenz@902: if ctxt.get('_i18n.domain'): cmlenz@902: dgettext = ctxt.get('_i18n.dgettext') cmlenz@902: assert hasattr(dgettext, '__call__'), \ cmlenz@902: 'No domain gettext function passed' cmlenz@902: gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg) cmlenz@902: cmlenz@902: def _generate(): cmlenz@902: msgbuf = MessageBuffer(self) cmlenz@902: previous = stream.next() cmlenz@902: if previous[0] is START: cmlenz@902: yield previous cmlenz@902: else: cmlenz@902: msgbuf.append(*previous) cmlenz@902: previous = stream.next() cmlenz@902: for kind, data, pos in stream: cmlenz@902: msgbuf.append(*previous) cmlenz@902: previous = kind, data, pos cmlenz@902: if previous[0] is not END: cmlenz@902: msgbuf.append(*previous) cmlenz@902: previous = None cmlenz@902: for event in msgbuf.translate(gettext(msgbuf.format())): cmlenz@902: yield event cmlenz@902: if previous: cmlenz@902: yield previous cmlenz@902: cmlenz@902: return _apply_directives(_generate(), directives, ctxt, vars) cmlenz@902: cmlenz@902: def extract(self, translator, stream, gettext_functions=GETTEXT_FUNCTIONS, cmlenz@902: search_text=True, comment_stack=None): cmlenz@902: msgbuf = MessageBuffer(self) cmlenz@902: strip = False cmlenz@902: cmlenz@902: stream = iter(stream) cmlenz@902: previous = stream.next() cmlenz@902: if previous[0] is START: cmlenz@902: for message in translator._extract_attrs(previous, cmlenz@902: gettext_functions, cmlenz@902: search_text=search_text): cmlenz@902: yield message cmlenz@902: previous = stream.next() cmlenz@902: strip = True cmlenz@902: for event in stream: cmlenz@902: if event[0] is START: cmlenz@902: for message in translator._extract_attrs(event, cmlenz@902: gettext_functions, cmlenz@902: search_text=search_text): cmlenz@902: yield message cmlenz@902: msgbuf.append(*previous) cmlenz@902: previous = event cmlenz@902: if not strip: cmlenz@902: msgbuf.append(*previous) cmlenz@902: cmlenz@902: yield self.lineno, None, msgbuf.format(), comment_stack[-1:] cmlenz@820: cmlenz@820: cmlenz@902: class ChooseBranchDirective(I18NDirective): cmlenz@820: __slots__ = ['params'] cmlenz@820: cmlenz@902: def __call__(self, stream, directives, ctxt, **vars): cmlenz@902: self.params = ctxt.get('_i18n.choose.params', [])[:] cmlenz@902: msgbuf = MessageBuffer(self) cmlenz@902: stream = _apply_directives(stream, directives, ctxt, vars) cmlenz@820: cmlenz@902: previous = stream.next() cmlenz@902: if previous[0] is START: cmlenz@902: yield previous cmlenz@902: else: cmlenz@902: msgbuf.append(*previous) cmlenz@820: cmlenz@902: try: cmlenz@902: previous = stream.next() cmlenz@902: except StopIteration: cmlenz@902: # For exampleThere is $num coin
cmlenz@902: ...There are $num coins
cmlenz@902: ...There is $num coin
cmlenz@902: ...There are $num coins
cmlenz@902: ...There is 1 coin
cmlenz@902:There are 2 coins
cmlenz@902:There is $num coin
cmlenz@902: ...There are $num coins
cmlenz@902: ...Bar
cmlenz@902: ...FooBar
cmlenz@902: ...Bar
cmlenz@902: ...Bar
cmlenz@902: ...Bar
cmlenz@902: ...Bar
cmlenz@902: ... ''') cmlenz@902: cmlenz@902: >>> translations = DummyTranslations({'Bar': 'Voh'}) cmlenz@902: >>> translations.add_domain('foo', {'FooBar': 'BarFoo', 'Bar': 'foo_Bar'}) cmlenz@902: >>> translations.add_domain('bar', {'Bar': 'bar_Bar'}) cmlenz@902: >>> translator = Translator(translations) cmlenz@902: >>> translator.setup(tmpl) cmlenz@902: cmlenz@902: >>> print(tmpl.generate().render()) cmlenz@902: cmlenz@902:Voh
cmlenz@902:BarFoo
cmlenz@902:foo_Bar
cmlenz@902:bar_Bar
cmlenz@902:Voh
cmlenz@902:Voh
cmlenz@902: cmlenz@902: """ cmlenz@902: __slots__ = ['domain'] cmlenz@902: cmlenz@902: def __init__(self, value, template=None, namespaces=None, lineno=-1, cmlenz@902: offset=-1): cmlenz@902: Directive.__init__(self, None, template, namespaces, lineno, offset) cmlenz@902: self.domain = value and value.strip() or '__DEFAULT__' cmlenz@902: cmlenz@902: @classmethod cmlenz@902: def attach(cls, template, stream, value, namespaces, pos): cmlenz@902: if type(value) is dict: cmlenz@902: value = value.get('name') cmlenz@902: return super(DomainDirective, cls).attach(template, stream, value, cmlenz@902: namespaces, pos) cmlenz@902: cmlenz@902: def __call__(self, stream, directives, ctxt, **vars): cmlenz@902: ctxt.push({'_i18n.domain': self.domain}) cmlenz@902: for event in _apply_directives(stream, directives, ctxt, vars): cmlenz@820: yield event cmlenz@902: ctxt.pop() cmlenz@820: cmlenz@820: cmlenz@820: class Translator(DirectiveFactory): cmlenz@500: """Can extract and translate localizable strings from markup streams and cmlenz@500: templates. cmlenz@500: cmlenz@902: For example, assume the following template: cmlenz@500: cmlenz@500: >>> tmpl = MarkupTemplate(''' cmlenz@500: ... cmlenz@500: ...${_("Hello, %(name)s") % dict(name=username)}
cmlenz@500: ... cmlenz@500: ... ''', filename='example.html') cmlenz@500: cmlenz@500: For demonstration, we define a dummy ``gettext``-style function with a cmlenz@500: hard-coded translation table, and pass that to the `Translator` initializer: cmlenz@500: cmlenz@500: >>> def pseudo_gettext(string): cmlenz@500: ... return { cmlenz@500: ... 'Example': 'Beispiel', cmlenz@500: ... 'Hello, %(name)s': 'Hallo, %(name)s' cmlenz@500: ... }[string] cmlenz@500: >>> translator = Translator(pseudo_gettext) cmlenz@500: cmlenz@500: Next, the translator needs to be prepended to any already defined filters cmlenz@500: on the template: cmlenz@500: cmlenz@500: >>> tmpl.filters.insert(0, translator) cmlenz@500: cmlenz@500: When generating the template output, our hard-coded translations should be cmlenz@500: applied as expected: cmlenz@500: cmlenz@902: >>> print(tmpl.generate(username='Hans', _=pseudo_gettext)) cmlenz@500: cmlenz@500: cmlenz@500:Hallo, Hans
cmlenz@500: cmlenz@500: cmlenz@902: cmlenz@820: Note that elements defining ``xml:lang`` attributes that do not contain cmlenz@820: variable expressions are ignored by this filter. That can be used to cmlenz@820: exclude specific parts of a template from being extracted and translated. cmlenz@500: """ cmlenz@500: cmlenz@820: directives = [ cmlenz@902: ('domain', DomainDirective), cmlenz@820: ('comment', CommentDirective), cmlenz@902: ('msg', MsgDirective), cmlenz@902: ('choose', ChooseDirective), cmlenz@902: ('singular', SingularDirective), cmlenz@902: ('plural', PluralDirective) cmlenz@820: ] cmlenz@820: cmlenz@500: IGNORE_TAGS = frozenset([ cmlenz@500: QName('script'), QName('http://www.w3.org/1999/xhtml}script'), cmlenz@500: QName('style'), QName('http://www.w3.org/1999/xhtml}style') cmlenz@500: ]) cmlenz@902: INCLUDE_ATTRS = frozenset([ cmlenz@902: 'abbr', 'alt', 'label', 'prompt', 'standby', 'summary', 'title' cmlenz@902: ]) cmlenz@820: NAMESPACE = I18N_NAMESPACE cmlenz@500: cmlenz@820: def __init__(self, translate=NullTranslations(), ignore_tags=IGNORE_TAGS, cmlenz@820: include_attrs=INCLUDE_ATTRS, extract_text=True): cmlenz@500: """Initialize the translator. cmlenz@500: cmlenz@500: :param translate: the translation function, for example ``gettext`` or cmlenz@500: ``ugettext``. cmlenz@500: :param ignore_tags: a set of tag names that should not be localized cmlenz@500: :param include_attrs: a set of attribute names should be localized cmlenz@820: :param extract_text: whether the content of text nodes should be cmlenz@820: extracted, or only text in explicit ``gettext`` cmlenz@820: function calls cmlenz@902: cmlenz@820: :note: Changed in 0.6: the `translate` parameter can now be either cmlenz@820: a ``gettext``-style function, or an object compatible with the cmlenz@820: ``NullTransalations`` or ``GNUTranslations`` interface cmlenz@500: """ cmlenz@500: self.translate = translate cmlenz@500: self.ignore_tags = ignore_tags cmlenz@500: self.include_attrs = include_attrs cmlenz@820: self.extract_text = extract_text cmlenz@500: cmlenz@902: def __call__(self, stream, ctxt=None, translate_text=True, cmlenz@902: translate_attrs=True): cmlenz@500: """Translate any localizable strings in the given stream. cmlenz@500: cmlenz@500: This function shouldn't be called directly. Instead, an instance of cmlenz@500: the `Translator` class should be registered as a filter with the cmlenz@500: `Template` or the `TemplateLoader`, or applied as a regular stream cmlenz@500: filter. If used as a template filter, it should be inserted in front of cmlenz@500: all the default filters. cmlenz@500: cmlenz@500: :param stream: the markup event stream cmlenz@500: :param ctxt: the template context (not used) cmlenz@902: :param translate_text: whether text nodes should be translated (used cmlenz@902: internally) cmlenz@902: :param translate_attrs: whether attribute values should be translated cmlenz@902: (used internally) cmlenz@500: :return: the localized stream cmlenz@500: """ cmlenz@500: ignore_tags = self.ignore_tags cmlenz@500: include_attrs = self.include_attrs cmlenz@500: skip = 0 cmlenz@820: xml_lang = XML_NAMESPACE['lang'] cmlenz@902: if not self.extract_text: cmlenz@902: translate_text = False cmlenz@902: translate_attrs = False cmlenz@820: cmlenz@820: if type(self.translate) is FunctionType: cmlenz@820: gettext = self.translate cmlenz@902: if ctxt: cmlenz@902: ctxt['_i18n.gettext'] = gettext cmlenz@820: else: cmlenz@820: gettext = self.translate.ugettext cmlenz@902: ngettext = self.translate.ungettext cmlenz@902: try: cmlenz@902: dgettext = self.translate.dugettext cmlenz@902: dngettext = self.translate.dungettext cmlenz@902: except AttributeError: cmlenz@902: dgettext = lambda _, y: gettext(y) cmlenz@902: dngettext = lambda _, s, p, n: ngettext(s, p, n) cmlenz@902: if ctxt: cmlenz@902: ctxt['_i18n.gettext'] = gettext cmlenz@902: ctxt['_i18n.ngettext'] = ngettext cmlenz@902: ctxt['_i18n.dgettext'] = dgettext cmlenz@902: ctxt['_i18n.dngettext'] = dngettext cmlenz@820: cmlenz@902: if ctxt and ctxt.get('_i18n.domain'): cmlenz@902: gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg) cmlenz@500: cmlenz@500: for kind, data, pos in stream: cmlenz@500: cmlenz@500: # skip chunks that should not be localized cmlenz@500: if skip: cmlenz@500: if kind is START: cmlenz@820: skip += 1 cmlenz@500: elif kind is END: cmlenz@820: skip -= 1 cmlenz@500: yield kind, data, pos cmlenz@500: continue cmlenz@500: cmlenz@500: # handle different events that can be localized cmlenz@500: if kind is START: cmlenz@500: tag, attrs = data cmlenz@820: if tag in self.ignore_tags or \ cmlenz@820: isinstance(attrs.get(xml_lang), basestring): cmlenz@500: skip += 1 cmlenz@500: yield kind, data, pos cmlenz@500: continue cmlenz@500: cmlenz@500: new_attrs = [] cmlenz@500: changed = False cmlenz@902: cmlenz@500: for name, value in attrs: cmlenz@500: newval = value cmlenz@902: if isinstance(value, basestring): cmlenz@902: if translate_attrs and name in include_attrs: cmlenz@820: newval = gettext(value) cmlenz@500: else: cmlenz@902: newval = list( cmlenz@902: self(_ensure(value), ctxt, translate_text=False) cmlenz@500: ) cmlenz@500: if newval != value: cmlenz@500: value = newval cmlenz@500: changed = True cmlenz@500: new_attrs.append((name, value)) cmlenz@500: if changed: cmlenz@820: attrs = Attrs(new_attrs) cmlenz@500: cmlenz@500: yield kind, (tag, attrs), pos cmlenz@500: cmlenz@902: elif translate_text and kind is TEXT: cmlenz@500: text = data.strip() cmlenz@500: if text: cmlenz@820: data = data.replace(text, unicode(gettext(text))) cmlenz@500: yield kind, data, pos cmlenz@500: cmlenz@500: elif kind is SUB: cmlenz@820: directives, substream = data cmlenz@902: current_domain = None cmlenz@902: for idx, directive in enumerate(directives): cmlenz@902: # Organize directives to make everything work cmlenz@902: # FIXME: There's got to be a better way to do this! cmlenz@902: if isinstance(directive, DomainDirective): cmlenz@902: # Grab current domain and update context cmlenz@902: current_domain = directive.domain cmlenz@902: ctxt.push({'_i18n.domain': current_domain}) cmlenz@902: # Put domain directive as the first one in order to cmlenz@902: # update context before any other directives evaluation cmlenz@902: directives.insert(0, directives.pop(idx)) cmlenz@902: cmlenz@902: # If this is an i18n directive, no need to translate text cmlenz@820: # nodes here cmlenz@902: is_i18n_directive = any([ cmlenz@902: isinstance(d, ExtractableI18NDirective) cmlenz@902: for d in directives cmlenz@902: ]) cmlenz@820: substream = list(self(substream, ctxt, cmlenz@902: translate_text=not is_i18n_directive, cmlenz@902: translate_attrs=translate_attrs)) cmlenz@820: yield kind, (directives, substream), pos cmlenz@500: cmlenz@902: if current_domain: cmlenz@902: ctxt.pop() cmlenz@500: else: cmlenz@500: yield kind, data, pos cmlenz@500: cmlenz@500: def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, cmlenz@902: search_text=True, comment_stack=None): cmlenz@500: """Extract localizable strings from the given template stream. cmlenz@500: cmlenz@500: For every string found, this function yields a ``(lineno, function, cmlenz@820: message, comments)`` tuple, where: cmlenz@500: cmlenz@500: * ``lineno`` is the number of the line on which the string was found, cmlenz@500: * ``function`` is the name of the ``gettext`` function used (if the cmlenz@500: string was extracted from embedded Python code), and cmlenz@500: * ``message`` is the string itself (a ``unicode`` object, or a tuple cmlenz@820: of ``unicode`` objects for functions with multiple string cmlenz@820: arguments). cmlenz@820: * ``comments`` is a list of comments related to the message, extracted cmlenz@820: from ``i18n:comment`` attributes found in the markup cmlenz@500: cmlenz@500: >>> tmpl = MarkupTemplate(''' cmlenz@500: ... cmlenz@500: ...${_("Hello, %(name)s") % dict(name=username)}
cmlenz@500: ...${ngettext("You have %d item", "You have %d items", num)}
cmlenz@500: ... cmlenz@500: ... ''', filename='example.html') cmlenz@820: >>> for line, func, msg, comments in Translator().extract(tmpl.stream): cmlenz@902: ... print('%d, %r, %r' % (line, func, msg)) cmlenz@500: 3, None, u'Example' cmlenz@500: 6, None, u'Example' cmlenz@500: 7, '_', u'Hello, %(name)s' cmlenz@820: 8, 'ngettext', (u'You have %d item', u'You have %d items', None) cmlenz@500: cmlenz@500: :param stream: the event stream to extract strings from; can be a cmlenz@500: regular stream or a template stream cmlenz@500: :param gettext_functions: a sequence of function names that should be cmlenz@500: treated as gettext-style localization cmlenz@500: functions cmlenz@500: :param search_text: whether the content of text nodes should be cmlenz@500: extracted (used internally) cmlenz@500: cmlenz@500: :note: Changed in 0.4.1: For a function with multiple string arguments cmlenz@500: (such as ``ngettext``), a single item with a tuple of strings is cmlenz@500: yielded, instead an item for each string argument. cmlenz@902: :note: Changed in 0.6: The returned tuples now include a fourth cmlenz@902: element, which is a list of comments for the translator. cmlenz@500: """ cmlenz@820: if not self.extract_text: cmlenz@820: search_text = False cmlenz@902: if comment_stack is None: cmlenz@902: comment_stack = [] cmlenz@500: skip = 0 cmlenz@902: cmlenz@820: xml_lang = XML_NAMESPACE['lang'] cmlenz@500: cmlenz@500: for kind, data, pos in stream: cmlenz@500: if skip: cmlenz@500: if kind is START: cmlenz@820: skip += 1 cmlenz@500: if kind is END: cmlenz@820: skip -= 1 cmlenz@500: cmlenz@820: if kind is START and not skip: cmlenz@500: tag, attrs = data cmlenz@820: if tag in self.ignore_tags or \ cmlenz@820: isinstance(attrs.get(xml_lang), basestring): cmlenz@500: skip += 1 cmlenz@500: continue cmlenz@500: cmlenz@902: for message in self._extract_attrs((kind, data, pos), cmlenz@902: gettext_functions, cmlenz@902: search_text=search_text): cmlenz@902: yield message cmlenz@820: cmlenz@820: elif not skip and search_text and kind is TEXT: cmlenz@902: text = data.strip() cmlenz@902: if text and [ch for ch in text if ch.isalpha()]: cmlenz@902: yield pos[1], None, text, comment_stack[-1:] cmlenz@500: cmlenz@500: elif kind is EXPR or kind is EXEC: cmlenz@820: for funcname, strings in extract_from_code(data, cmlenz@820: gettext_functions): cmlenz@902: # XXX: Do we need to grab i18n:comment from comment_stack ??? cmlenz@820: yield pos[1], funcname, strings, [] cmlenz@500: cmlenz@500: elif kind is SUB: cmlenz@902: directives, substream = data cmlenz@902: in_comment = False cmlenz@902: cmlenz@902: for idx, directive in enumerate(directives): cmlenz@902: # Do a first loop to see if there's a comment directive cmlenz@902: # If there is update context and pop it from directives cmlenz@902: if isinstance(directive, CommentDirective): cmlenz@902: in_comment = True cmlenz@902: comment_stack.append(directive.comment) cmlenz@902: if len(directives) == 1: cmlenz@902: # in case we're in the presence of something like: cmlenz@902: #Foo
cmlenz@902: for message in self.extract( cmlenz@902: substream, gettext_functions, cmlenz@902: search_text=search_text and not skip, cmlenz@902: comment_stack=comment_stack): cmlenz@902: yield message cmlenz@902: directives.pop(idx) cmlenz@902: elif not isinstance(directive, I18NDirective): cmlenz@902: # Remove all other non i18n directives from the process cmlenz@902: directives.pop(idx) cmlenz@902: cmlenz@902: if not directives and not in_comment: cmlenz@902: # Extract content if there's no directives because cmlenz@902: # strip was pop'ed and not because comment was pop'ed. cmlenz@902: # Extraction in this case has been taken care of. cmlenz@902: for message in self.extract( cmlenz@902: substream, gettext_functions, cmlenz@902: search_text=search_text and not skip): cmlenz@902: yield message cmlenz@902: cmlenz@902: for directive in directives: cmlenz@902: if isinstance(directive, ExtractableI18NDirective): cmlenz@902: for message in directive.extract(self, cmlenz@902: substream, gettext_functions, cmlenz@902: search_text=search_text and not skip, cmlenz@902: comment_stack=comment_stack): cmlenz@902: yield message cmlenz@902: else: cmlenz@902: for message in self.extract( cmlenz@902: substream, gettext_functions, cmlenz@902: search_text=search_text and not skip, cmlenz@902: comment_stack=comment_stack): cmlenz@902: yield message cmlenz@902: cmlenz@902: if in_comment: cmlenz@902: comment_stack.pop() cmlenz@902: cmlenz@902: def get_directive_index(self, dir_cls): cmlenz@902: total = len(self._dir_order) cmlenz@902: if dir_cls in self._dir_order: cmlenz@902: return self._dir_order.index(dir_cls) - total cmlenz@902: return total cmlenz@902: cmlenz@902: def setup(self, template): cmlenz@902: """Convenience function to register the `Translator` filter and the cmlenz@902: related directives with the given template. cmlenz@902: cmlenz@902: :param template: a `Template` instance cmlenz@902: """ cmlenz@902: template.filters.insert(0, self) cmlenz@902: if hasattr(template, 'add_directives'): cmlenz@902: template.add_directives(Translator.NAMESPACE, self) cmlenz@902: cmlenz@902: def _extract_attrs(self, event, gettext_functions, search_text): cmlenz@902: for name, value in event[1][1]: cmlenz@902: if search_text and isinstance(value, basestring): cmlenz@902: if name in self.include_attrs: cmlenz@902: text = value.strip() cmlenz@902: if text: cmlenz@902: yield event[2][1], None, text, [] cmlenz@902: else: cmlenz@902: for message in self.extract(_ensure(value), gettext_functions, cmlenz@902: search_text=False): cmlenz@902: yield message cmlenz@820: cmlenz@820: cmlenz@820: class MessageBuffer(object): cmlenz@820: """Helper class for managing internationalized mixed content. cmlenz@820: cmlenz@820: :since: version 0.5 cmlenz@820: """ cmlenz@820: cmlenz@902: def __init__(self, directive=None): cmlenz@820: """Initialize the message buffer. cmlenz@820: cmlenz@902: :param directive: the directive owning the buffer cmlenz@902: :type directive: I18NDirective cmlenz@820: """ cmlenz@902: # params list needs to be copied so that directives can be evaluated cmlenz@902: # more than once cmlenz@902: self.orig_params = self.params = directive.params[:] cmlenz@902: self.directive = directive cmlenz@820: self.string = [] cmlenz@820: self.events = {} cmlenz@820: self.values = {} cmlenz@820: self.depth = 1 cmlenz@820: self.order = 1 cmlenz@820: self.stack = [0] cmlenz@902: self.subdirectives = {} cmlenz@820: cmlenz@820: def append(self, kind, data, pos): cmlenz@820: """Append a stream event to the buffer. cmlenz@820: cmlenz@820: :param kind: the stream event kind cmlenz@820: :param data: the event data cmlenz@820: :param pos: the position of the event in the source cmlenz@820: """ cmlenz@902: if kind is SUB: cmlenz@902: # The order needs to be +1 because a new START kind event will cmlenz@902: # happen and we we need to wrap those events into our custom kind(s) cmlenz@902: order = self.stack[-1] + 1 cmlenz@902: subdirectives, substream = data cmlenz@902: # Store the directives that should be applied after translation cmlenz@902: self.subdirectives.setdefault(order, []).extend(subdirectives) cmlenz@902: self.events.setdefault(order, []).append((SUB_START, None, pos)) cmlenz@902: for skind, sdata, spos in substream: cmlenz@902: self.append(skind, sdata, spos) cmlenz@902: self.events.setdefault(order, []).append((SUB_END, None, pos)) cmlenz@902: elif kind is TEXT: cmlenz@902: if '[' in data or ']' in data: cmlenz@902: # Quote [ and ] if it ain't us adding it, ie, if the user is cmlenz@902: # using those chars in his templates, escape them cmlenz@902: data = data.replace('[', '\[').replace(']', '\]') cmlenz@820: self.string.append(data) cmlenz@902: self.events.setdefault(self.stack[-1], []).append((kind, data, pos)) cmlenz@820: elif kind is EXPR: cmlenz@902: if self.params: cmlenz@902: param = self.params.pop(0) cmlenz@902: else: cmlenz@902: params = ', '.join(['"%s"' % p for p in self.orig_params if p]) cmlenz@902: if params: cmlenz@902: params = "(%s)" % params cmlenz@902: raise IndexError("%d parameters%s given to 'i18n:%s' but " cmlenz@902: "%d or more expressions used in '%s', line %s" cmlenz@902: % (len(self.orig_params), params, cmlenz@902: self.directive.tagname, cmlenz@902: len(self.orig_params) + 1, cmlenz@902: os.path.basename(pos[0] or cmlenz@902: 'In-memory Template'), cmlenz@902: pos[1])) cmlenz@820: self.string.append('%%(%s)s' % param) cmlenz@902: self.events.setdefault(self.stack[-1], []).append((kind, data, pos)) cmlenz@820: self.values[param] = (kind, data, pos) cmlenz@820: else: cmlenz@902: if kind is START: cmlenz@902: self.string.append('[%d:' % self.order) cmlenz@820: self.stack.append(self.order) cmlenz@902: self.events.setdefault(self.stack[-1], cmlenz@902: []).append((kind, data, pos)) cmlenz@820: self.depth += 1 cmlenz@820: self.order += 1 cmlenz@820: elif kind is END: cmlenz@820: self.depth -= 1 cmlenz@820: if self.depth: cmlenz@820: self.events[self.stack[-1]].append((kind, data, pos)) cmlenz@902: self.string.append(']') cmlenz@820: self.stack.pop() cmlenz@820: cmlenz@820: def format(self): cmlenz@820: """Return a message identifier representing the content in the cmlenz@820: buffer. cmlenz@820: """ cmlenz@902: return ''.join(self.string).strip() cmlenz@820: cmlenz@820: def translate(self, string, regex=re.compile(r'%\((\w+)\)s')): cmlenz@820: """Interpolate the given message translation with the events in the cmlenz@820: buffer and return the translated stream. cmlenz@820: cmlenz@820: :param string: the translated message string cmlenz@820: """ cmlenz@902: substream = None cmlenz@902: cmlenz@902: def yield_parts(string): cmlenz@902: for idx, part in enumerate(regex.split(string)): cmlenz@902: if idx % 2: cmlenz@902: yield self.values[part] cmlenz@902: elif part: cmlenz@902: yield (TEXT, cmlenz@902: part.replace('\[', '[').replace('\]', ']'), cmlenz@902: (None, -1, -1) cmlenz@902: ) cmlenz@902: cmlenz@820: parts = parse_msg(string) cmlenz@902: parts_counter = {} cmlenz@820: for order, string in parts: cmlenz@902: parts_counter.setdefault(order, []).append(None) cmlenz@902: cmlenz@902: while parts: cmlenz@902: order, string = parts.pop(0) cmlenz@902: if len(parts_counter[order]) == 1: cmlenz@902: events = self.events[order] cmlenz@902: else: cmlenz@902: events = [self.events[order].pop(0)] cmlenz@902: parts_counter[order].pop() cmlenz@902: cmlenz@902: for event in events: cmlenz@902: if event[0] is SUB_START: cmlenz@902: substream = [] cmlenz@902: elif event[0] is SUB_END: cmlenz@902: # Yield a substream which might have directives to be cmlenz@902: # applied to it (after translation events) cmlenz@902: yield SUB, (self.subdirectives[order], substream), event[2] cmlenz@902: substream = None cmlenz@902: elif event[0] is TEXT: cmlenz@902: if string: cmlenz@902: for part in yield_parts(string): cmlenz@902: if substream is not None: cmlenz@902: substream.append(part) cmlenz@902: else: cmlenz@902: yield part cmlenz@902: # String handled, reset it cmlenz@902: string = None cmlenz@902: elif event[0] is START: cmlenz@902: if substream is not None: cmlenz@902: substream.append(event) cmlenz@902: else: cmlenz@902: yield event cmlenz@902: if string: cmlenz@902: for part in yield_parts(string): cmlenz@902: if substream is not None: cmlenz@902: substream.append(part) cmlenz@902: else: cmlenz@902: yield part cmlenz@902: # String handled, reset it cmlenz@902: string = None cmlenz@902: elif event[0] is END: cmlenz@902: if string: cmlenz@902: for part in yield_parts(string): cmlenz@902: if substream is not None: cmlenz@902: substream.append(part) cmlenz@902: else: cmlenz@902: yield part cmlenz@902: # String handled, reset it cmlenz@902: string = None cmlenz@902: if substream is not None: cmlenz@902: substream.append(event) cmlenz@902: else: cmlenz@902: yield event cmlenz@902: elif event[0] is EXPR: cmlenz@902: # These are handled on the strings itself cmlenz@902: continue cmlenz@820: else: cmlenz@902: if string: cmlenz@902: for part in yield_parts(string): cmlenz@902: if substream is not None: cmlenz@902: substream.append(part) cmlenz@902: else: cmlenz@902: yield part cmlenz@902: # String handled, reset it cmlenz@902: string = None cmlenz@902: if substream is not None: cmlenz@902: substream.append(event) cmlenz@902: else: cmlenz@902: yield event cmlenz@820: cmlenz@820: cmlenz@902: def parse_msg(string, regex=re.compile(r'(?:\[(\d+)\:)|(?>> parse_msg("See [1:Help].") cmlenz@820: [(0, 'See '), (1, 'Help'), (0, '.')] cmlenz@902: cmlenz@820: >>> parse_msg("See [1:our [2:Help] page] for details.") cmlenz@820: [(0, 'See '), (1, 'our '), (2, 'Help'), (1, ' page'), (0, ' for details.')] cmlenz@902: cmlenz@820: >>> parse_msg("[2:Details] finden Sie in [1:Hilfe].") cmlenz@820: [(2, 'Details'), (0, ' finden Sie in '), (1, 'Hilfe'), (0, '.')] cmlenz@902: cmlenz@820: >>> parse_msg("[1:] Bilder pro Seite anzeigen.") cmlenz@820: [(1, ''), (0, ' Bilder pro Seite anzeigen.')] cmlenz@902: cmlenz@820: :param string: the translated message string cmlenz@820: :return: a list of ``(order, string)`` tuples cmlenz@820: :rtype: `list` cmlenz@820: """ cmlenz@820: parts = [] cmlenz@820: stack = [0] cmlenz@820: while True: cmlenz@820: mo = regex.search(string) cmlenz@820: if not mo: cmlenz@820: break cmlenz@820: cmlenz@820: if mo.start() or stack[-1]: cmlenz@820: parts.append((stack[-1], string[:mo.start()])) cmlenz@820: string = string[mo.end():] cmlenz@820: cmlenz@820: orderno = mo.group(1) cmlenz@820: if orderno is not None: cmlenz@820: stack.append(int(orderno)) cmlenz@820: else: cmlenz@820: stack.pop() cmlenz@820: if not stack: cmlenz@820: break cmlenz@820: cmlenz@820: if string: cmlenz@820: parts.append((stack[-1], string)) cmlenz@820: cmlenz@820: return parts cmlenz@820: cmlenz@820: cmlenz@820: def extract_from_code(code, gettext_functions): cmlenz@820: """Extract strings from Python bytecode. cmlenz@820: cmlenz@820: >>> from genshi.template.eval import Expression cmlenz@820: >>> expr = Expression('_("Hello")') cmlenz@902: >>> list(extract_from_code(expr, GETTEXT_FUNCTIONS)) cmlenz@820: [('_', u'Hello')] cmlenz@902: cmlenz@820: >>> expr = Expression('ngettext("You have %(num)s item", ' cmlenz@820: ... '"You have %(num)s items", num)') cmlenz@902: >>> list(extract_from_code(expr, GETTEXT_FUNCTIONS)) cmlenz@820: [('ngettext', (u'You have %(num)s item', u'You have %(num)s items', None))] cmlenz@820: cmlenz@820: :param code: the `Code` object cmlenz@820: :type code: `genshi.template.eval.Code` cmlenz@820: :param gettext_functions: a sequence of function names cmlenz@820: :since: version 0.5 cmlenz@820: """ cmlenz@820: def _walk(node): cmlenz@820: if isinstance(node, _ast.Call) and isinstance(node.func, _ast.Name) \ cmlenz@820: and node.func.id in gettext_functions: cmlenz@820: strings = [] cmlenz@820: def _add(arg): cmlenz@820: if isinstance(arg, _ast.Str) and isinstance(arg.s, basestring): cmlenz@820: strings.append(unicode(arg.s, 'utf-8')) cmlenz@820: elif arg: cmlenz@820: strings.append(None) cmlenz@820: [_add(arg) for arg in node.args] cmlenz@820: _add(node.starargs) cmlenz@820: _add(node.kwargs) cmlenz@820: if len(strings) == 1: cmlenz@820: strings = strings[0] cmlenz@820: else: cmlenz@820: strings = tuple(strings) cmlenz@820: yield node.func.id, strings cmlenz@820: elif node._fields: cmlenz@820: children = [] cmlenz@820: for field in node._fields: cmlenz@820: child = getattr(node, field, None) cmlenz@820: if isinstance(child, list): cmlenz@820: for elem in child: cmlenz@820: children.append(elem) cmlenz@820: elif isinstance(child, _ast.AST): cmlenz@820: children.append(child) cmlenz@820: for child in children: cmlenz@820: for funcname, strings in _walk(child): cmlenz@820: yield funcname, strings cmlenz@820: return _walk(code.ast) cmlenz@820: cmlenz@820: cmlenz@820: def extract(fileobj, keywords, comment_tags, options): cmlenz@820: """Babel extraction method for Genshi templates. cmlenz@820: cmlenz@820: :param fileobj: the file-like object the messages should be extracted from cmlenz@820: :param keywords: a list of keywords (i.e. function names) that should be cmlenz@820: recognized as translation functions cmlenz@820: :param comment_tags: a list of translator tags to search for and include cmlenz@820: in the results cmlenz@820: :param options: a dictionary of additional options (optional) cmlenz@820: :return: an iterator over ``(lineno, funcname, message, comments)`` tuples cmlenz@820: :rtype: ``iterator`` cmlenz@820: """ cmlenz@820: template_class = options.get('template_class', MarkupTemplate) cmlenz@820: if isinstance(template_class, basestring): cmlenz@820: module, clsname = template_class.split(':', 1) cmlenz@820: template_class = getattr(__import__(module, {}, {}, [clsname]), clsname) cmlenz@820: encoding = options.get('encoding', None) cmlenz@820: cmlenz@820: extract_text = options.get('extract_text', True) cmlenz@820: if isinstance(extract_text, basestring): cmlenz@820: extract_text = extract_text.lower() in ('1', 'on', 'yes', 'true') cmlenz@820: cmlenz@820: ignore_tags = options.get('ignore_tags', Translator.IGNORE_TAGS) cmlenz@820: if isinstance(ignore_tags, basestring): cmlenz@820: ignore_tags = ignore_tags.split() cmlenz@820: ignore_tags = [QName(tag) for tag in ignore_tags] cmlenz@820: cmlenz@820: include_attrs = options.get('include_attrs', Translator.INCLUDE_ATTRS) cmlenz@820: if isinstance(include_attrs, basestring): cmlenz@820: include_attrs = include_attrs.split() cmlenz@820: include_attrs = [QName(attr) for attr in include_attrs] cmlenz@820: cmlenz@820: tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None), cmlenz@820: encoding=encoding) cmlenz@902: tmpl.loader = None cmlenz@902: cmlenz@820: translator = Translator(None, ignore_tags, include_attrs, extract_text) cmlenz@902: if hasattr(tmpl, 'add_directives'): cmlenz@902: tmpl.add_directives(Translator.NAMESPACE, translator) cmlenz@820: for message in translator.extract(tmpl.stream, gettext_functions=keywords): cmlenz@820: yield message