Mercurial > babel > old > mirror
changeset 82:f421e5576d26
Added support for translator comments at the API and frontends levels.(See #12, item 1). Updated docs and tests accordingly.
| author | palgarvio |
|---|---|
| date | Sun, 10 Jun 2007 14:21:01 +0000 |
| parents | 51f73a110a84 |
| children | 80e895a498a3 |
| files | babel/messages/catalog.py babel/messages/extract.py babel/messages/frontend.py babel/messages/pofile.py babel/messages/tests/catalog.py babel/messages/tests/extract.py babel/messages/tests/pofile.py doc/cmdline.txt doc/setup.txt |
| diffstat | 9 files changed, 118 insertions(+), 34 deletions(-) [+] |
line wrap: on
line diff
--- a/babel/messages/catalog.py +++ b/babel/messages/catalog.py @@ -35,7 +35,7 @@ class Message(object): """Representation of a single message in a catalog.""" - def __init__(self, id, string='', locations=(), flags=()): + def __init__(self, id, string='', locations=(), flags=(), comments=[]): """Create the message object. :param id: the message ID, or a ``(singular, plural)`` tuple for @@ -44,6 +44,7 @@ ``(singular, plural)`` tuple for pluralizable messages :param locations: a sequence of ``(filenname, lineno)`` tuples :param flags: a set or sequence of flags + :param comments: a list of comments for the msgid """ self.id = id if not string and self.pluralizable: @@ -55,6 +56,7 @@ self.flags.add('python-format') else: self.flags.discard('python-format') + self.comments = comments def __repr__(self): return '<%s %r>' % (type(self).__name__, self.id) @@ -328,7 +330,7 @@ assert isinstance(message.string, (list, tuple)) self._messages[key] = message - def add(self, id, string=None, locations=(), flags=()): + def add(self, id, string=None, locations=(), flags=(), comments=[]): """Add or update the message with the specified ID. >>> catalog = Catalog() @@ -345,8 +347,9 @@ ``(singular, plural)`` tuple for pluralizable messages :param locations: a sequence of ``(filenname, lineno)`` tuples :param flags: a set or sequence of flags + :param comments: a list of comments for the msgid """ - self[id] = Message(id, string, list(locations), flags) + self[id] = Message(id, string, list(locations), flags, comments) def _key_for(self, id): """The key for a message is just the singular ID even for pluralizable
--- a/babel/messages/extract.py +++ b/babel/messages/extract.py @@ -27,7 +27,7 @@ except NameError: from sets import Set as set import sys -from tokenize import generate_tokens, NAME, OP, STRING +from tokenize import generate_tokens, NAME, OP, STRING, COMMENT from babel.util import pathmatch, relpath @@ -50,7 +50,7 @@ def extract_from_dir(dirname=os.getcwd(), method_map=DEFAULT_MAPPING, options_map=None, keywords=DEFAULT_KEYWORDS, - callback=None): + comments_tags=[], callback=None): """Extract messages from any source files found in the given directory. This function generates tuples of the form: @@ -137,14 +137,16 @@ options = odict if callback: callback(filename, method, options) - for lineno, message in extract_from_file(method, filepath, - keywords=keywords, - options=options): - yield filename, lineno, message + for lineno, message, comments in \ + extract_from_file(method, filepath, + keywords=keywords, + comments_tags=comments_tags, + options=options): + yield filename, lineno, message, comments break def extract_from_file(method, filename, keywords=DEFAULT_KEYWORDS, - options=None): + comments_tags=[], options=None): """Extract messages from a specific file. This function returns a list of tuples of the form: @@ -163,17 +165,19 @@ """ fileobj = open(filename, 'U') try: - return list(extract(method, fileobj, keywords, options=options)) + return list(extract(method, fileobj, keywords, + comments_tags=comments_tags, options=options)) finally: fileobj.close() -def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, options=None): +def extract(method, fileobj, keywords=DEFAULT_KEYWORDS, comments_tags=[], + options=None): """Extract messages from the given file-like object using the specified extraction method. This function returns a list of tuples of the form: - ``(lineno, message)`` + ``(lineno, message, comments)`` The implementation dispatches the actual extraction to plugins, based on the value of the ``method`` parameter. @@ -186,7 +190,7 @@ >>> from StringIO import StringIO >>> for message in extract('python', StringIO(source)): ... print message - (3, 'Hello, world!') + (3, 'Hello, world!', []) :param method: a string specifying the extraction method (.e.g. "python") :param fileobj: the file-like object the messages should be extracted from @@ -194,6 +198,8 @@ that should be recognized as translation functions) to tuples that specify which of their arguments contain localizable strings + :param comments_tags: a list of translator tags to search for and include in + output :param options: a dictionary of additional options (optional) :return: the list of extracted messages :rtype: `list` @@ -204,8 +210,11 @@ for entry_point in working_set.iter_entry_points(GROUP_NAME, method): func = entry_point.load(require=True) m = [] - for lineno, funcname, messages in func(fileobj, keywords.keys(), - options=options or {}): + for lineno, funcname, messages, comments in \ + func(fileobj, + keywords.keys(), + comments_tags=comments_tags, + options=options or {}): if isinstance(messages, (list, tuple)): msgs = [] for index in keywords[funcname]: @@ -213,18 +222,18 @@ messages = tuple(msgs) if len(messages) == 1: messages = messages[0] - yield lineno, messages + yield lineno, messages, comments return raise ValueError('Unknown extraction method %r' % method) -def extract_nothing(fileobj, keywords, options): +def extract_nothing(fileobj, keywords, comments_tags, options): """Pseudo extractor that does not actually extract anything, but simply returns an empty list. """ return [] -def extract_genshi(fileobj, keywords, options): +def extract_genshi(fileobj, keywords, comments_tags, options): """Extract messages from Genshi templates. :param fileobj: the file-like object the messages should be extracted from @@ -253,10 +262,11 @@ tmpl = template_class(fileobj, filename=getattr(fileobj, 'name'), encoding=encoding) translator = Translator(None, ignore_tags, include_attrs) - for message in translator.extract(tmpl.stream, gettext_functions=keywords): - yield message + for lineno, func, message in translator.extract(tmpl.stream, + gettext_functions=keywords): + yield lineno, func, message, [] -def extract_python(fileobj, keywords, options): +def extract_python(fileobj, keywords, comments_tags, options): """Extract messages from Python source code. :param fileobj: the file-like object the messages should be extracted from @@ -270,15 +280,26 @@ lineno = None buf = [] messages = [] + translator_comments = [] in_args = False + in_translator_comments = False tokens = generate_tokens(fileobj.readline) for tok, value, (lineno, _), _, _ in tokens: if funcname and tok == OP and value == '(': in_args = True + elif tok == COMMENT: + if in_translator_comments is True: + translator_comments.append(value[1:].strip()) + continue + for comments_tag in comments_tags: + if comments_tag in value: + if in_translator_comments is not True: + in_translator_comments = True + translator_comments.append(value[1:].strip()) elif funcname and in_args: if tok == OP and value == ')': - in_args = False + in_args = in_translator_comments = False if buf: messages.append(''.join(buf)) del buf[:] @@ -287,9 +308,10 @@ messages = tuple(messages) else: messages = messages[0] - yield lineno, funcname, messages + yield lineno, funcname, messages, translator_comments funcname = lineno = None messages = [] + translator_comments = [] elif tok == STRING: if lineno is None: lineno = stup[0]
--- a/babel/messages/frontend.py +++ b/babel/messages/frontend.py @@ -87,6 +87,9 @@ 'set report address for msgid'), ('copyright-holder=', None, 'set copyright holder in output'), + ('add-comments=', 'c', + 'place comment block with TAG (or those preceding keyword lines) in ' + 'output file. Seperate multiple TAGs with commas(,)'), ('input-dirs=', None, 'directories that should be scanned for messages'), ] @@ -110,6 +113,8 @@ self.sort_by_file = False self.msgid_bugs_address = None self.copyright_holder = None + self.add_comments = None + self._add_comments = None def finalize_options(self): if self.no_default_keywords and not self.keywords: @@ -137,6 +142,9 @@ self.input_dirs = dict.fromkeys([k.split('.',1)[0] for k in self.distribution.packages ]).keys() + + if self.add_comments: + self._add_comments = self.add_comments.split(',') def run(self): mappings = self._get_mappings() @@ -157,10 +165,12 @@ extracted = extract_from_dir(dirname, method_map, options_map, keywords=self.keywords, + comments_tags=self._add_comments, callback=callback) - for filename, lineno, message in extracted: + for filename, lineno, message, comments in extracted: filepath = os.path.normpath(os.path.join(dirname, filename)) - catalog.add(message, None, [(filepath, lineno)]) + catalog.add(message, None, [(filepath, lineno)], + comments=comments) log.info('writing PO template file to %s' % self.output_file) write_pot(outfile, catalog, project=self.distribution.get_name(), @@ -414,11 +424,17 @@ help='set report address for msgid') parser.add_option('--copyright-holder', dest='copyright_holder', help='set copyright holder in output') + parser.add_option('--add-comments', '-c', dest='add_comments', + metavar='TAG', action='append', + help='place comment block with TAG (or those ' + 'preceding keyword lines) in output file. One ' + 'TAG per argument call') parser.set_defaults(charset='utf-8', keywords=[], no_default_keywords=False, no_location=False, omit_header = False, width=76, no_wrap=False, - sort_output=False, sort_by_file=False) + sort_output=False, sort_by_file=False, + add_comments=[]) options, args = parser.parse_args(argv) if not args: parser.error('incorrect number of arguments') @@ -463,11 +479,13 @@ for dirname in args: if not os.path.isdir(dirname): parser.error('%r is not a directory' % dirname) - extracted = extract_from_dir(dirname, method_map, options_map, - keywords) - for filename, lineno, message in extracted: + extracted = extract_from_dir(dirname, method_map, + options_map, keywords, + comments=options.comments) + for filename, lineno, message, comments in extracted: filepath = os.path.normpath(os.path.join(dirname, filename)) - catalog.add(message, None, [(filepath, lineno)]) + catalog.add(message, None, [(filepath, lineno)], + comments=comments) write_pot(outfile, catalog, width=options.width, charset=options.charset, no_location=options.no_location,
--- a/babel/messages/pofile.py +++ b/babel/messages/pofile.py @@ -234,7 +234,6 @@ >>> from StringIO import StringIO >>> buf = StringIO() >>> write_pot(buf, catalog, omit_header=True) - >>> print buf.getvalue() #: main.py:1 #, fuzzy, python-format @@ -293,6 +292,12 @@ 'project': project, 'copyright_holder': _copyright_holder, }) + + if message.comments: + for comment in message.comments: + for line in textwrap.wrap(comment, + width, break_long_words=False): + _write('#. %s\n' % line.strip()) if not no_location: locs = u' '.join([u'%s:%d' % item for item in message.locations])
--- a/babel/messages/tests/catalog.py +++ b/babel/messages/tests/catalog.py @@ -23,7 +23,16 @@ def test_python_format(self): assert catalog.PYTHON_FORMAT('foo %d bar') assert catalog.PYTHON_FORMAT('foo %s bar') - assert catalog.PYTHON_FORMAT('foo %r bar') + assert catalog.PYTHON_FORMAT('foo %r bar') + + def test_translator_comments(self): + mess = catalog.Message('foo', comments=['Comment About `foo`']) + self.assertEqual(mess.comments, ['Comment About `foo`']) + mess = catalog.Message('foo', + comments=['Comment 1 About `foo`', + 'Comment 2 About `foo`']) + self.assertEqual(mess.comments, ['Comment 1 About `foo`', + 'Comment 2 About `foo`']) class CatalogTestCase(unittest.TestCase):
--- a/babel/messages/tests/extract.py +++ b/babel/messages/tests/extract.py @@ -22,7 +22,7 @@ def test_unicode_string_arg(self): buf = StringIO("msg = _(u'Foo Bar')") - messages = list(extract.extract_python(buf, ('_',), {})) + messages = list(extract.extract_python(buf, ('_',), {}, [])) self.assertEqual('Foo Bar', messages[0][2])
--- a/babel/messages/tests/pofile.py +++ b/babel/messages/tests/pofile.py @@ -67,6 +67,26 @@ " throw us into an infinite " "loop\n" msgstr ""''', buf.getvalue().strip()) + + def test_pot_with_translator_comments(self): + catalog = Catalog() + catalog.add(u'foo', locations=[('main.py', 1)], + comments=['Comment About `foo`']) + catalog.add(u'bar', locations=[('utils.py', 3)], + comments=['Comment About `bar` with', + 'multiple lines.']) + buf = StringIO() + pofile.write_pot(buf, catalog, omit_header=True) + self.assertEqual('''#. Comment About `foo` +#: main.py:1 +msgid "foo" +msgstr "" + +#. Comment About `bar` with +#. multiple lines. +#: utils.py:3 +msgid "bar" +msgstr ""''', buf.getvalue().strip()) def suite():
--- a/doc/cmdline.txt +++ b/doc/cmdline.txt @@ -67,6 +67,10 @@ set report address for msgid --copyright-holder=COPYRIGHT_HOLDER set copyright holder in output + -c TAG, --add-comments=TAG + place comment block with TAG (or those preceding + keyword lines) in output file. One TAG per argument + call init
--- a/doc/setup.txt +++ b/doc/setup.txt @@ -62,6 +62,9 @@ --sort-by-file sort output by file location (default False) --msgid-bugs-address set report address for msgid --copyright-holder set copyright holder in output + --add-comments (-c) place comment block with TAG (or those preceding + keyword lines) in output file. Seperate multiple TAGs + with commas(,) --input-dirs directories that should be scanned for messages usage: setup.py [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...]