# HG changeset patch # User cmlenz # Date 1182436710 0 # Node ID 80e51aabc4409e11324f666fff6bca0613e34d4e # Parent 17dd31f104f59b70960b29e3bcdb296618d66c86 Add MO file generation. Closes #21. diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Version 0.9 +http://svn.edgewall.org/repos/babel/tags/0.9.0/ +(?, from branches/stable/0.9.x) + + * Added compilation of message catalogs to MO files. + Version 0.8.1 http://svn.edgewall.org/repos/babel/tags/0.8.1/ (?, from branches/stable/0.8.x) diff --git a/babel/messages/frontend.py b/babel/messages/frontend.py --- a/babel/messages/frontend.py +++ b/babel/messages/frontend.py @@ -31,6 +31,7 @@ from babel.messages.catalog import Catalog from babel.messages.extract import extract_from_dir, DEFAULT_KEYWORDS, \ DEFAULT_MAPPING +from babel.messages.mofile import write_mo from babel.messages.pofile import read_po, write_po from babel.messages.plurals import PLURALS from babel.util import odict, LOCALTZ @@ -40,6 +41,90 @@ __docformat__ = 'restructuredtext en' +class compile_catalog(Command): + """Catalog compilation command for use in ``setup.py`` scripts. + + If correctly installed, this command is available to Setuptools-using + setup scripts automatically. For projects using plain old ``distutils``, + the command needs to be registered explicitly in ``setup.py``:: + + from babel.messages.frontend import compile_catalog + + setup( + ... + cmdclass = {'new_catalog': compile_catalog} + ) + + :see: `Integrating new distutils commands `_ + :see: `setuptools `_ + """ + + description = 'compile a catalog to a binary MO file' + user_options = [ + ('domain=', 'D', + "domain of PO file (default 'messages')"), + ('directory=', 'd', + 'path to base directory containing the catalogs'), + ('input-file=', 'i', + 'name of the input file'), + ('output-file=', 'o', + "name of the output file (default " + "'//LC_MESSAGES/.po')"), + ('locale=', 'l', + 'locale of the catalog to compile'), + ('use-fuzzy', 'f', + 'also include fuzzy translations'), + ] + boolean_options = ['use-fuzzy'] + + def initialize_options(self): + self.domain = 'messages' + self.directory = None + self.input_file = None + self.output_file = None + self.locale = None + self.use_fuzzy = False + + def finalize_options(self): + if not self.locale: + raise DistutilsOptionError('you must specify the locale for the ' + 'catalog to compile') + try: + self._locale = Locale.parse(self.locale) + except UnknownLocaleError, e: + raise DistutilsOptionError(e) + + if not self.directory and not self.input_file: + raise DistutilsOptionError('you must specify the input file') + if not self.input_file: + self.input_file = os.path.join(self.directory, self.locale, + 'LC_MESSAGES', self.domain + '.po') + + if not self.directory and not self.output_file: + raise DistutilsOptionError('you must specify the output file') + if not self.output_file: + self.output_file = os.path.join(self.directory, self.locale, + 'LC_MESSAGES', self.domain + '.mo') + + if not os.path.exists(os.path.dirname(self.output_file)): + os.makedirs(os.path.dirname(self.output_file)) + + def run(self): + log.info('compiling catalog to %s', self.output_file) + + infile = open(self.input_file, 'r') + try: + catalog = read_po(infile) + finally: + infile.close() + + outfile = open(self.output_file, 'w') + try: + write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy) + finally: + outfile.close() + + class extract_messages(Command): """Message extraction command for use in ``setup.py`` scripts. @@ -326,8 +411,9 @@ usage = '%%prog %s [options] %s' version = '%%prog %s' % VERSION - commands = ['extract', 'init'] + commands = ['compile', 'extract', 'init'] command_descriptions = { + 'compile': 'compile a message catalog to a MO file', 'extract': 'extract messages from source files and generate a POT file', 'init': 'create new message catalogs from a template' } @@ -360,6 +446,72 @@ for command in self.commands: print format % (command, self.command_descriptions[command]) + def compile(self, argv): + """Subcommand for compiling a message catalog to a MO file. + + :param argv: the command arguments + """ + parser = OptionParser(usage=self.usage % ('init',''), + description=self.command_descriptions['init']) + parser.add_option('--domain', '-D', dest='domain', + help="domain of MO and PO files (default '%default')") + parser.add_option('--directory', '-d', dest='directory', + metavar='DIR', help='base directory of catalog files') + parser.add_option('--input-file', '-i', dest='input_file', + metavar='FILE', help='name of the input file') + parser.add_option('--output-file', '-o', dest='output_file', + metavar='FILE', + help="name of the output file (default " + "'//LC_MESSAGES/" + ".mo')") + parser.add_option('--locale', '-l', dest='locale', metavar='LOCALE', + help='locale of the catalog') + parser.add_option('--use-fuzzy', '-f', dest='use_fuzzy', + action='store_true', + help='also include fuzzy translations (default ' + '%default)') + + parser.set_defaults(domain='messages', use_fuzzy=False) + options, args = parser.parse_args(argv) + + if not options.locale: + parser.error('you must provide a locale for the new catalog') + try: + locale = Locale.parse(options.locale) + except UnknownLocaleError, e: + parser.error(e) + + if not options.directory and not options.input_file: + parser.error('you must specify the base directory or input file') + if not options.input_file: + options.input_file = os.path.join(options.directory, + options.locale, 'LC_MESSAGES', + options.domain + '.po') + + if not options.directory and not options.output_file: + parser.error('you must specify the base directory or output file') + + if not options.output_file: + options.output_file = os.path.join(options.directory, + options.locale, 'LC_MESSAGES', + options.domain + '.mo') + if not os.path.exists(os.path.dirname(options.output_file)): + os.makedirs(os.path.dirname(options.output_file)) + + infile = open(options.input_file, 'r') + try: + catalog = read_po(infile) + finally: + infile.close() + + print 'compiling catalog to %r' % options.output_file + + outfile = open(options.output_file, 'w') + try: + write_mo(outfile, catalog, use_fuzzy=options.use_fuzzy) + finally: + outfile.close() + def extract(self, argv): """Subcommand for extracting messages from source files and generating a POT file. @@ -369,7 +521,8 @@ parser = OptionParser(usage=self.usage % ('extract', 'dir1 ...'), description=self.command_descriptions['extract']) parser.add_option('--charset', dest='charset', - help='charset to use in the output') + help='charset to use in the output (default ' + '"%default")') parser.add_option('-k', '--keyword', dest='keywords', action='append', help='keywords to look for in addition to the ' 'defaults. You can specify multiple -k flags on ' diff --git a/babel/messages/mofile.py b/babel/messages/mofile.py new file mode 100644 --- /dev/null +++ b/babel/messages/mofile.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2007 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://babel.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://babel.edgewall.org/log/. + +"""Writing of files in the ``gettext`` MO (machine object) format. + +:see: `The Format of MO Files + `_ +""" + +import array +import struct + +def write_mo(fileobj, catalog, use_fuzzy=False): + """Write a catalog to the specified file-like object using the GNU MO file + format. + + >>> from babel.messages import Catalog + >>> from gettext import GNUTranslations + >>> from StringIO import StringIO + + >>> catalog = Catalog(locale='en_US') + >>> catalog.add('foo', 'Voh') + >>> catalog.add((u'bar', u'baz'), (u'Bahr', u'Batz')) + >>> catalog.add('fuz', 'Futz', flags=['fuzzy']) + >>> buf = StringIO() + + >>> write_mo(buf, catalog) + >>> buf.seek(0) + >>> translations = GNUTranslations(fp=buf) + >>> translations.ugettext('foo') + u'Voh' + >>> translations.ungettext('bar', 'baz', 1) + u'Bahr' + >>> translations.ungettext('bar', 'baz', 2) + u'Batz' + >>> translations.ugettext('fuz') + u'fuz' + + :param fileobj: the file-like object to write to + :param catalog: the `Catalog` instance + :param use_fuzzy: whether translations marked as "fuzzy" should be included + in the output + """ + messages = list(catalog) + if not use_fuzzy: + messages[1:] = [m for m in messages[1:] if not m.fuzzy] + messages.sort(lambda x,y: cmp(x.id, y.id)) + + ids = strs = '' + offsets = [] + + for message in messages: + # For each string, we need size and file offset. Each string is NUL + # terminated; the NUL does not count into the size. + if message.pluralizable: + msgid = '\x00'.join([ + msgid.encode(catalog.charset) for msgid in message.id + ]) + msgstr = '\x00'.join([ + msgstr.encode(catalog.charset) for msgstr in message.string + ]) + else: + msgid = message.id.encode(catalog.charset) + msgstr = message.string.encode(catalog.charset) + offsets.append((len(ids), len(msgid), len(strs), len(msgstr))) + ids += msgid + '\x00' + strs += msgstr + '\x00' + + # The header is 7 32-bit unsigned integers. We don't use hash tables, so + # the keys start right after the index tables. + keystart = 7 * 4 + 16 * len(messages) + valuestart = keystart + len(ids) + + # The string table first has the list of keys, then the list of values. + # Each entry has first the size of the string, then the file offset. + koffsets = [] + voffsets = [] + for o1, l1, o2, l2 in offsets: + koffsets += [l1, o1 + keystart] + voffsets += [l2, o2 + valuestart] + offsets = koffsets + voffsets + + fileobj.write(struct.pack('Iiiiiii', + 0x950412deL, # magic + 0, # version + len(messages), # number of entries + 7 * 4, # start of key index + 7 * 4 + len(messages) * 8, # start of value index + 0, 0 # size and offset of hash table + ) + array.array("i", offsets).tostring() + ids + strs) diff --git a/babel/messages/tests/__init__.py b/babel/messages/tests/__init__.py --- a/babel/messages/tests/__init__.py +++ b/babel/messages/tests/__init__.py @@ -14,11 +14,12 @@ import unittest def suite(): - from babel.messages.tests import catalog, extract, frontend, pofile + from babel.messages.tests import catalog, extract, frontend, mofile, pofile suite = unittest.TestSuite() suite.addTest(catalog.suite()) suite.addTest(extract.suite()) suite.addTest(frontend.suite()) + suite.addTest(mofile.suite()) suite.addTest(pofile.suite()) return suite diff --git a/babel/messages/tests/frontend.py b/babel/messages/tests/frontend.py --- a/babel/messages/tests/frontend.py +++ b/babel/messages/tests/frontend.py @@ -29,6 +29,40 @@ from babel.util import LOCALTZ +class CompileCatalogTestCase(unittest.TestCase): + + def setUp(self): + self.olddir = os.getcwd() + self.datadir = os.path.join(os.path.dirname(__file__), 'data') + os.chdir(self.datadir) + _global_log.threshold = 5 # shut up distutils logging + + self.dist = Distribution(dict( + name='TestProject', + version='0.1', + packages=['project'] + )) + self.cmd = frontend.compile_catalog(self.dist) + self.cmd.initialize_options() + + def tearDown(self): + os.chdir(self.olddir) + + def test_no_locale_specified(self): + self.cmd.directory = 'dummy' + self.assertRaises(DistutilsOptionError, self.cmd.finalize_options) + + def test_no_directory_or_output_file_specified(self): + self.cmd.locale = 'en_US' + self.cmd.input_file = 'dummy' + self.assertRaises(DistutilsOptionError, self.cmd.finalize_options) + + def test_no_directory_or_input_file_specified(self): + self.cmd.locale = 'en_US' + self.cmd.output_file = 'dummy' + self.assertRaises(DistutilsOptionError, self.cmd.finalize_options) + + class ExtractMessagesTestCase(unittest.TestCase): def setUp(self): @@ -361,6 +395,7 @@ -h, --help show this help message and exit commands: + compile compile a message catalog to a mo file extract extract messages from source files and generate a pot file init create new message catalogs from a template """, sys.stdout.getvalue().lower()) @@ -527,6 +562,7 @@ def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(frontend)) + suite.addTest(unittest.makeSuite(CompileCatalogTestCase)) suite.addTest(unittest.makeSuite(ExtractMessagesTestCase)) suite.addTest(unittest.makeSuite(NewCatalogTestCase)) suite.addTest(unittest.makeSuite(CommandLineInterfaceTestCase)) diff --git a/babel/messages/tests/mofile.py b/babel/messages/tests/mofile.py new file mode 100644 --- /dev/null +++ b/babel/messages/tests/mofile.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2007 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://babel.edgewall.org/wiki/License. +# +# This software consists of voluntary contributions made by many +# individuals. For the exact contribution history, see the revision +# history and logs, available at http://babel.edgewall.org/log/. + +import doctest +import unittest + +from babel.messages import mofile + +def suite(): + suite = unittest.TestSuite() + suite.addTest(doctest.DocTestSuite(mofile)) + return suite + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/doc/setup.txt b/doc/setup.txt --- a/doc/setup.txt +++ b/doc/setup.txt @@ -19,7 +19,8 @@ setup( ... - cmd_class = {'extract_messages': babel.extract_messages, + cmd_class = {'compile_catalog': babel.compile_catalog, + 'extract_messages': babel.extract_messages, 'new_catalog': babel.new_catalog} ) @@ -29,6 +30,63 @@ .. sectnum:: +compile_catalog +=============== + +The ``compile_catalog`` command is similar to the GNU ``msgfmt`` tool, in that +it takes a message catalog from a PO file and compiles it to a binary MO file. + +If the command has been correctly installed or registered, a project's +``setup.py`` script should allow you to use the command:: + + $ ./setup.py compile_catalog --help + Global options: + --verbose (-v) run verbosely (default) + --quiet (-q) run quietly (turns verbosity off) + --dry-run (-n) don't actually do anything + --help (-h) show detailed help message + + Options for 'compile_catalog' command: + ... + +Running the command will produce a PO template file:: + + $ ./setup.py compile_catalog --directory foobar/locale --locale pt_BR + running compile_catalog + compiling catalog to to foobar/locale/pt_BR/LC_MESSAGES/messages.mo + + +Options +------- + +The ``compile_catalog`` command accepts the following options: + + +-----------------------------+----------------------------------------------+ + | Option | Description | + +=============================+==============================================+ + | ``--domain`` | domain of the PO file (defaults to | + | | lower-cased project name) | + +-----------------------------+----------------------------------------------+ + | ``--directory`` (``-d``) | name of the base directory | + +-----------------------------+----------------------------------------------+ + | ``--input-file`` (``-i``) | name of the input file | + +-----------------------------+----------------------------------------------+ + | ``--output-file`` (``-o``) | name of the output file | + +-----------------------------+----------------------------------------------+ + | ``--locale`` | locale for the new localized string | + +-----------------------------+----------------------------------------------+ + | ``--use-fuzzy`` | also include "fuzzy" translations | + +-----------------------------+----------------------------------------------+ + +If ``directory`` is specified, but ``output-file`` is not, the default filename +of the output file will be:: + + //LC_MESSAGES/.mo + +These options can either be specified on the command-line, or in the +``setup.cfg`` file. + + extract_messages ================ @@ -36,7 +94,7 @@ it can extract localizable messages from a variety of difference source files, and generate a PO (portable object) template file from the collected messages. -If the command has been correctly installed or registered, another project's +If the command has been correctly installed or registered, a project's ``setup.py`` script should allow you to use the command:: $ ./setup.py extract_messages --help @@ -158,7 +216,7 @@ The ``new_catalog`` command is basically equivalent to the GNU ``msginit`` program: it creates a new translation catalog based on a PO template file (POT). -If the command has been correctly installed or registered, another project's +If the command has been correctly installed or registered, a project's ``setup.py`` script should allow you to use the command:: $ ./setup.py new_catalog --help diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -138,6 +138,7 @@ babel = babel.messages.frontend:main [distutils.commands] + compile_catalog = babel.messages.frontend:compile_catalog extract_messages = babel.messages.frontend:extract_messages new_catalog = babel.messages.frontend:new_catalog