changeset 162:661cb602781d

Add MO file generation. Closes #21.
author cmlenz
date Thu, 21 Jun 2007 14:38:30 +0000
parents b5659b7779be
children 2faa5dc63068
files ChangeLog babel/messages/frontend.py babel/messages/mofile.py babel/messages/tests/__init__.py babel/messages/tests/frontend.py babel/messages/tests/mofile.py doc/setup.txt setup.py
diffstat 8 files changed, 386 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+Version 0.9
+http://svn.edgewall.org/repos/babel/tags/0.9.0/
+(?, from branches/stable/0.9.x)
+
+ * Added compilation of message catalogs to MO files.
+
 Version 0.8.1
 http://svn.edgewall.org/repos/babel/tags/0.8.1/
 (?, from branches/stable/0.8.x)
--- a/babel/messages/frontend.py
+++ b/babel/messages/frontend.py
@@ -31,6 +31,7 @@
 from babel.messages.catalog import Catalog
 from babel.messages.extract import extract_from_dir, DEFAULT_KEYWORDS, \
                                    DEFAULT_MAPPING
+from babel.messages.mofile import write_mo
 from babel.messages.pofile import read_po, write_po
 from babel.messages.plurals import PLURALS
 from babel.util import odict, LOCALTZ
@@ -40,6 +41,90 @@
 __docformat__ = 'restructuredtext en'
 
 
+class compile_catalog(Command):
+    """Catalog compilation command for use in ``setup.py`` scripts.
+
+    If correctly installed, this command is available to Setuptools-using
+    setup scripts automatically. For projects using plain old ``distutils``,
+    the command needs to be registered explicitly in ``setup.py``::
+
+        from babel.messages.frontend import compile_catalog
+
+        setup(
+            ...
+            cmdclass = {'new_catalog': compile_catalog}
+        )
+
+    :see: `Integrating new distutils commands <http://docs.python.org/dist/node32.html>`_
+    :see: `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
+    """
+
+    description = 'compile a catalog to a binary MO file'
+    user_options = [
+        ('domain=', 'D',
+         "domain of PO file (default 'messages')"),
+        ('directory=', 'd',
+         'path to base directory containing the catalogs'),
+        ('input-file=', 'i',
+         'name of the input file'),
+        ('output-file=', 'o',
+         "name of the output file (default "
+         "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"),
+        ('locale=', 'l',
+         'locale of the catalog to compile'),
+        ('use-fuzzy', 'f',
+         'also include fuzzy translations'),
+    ]
+    boolean_options = ['use-fuzzy']
+
+    def initialize_options(self):
+        self.domain = 'messages'
+        self.directory = None
+        self.input_file = None
+        self.output_file = None
+        self.locale = None
+        self.use_fuzzy = False
+
+    def finalize_options(self):
+        if not self.locale:
+            raise DistutilsOptionError('you must specify the locale for the '
+                                       'catalog to compile')
+        try:
+            self._locale = Locale.parse(self.locale)
+        except UnknownLocaleError, e:
+            raise DistutilsOptionError(e)
+
+        if not self.directory and not self.input_file:
+            raise DistutilsOptionError('you must specify the input file')
+        if not self.input_file:
+            self.input_file = os.path.join(self.directory, self.locale,
+                                           'LC_MESSAGES', self.domain + '.po')
+
+        if not self.directory and not self.output_file:
+            raise DistutilsOptionError('you must specify the output file')
+        if not self.output_file:
+            self.output_file = os.path.join(self.directory, self.locale,
+                                            'LC_MESSAGES', self.domain + '.mo')
+
+        if not os.path.exists(os.path.dirname(self.output_file)):
+            os.makedirs(os.path.dirname(self.output_file))
+
+    def run(self):
+        log.info('compiling catalog to %s', self.output_file)
+
+        infile = open(self.input_file, 'r')
+        try:
+            catalog = read_po(infile)
+        finally:
+            infile.close()
+
+        outfile = open(self.output_file, 'w')
+        try:
+            write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy)
+        finally:
+            outfile.close()
+
+
 class extract_messages(Command):
     """Message extraction command for use in ``setup.py`` scripts.
 
@@ -326,8 +411,9 @@
 
     usage = '%%prog %s [options] %s'
     version = '%%prog %s' % VERSION
-    commands = ['extract', 'init']
+    commands = ['compile', 'extract', 'init']
     command_descriptions = {
+        'compile': 'compile a message catalog to a MO file',
         'extract': 'extract messages from source files and generate a POT file',
         'init': 'create new message catalogs from a template'
     }
@@ -360,6 +446,72 @@
         for command in self.commands:
             print format % (command, self.command_descriptions[command])
 
+    def compile(self, argv):
+        """Subcommand for compiling a message catalog to a MO file.
+
+        :param argv: the command arguments
+        """
+        parser = OptionParser(usage=self.usage % ('init',''),
+                              description=self.command_descriptions['init'])
+        parser.add_option('--domain', '-D', dest='domain',
+                          help="domain of MO and PO files (default '%default')")
+        parser.add_option('--directory', '-d', dest='directory',
+                          metavar='DIR', help='base directory of catalog files')
+        parser.add_option('--input-file', '-i', dest='input_file',
+                          metavar='FILE', help='name of the input file')
+        parser.add_option('--output-file', '-o', dest='output_file',
+                          metavar='FILE',
+                          help="name of the output file (default "
+                               "'<output_dir>/<locale>/LC_MESSAGES/"
+                               "<domain>.mo')")
+        parser.add_option('--locale', '-l', dest='locale', metavar='LOCALE',
+                          help='locale of the catalog')
+        parser.add_option('--use-fuzzy', '-f', dest='use_fuzzy',
+                          action='store_true',
+                          help='also include fuzzy translations (default '
+                               '%default)')
+
+        parser.set_defaults(domain='messages', use_fuzzy=False)
+        options, args = parser.parse_args(argv)
+
+        if not options.locale:
+            parser.error('you must provide a locale for the new catalog')
+        try:
+            locale = Locale.parse(options.locale)
+        except UnknownLocaleError, e:
+            parser.error(e)
+
+        if not options.directory and not options.input_file:
+            parser.error('you must specify the base directory or input file')
+        if not options.input_file:
+            options.input_file = os.path.join(options.directory,
+                                              options.locale, 'LC_MESSAGES',
+                                              options.domain + '.po')
+
+        if not options.directory and not options.output_file:
+            parser.error('you must specify the base directory or output file')
+
+        if not options.output_file:
+            options.output_file = os.path.join(options.directory,
+                                               options.locale, 'LC_MESSAGES',
+                                               options.domain + '.mo')
+        if not os.path.exists(os.path.dirname(options.output_file)):
+            os.makedirs(os.path.dirname(options.output_file))
+
+        infile = open(options.input_file, 'r')
+        try:
+            catalog = read_po(infile)
+        finally:
+            infile.close()
+
+        print 'compiling catalog to %r' % options.output_file
+
+        outfile = open(options.output_file, 'w')
+        try:
+            write_mo(outfile, catalog, use_fuzzy=options.use_fuzzy)
+        finally:
+            outfile.close()
+
     def extract(self, argv):
         """Subcommand for extracting messages from source files and generating
         a POT file.
@@ -369,7 +521,8 @@
         parser = OptionParser(usage=self.usage % ('extract', 'dir1 <dir2> ...'),
                               description=self.command_descriptions['extract'])
         parser.add_option('--charset', dest='charset',
-                          help='charset to use in the output')
+                          help='charset to use in the output (default '
+                               '"%default")')
         parser.add_option('-k', '--keyword', dest='keywords', action='append',
                           help='keywords to look for in addition to the '
                                'defaults. You can specify multiple -k flags on '
new file mode 100644
--- /dev/null
+++ b/babel/messages/mofile.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2007 Edgewall Software
+# All rights reserved.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at http://babel.edgewall.org/wiki/License.
+#
+# This software consists of voluntary contributions made by many
+# individuals. For the exact contribution history, see the revision
+# history and logs, available at http://babel.edgewall.org/log/.
+
+"""Writing of files in the ``gettext`` MO (machine object) format.
+
+:see: `The Format of MO Files
+       <http://www.gnu.org/software/gettext/manual/gettext.html#MO-Files>`_
+"""
+
+import array
+import struct
+
+def write_mo(fileobj, catalog, use_fuzzy=False):
+    """Write a catalog to the specified file-like object using the GNU MO file
+    format.
+    
+    >>> from babel.messages import Catalog
+    >>> from gettext import GNUTranslations
+    >>> from StringIO import StringIO
+    
+    >>> catalog = Catalog(locale='en_US')
+    >>> catalog.add('foo', 'Voh')
+    >>> catalog.add((u'bar', u'baz'), (u'Bahr', u'Batz'))
+    >>> catalog.add('fuz', 'Futz', flags=['fuzzy'])
+    >>> buf = StringIO()
+    
+    >>> write_mo(buf, catalog)
+    >>> buf.seek(0)
+    >>> translations = GNUTranslations(fp=buf)
+    >>> translations.ugettext('foo')
+    u'Voh'
+    >>> translations.ungettext('bar', 'baz', 1)
+    u'Bahr'
+    >>> translations.ungettext('bar', 'baz', 2)
+    u'Batz'
+    >>> translations.ugettext('fuz')
+    u'fuz'
+    
+    :param fileobj: the file-like object to write to
+    :param catalog: the `Catalog` instance
+    :param use_fuzzy: whether translations marked as "fuzzy" should be included
+                      in the output
+    """
+    messages = list(catalog)
+    if not use_fuzzy:
+        messages[1:] = [m for m in messages[1:] if not m.fuzzy]
+    messages.sort(lambda x,y: cmp(x.id, y.id))
+
+    ids = strs = ''
+    offsets = []
+
+    for message in messages:
+        # For each string, we need size and file offset.  Each string is NUL
+        # terminated; the NUL does not count into the size.
+        if message.pluralizable:
+            msgid = '\x00'.join([
+                msgid.encode(catalog.charset) for msgid in message.id
+            ])
+            msgstr = '\x00'.join([
+                msgstr.encode(catalog.charset) for msgstr in message.string
+            ])
+        else:
+            msgid = message.id.encode(catalog.charset)
+            msgstr = message.string.encode(catalog.charset)
+        offsets.append((len(ids), len(msgid), len(strs), len(msgstr)))
+        ids += msgid + '\x00'
+        strs += msgstr + '\x00'
+
+    # The header is 7 32-bit unsigned integers.  We don't use hash tables, so
+    # the keys start right after the index tables.
+    keystart = 7 * 4 + 16 * len(messages)
+    valuestart = keystart + len(ids)
+
+    # The string table first has the list of keys, then the list of values.
+    # Each entry has first the size of the string, then the file offset.
+    koffsets = []
+    voffsets = []
+    for o1, l1, o2, l2 in offsets:
+        koffsets += [l1, o1 + keystart]
+        voffsets += [l2, o2 + valuestart]
+    offsets = koffsets + voffsets
+
+    fileobj.write(struct.pack('Iiiiiii',
+        0x950412deL,                # magic
+        0,                          # version
+        len(messages),              # number of entries
+        7 * 4,                      # start of key index
+        7 * 4 + len(messages) * 8,  # start of value index
+        0, 0                        # size and offset of hash table
+    ) + array.array("i", offsets).tostring() + ids + strs)
--- a/babel/messages/tests/__init__.py
+++ b/babel/messages/tests/__init__.py
@@ -14,11 +14,12 @@
 import unittest
 
 def suite():
-    from babel.messages.tests import catalog, extract, frontend, pofile
+    from babel.messages.tests import catalog, extract, frontend, mofile, pofile
     suite = unittest.TestSuite()
     suite.addTest(catalog.suite())
     suite.addTest(extract.suite())
     suite.addTest(frontend.suite())
+    suite.addTest(mofile.suite())
     suite.addTest(pofile.suite())
     return suite
 
--- a/babel/messages/tests/frontend.py
+++ b/babel/messages/tests/frontend.py
@@ -29,6 +29,40 @@
 from babel.util import LOCALTZ
 
 
+class CompileCatalogTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.olddir = os.getcwd()
+        self.datadir = os.path.join(os.path.dirname(__file__), 'data')
+        os.chdir(self.datadir)
+        _global_log.threshold = 5 # shut up distutils logging
+
+        self.dist = Distribution(dict(
+            name='TestProject',
+            version='0.1',
+            packages=['project']
+        ))
+        self.cmd = frontend.compile_catalog(self.dist)
+        self.cmd.initialize_options()
+
+    def tearDown(self):
+        os.chdir(self.olddir)
+
+    def test_no_locale_specified(self):
+        self.cmd.directory = 'dummy'
+        self.assertRaises(DistutilsOptionError, self.cmd.finalize_options)
+
+    def test_no_directory_or_output_file_specified(self):
+        self.cmd.locale = 'en_US'
+        self.cmd.input_file = 'dummy'
+        self.assertRaises(DistutilsOptionError, self.cmd.finalize_options)
+
+    def test_no_directory_or_input_file_specified(self):
+        self.cmd.locale = 'en_US'
+        self.cmd.output_file = 'dummy'
+        self.assertRaises(DistutilsOptionError, self.cmd.finalize_options)
+
+
 class ExtractMessagesTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -361,6 +395,7 @@
   -h, --help  show this help message and exit
 
 commands:
+  compile     compile a message catalog to a mo file
   extract     extract messages from source files and generate a pot file
   init        create new message catalogs from a template
 """, sys.stdout.getvalue().lower())
@@ -527,6 +562,7 @@
 def suite():
     suite = unittest.TestSuite()
     suite.addTest(doctest.DocTestSuite(frontend))
+    suite.addTest(unittest.makeSuite(CompileCatalogTestCase))
     suite.addTest(unittest.makeSuite(ExtractMessagesTestCase))
     suite.addTest(unittest.makeSuite(NewCatalogTestCase))
     suite.addTest(unittest.makeSuite(CommandLineInterfaceTestCase))
new file mode 100644
--- /dev/null
+++ b/babel/messages/tests/mofile.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2007 Edgewall Software
+# All rights reserved.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at http://babel.edgewall.org/wiki/License.
+#
+# This software consists of voluntary contributions made by many
+# individuals. For the exact contribution history, see the revision
+# history and logs, available at http://babel.edgewall.org/log/.
+
+import doctest
+import unittest
+
+from babel.messages import mofile
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(doctest.DocTestSuite(mofile))
+    return suite
+
+if __name__ == '__main__':
+    unittest.main(defaultTest='suite')
--- a/doc/setup.txt
+++ b/doc/setup.txt
@@ -19,7 +19,8 @@
     
     setup(
         ...
-        cmd_class = {'extract_messages': babel.extract_messages,
+        cmd_class = {'compile_catalog': babel.compile_catalog,
+                     'extract_messages': babel.extract_messages,
                      'new_catalog': babel.new_catalog}
     )
 
@@ -29,6 +30,63 @@
 .. sectnum::
 
 
+compile_catalog
+===============
+
+The ``compile_catalog`` command is similar to the GNU ``msgfmt`` tool, in that
+it takes a message catalog from a PO file and compiles it to a binary MO file.
+
+If the command has been correctly installed or registered, a project's
+``setup.py`` script should allow you to use the command::
+
+    $ ./setup.py compile_catalog --help
+    Global options:
+      --verbose (-v)  run verbosely (default)
+      --quiet (-q)    run quietly (turns verbosity off)
+      --dry-run (-n)  don't actually do anything
+      --help (-h)     show detailed help message
+    
+    Options for 'compile_catalog' command:
+       ...
+
+Running the command will produce a PO template file::
+
+    $ ./setup.py compile_catalog --directory foobar/locale --locale pt_BR
+    running compile_catalog
+    compiling catalog to to foobar/locale/pt_BR/LC_MESSAGES/messages.mo
+
+
+Options
+-------
+
+The ``compile_catalog`` command accepts the following options:
+
+  +-----------------------------+----------------------------------------------+
+  | Option                      | Description                                  |
+  +=============================+==============================================+
+  | ``--domain``                | domain of the PO file (defaults to           |
+  |                             | lower-cased project name)                    |
+  +-----------------------------+----------------------------------------------+
+  | ``--directory`` (``-d``)    | name of the base directory                   |
+  +-----------------------------+----------------------------------------------+
+  | ``--input-file`` (``-i``)   | name of the input file                       |
+  +-----------------------------+----------------------------------------------+
+  | ``--output-file`` (``-o``)  | name of the output file                      |
+  +-----------------------------+----------------------------------------------+
+  | ``--locale``                | locale for the new localized string          |
+  +-----------------------------+----------------------------------------------+
+  | ``--use-fuzzy``             | also include "fuzzy" translations            |
+  +-----------------------------+----------------------------------------------+
+
+If ``directory`` is specified, but ``output-file`` is not, the default filename
+of the output file will be::
+
+    <output_dir>/<locale>/LC_MESSAGES/<domain>.mo
+
+These options can either be specified on the command-line, or in the
+``setup.cfg`` file.
+
+
 extract_messages
 ================
 
@@ -36,7 +94,7 @@
 it can extract localizable messages from a variety of difference source files,
 and generate a PO (portable object) template file from the collected messages.
 
-If the command has been correctly installed or registered, another project's
+If the command has been correctly installed or registered, a project's
 ``setup.py`` script should allow you to use the command::
 
     $ ./setup.py extract_messages --help
@@ -158,7 +216,7 @@
 The ``new_catalog`` command is basically equivalent to the GNU ``msginit``
 program: it creates a new translation catalog based on a PO template file (POT).
 
-If the command has been correctly installed or registered, another project's
+If the command has been correctly installed or registered, a project's
 ``setup.py`` script should allow you to use the command::
 
     $ ./setup.py new_catalog --help
--- a/setup.py
+++ b/setup.py
@@ -138,6 +138,7 @@
     babel = babel.messages.frontend:main
     
     [distutils.commands]
+    compile_catalog = babel.messages.frontend:compile_catalog
     extract_messages = babel.messages.frontend:extract_messages
     new_catalog = babel.messages.frontend:new_catalog
     
Copyright (C) 2012-2017 Edgewall Software