Mercurial > babel > mirror
annotate babel/messages/mofile.py @ 582:3fd7fb953633 trunk
fix handling of messages containing '\\n' (#171)
| author | fschwarz |
|---|---|
| date | Fri, 03 Aug 2012 22:41:49 +0000 |
| parents | ea0254950175 |
| children |
| rev | line source |
|---|---|
| 160 | 1 # -*- coding: utf-8 -*- |
| 2 # | |
| 530 | 3 # Copyright (C) 2007-2011 Edgewall Software |
| 160 | 4 # All rights reserved. |
| 5 # | |
| 6 # This software is licensed as described in the file COPYING, which | |
| 7 # you should have received as part of this distribution. The terms | |
| 8 # are also available at http://babel.edgewall.org/wiki/License. | |
| 9 # | |
| 10 # This software consists of voluntary contributions made by many | |
| 11 # individuals. For the exact contribution history, see the revision | |
| 12 # history and logs, available at http://babel.edgewall.org/log/. | |
| 13 | |
| 14 """Writing of files in the ``gettext`` MO (machine object) format. | |
| 15 | |
| 234 | 16 :since: version 0.9 |
| 160 | 17 :see: `The Format of MO Files |
| 18 <http://www.gnu.org/software/gettext/manual/gettext.html#MO-Files>`_ | |
| 19 """ | |
| 20 | |
| 21 import array | |
| 22 import struct | |
| 23 | |
| 334 | 24 from babel.messages.catalog import Catalog, Message |
| 25 | |
| 26 __all__ = ['read_mo', 'write_mo'] | |
| 161 | 27 __docformat__ = 'restructuredtext en' |
| 28 | |
| 334 | 29 |
| 30 LE_MAGIC = 0x950412deL | |
| 31 BE_MAGIC = 0xde120495L | |
| 32 | |
| 33 def read_mo(fileobj): | |
| 34 """Read a binary MO file from the given file-like object and return a | |
| 35 corresponding `Catalog` object. | |
| 36 | |
| 37 :param fileobj: the file-like object to read the MO file from | |
| 38 :return: a catalog object representing the parsed MO file | |
| 39 :rtype: `Catalog` | |
| 40 | |
| 41 :note: The implementation of this function is heavily based on the | |
| 42 ``GNUTranslations._parse`` method of the ``gettext`` module in the | |
| 43 standard library. | |
| 44 """ | |
| 45 catalog = Catalog() | |
| 46 headers = {} | |
| 47 | |
| 48 filename = getattr(fileobj, 'name', '') | |
| 49 | |
| 50 buf = fileobj.read() | |
| 51 buflen = len(buf) | |
| 335 | 52 unpack = struct.unpack |
| 334 | 53 |
| 54 # Parse the .mo file header, which consists of 5 little endian 32 | |
| 55 # bit words. | |
| 56 magic = unpack('<I', buf[:4])[0] # Are we big endian or little endian? | |
| 57 if magic == LE_MAGIC: | |
| 335 | 58 version, msgcount, origidx, transidx = unpack('<4I', buf[4:20]) |
| 334 | 59 ii = '<II' |
| 60 elif magic == BE_MAGIC: | |
| 335 | 61 version, msgcount, origidx, transidx = unpack('>4I', buf[4:20]) |
| 334 | 62 ii = '>II' |
| 63 else: | |
| 64 raise IOError(0, 'Bad magic number', filename) | |
| 65 | |
| 66 # Now put all messages from the .mo file buffer into the catalog | |
| 67 # dictionary | |
| 68 for i in xrange(0, msgcount): | |
| 335 | 69 mlen, moff = unpack(ii, buf[origidx:origidx + 8]) |
| 334 | 70 mend = moff + mlen |
| 71 tlen, toff = unpack(ii, buf[transidx:transidx + 8]) | |
| 72 tend = toff + tlen | |
| 73 if mend < buflen and tend < buflen: | |
| 74 msg = buf[moff:mend] | |
| 75 tmsg = buf[toff:tend] | |
| 76 else: | |
| 77 raise IOError(0, 'File is corrupt', filename) | |
| 78 | |
| 79 # See if we're looking at GNU .mo conventions for metadata | |
| 80 if mlen == 0: | |
| 81 # Catalog description | |
| 82 lastkey = key = None | |
| 83 for item in tmsg.splitlines(): | |
| 84 item = item.strip() | |
| 85 if not item: | |
| 86 continue | |
| 87 if ':' in item: | |
| 88 key, value = item.split(':', 1) | |
| 89 lastkey = key = key.strip().lower() | |
| 335 | 90 headers[key] = value.strip() |
| 334 | 91 elif lastkey: |
| 335 | 92 headers[lastkey] += '\n' + item |
| 334 | 93 |
| 335 | 94 if '\x04' in msg: # context |
| 95 ctxt, msg = msg.split('\x04') | |
| 96 else: | |
| 97 ctxt = None | |
| 98 | |
| 99 if '\x00' in msg: # plural forms | |
| 334 | 100 msg = msg.split('\x00') |
| 101 tmsg = tmsg.split('\x00') | |
| 335 | 102 if catalog.charset: |
| 103 msg = [x.decode(catalog.charset) for x in msg] | |
| 104 tmsg = [x.decode(catalog.charset) for x in tmsg] | |
| 334 | 105 else: |
| 335 | 106 if catalog.charset: |
| 107 msg = msg.decode(catalog.charset) | |
| 108 tmsg = tmsg.decode(catalog.charset) | |
| 109 catalog[msg] = Message(msg, tmsg, context=ctxt) | |
| 334 | 110 |
| 111 # advance to next entry in the seek tables | |
| 335 | 112 origidx += 8 |
| 334 | 113 transidx += 8 |
| 114 | |
| 115 catalog.mime_headers = headers.items() | |
| 116 return catalog | |
| 117 | |
| 160 | 118 def write_mo(fileobj, catalog, use_fuzzy=False): |
| 119 """Write a catalog to the specified file-like object using the GNU MO file | |
| 120 format. | |
| 121 | |
| 122 >>> from babel.messages import Catalog | |
| 123 >>> from gettext import GNUTranslations | |
| 124 >>> from StringIO import StringIO | |
| 125 | |
| 126 >>> catalog = Catalog(locale='en_US') | |
| 127 >>> catalog.add('foo', 'Voh') | |
|
544
ea0254950175
catalog.add() now returns the message instance (closes #245)
fschwarz
parents:
531
diff
changeset
|
128 <Message ...> |
| 160 | 129 >>> catalog.add((u'bar', u'baz'), (u'Bahr', u'Batz')) |
|
544
ea0254950175
catalog.add() now returns the message instance (closes #245)
fschwarz
parents:
531
diff
changeset
|
130 <Message ...> |
| 160 | 131 >>> catalog.add('fuz', 'Futz', flags=['fuzzy']) |
|
544
ea0254950175
catalog.add() now returns the message instance (closes #245)
fschwarz
parents:
531
diff
changeset
|
132 <Message ...> |
| 172 | 133 >>> catalog.add('Fizz', '') |
|
544
ea0254950175
catalog.add() now returns the message instance (closes #245)
fschwarz
parents:
531
diff
changeset
|
134 <Message ...> |
|
174
bd256296086c
Extended the doctest to include tests for the fix on [176].
palgarvio
parents:
173
diff
changeset
|
135 >>> catalog.add(('Fuzz', 'Fuzzes'), ('', '')) |
|
544
ea0254950175
catalog.add() now returns the message instance (closes #245)
fschwarz
parents:
531
diff
changeset
|
136 <Message ...> |
| 160 | 137 >>> buf = StringIO() |
| 138 | |
| 139 >>> write_mo(buf, catalog) | |
| 140 >>> buf.seek(0) | |
| 141 >>> translations = GNUTranslations(fp=buf) | |
| 142 >>> translations.ugettext('foo') | |
| 143 u'Voh' | |
| 144 >>> translations.ungettext('bar', 'baz', 1) | |
| 145 u'Bahr' | |
| 146 >>> translations.ungettext('bar', 'baz', 2) | |
| 147 u'Batz' | |
| 148 >>> translations.ugettext('fuz') | |
| 149 u'fuz' | |
| 172 | 150 >>> translations.ugettext('Fizz') |
| 151 u'Fizz' | |
|
174
bd256296086c
Extended the doctest to include tests for the fix on [176].
palgarvio
parents:
173
diff
changeset
|
152 >>> translations.ugettext('Fuzz') |
|
bd256296086c
Extended the doctest to include tests for the fix on [176].
palgarvio
parents:
173
diff
changeset
|
153 u'Fuzz' |
|
bd256296086c
Extended the doctest to include tests for the fix on [176].
palgarvio
parents:
173
diff
changeset
|
154 >>> translations.ugettext('Fuzzes') |
|
bd256296086c
Extended the doctest to include tests for the fix on [176].
palgarvio
parents:
173
diff
changeset
|
155 u'Fuzzes' |
| 160 | 156 |
| 157 :param fileobj: the file-like object to write to | |
| 158 :param catalog: the `Catalog` instance | |
| 159 :param use_fuzzy: whether translations marked as "fuzzy" should be included | |
| 160 in the output | |
| 161 """ | |
| 162 messages = list(catalog) | |
| 163 if not use_fuzzy: | |
| 164 messages[1:] = [m for m in messages[1:] if not m.fuzzy] | |
|
248
f0b1ee94628c
add a __cmp__ to Message that correctly sorts by id, taking into account plurals
pjenvey
parents:
234
diff
changeset
|
165 messages.sort() |
| 160 | 166 |
| 167 ids = strs = '' | |
| 168 offsets = [] | |
| 169 | |
| 170 for message in messages: | |
| 171 # For each string, we need size and file offset. Each string is NUL | |
| 172 # terminated; the NUL does not count into the size. | |
| 173 if message.pluralizable: | |
| 174 msgid = '\x00'.join([ | |
| 175 msgid.encode(catalog.charset) for msgid in message.id | |
| 176 ]) | |
|
173
c100331c727c
Forgot to fix the pluralizable messages, regarding #28.
palgarvio
parents:
172
diff
changeset
|
177 msgstrs = [] |
|
c100331c727c
Forgot to fix the pluralizable messages, regarding #28.
palgarvio
parents:
172
diff
changeset
|
178 for idx, string in enumerate(message.string): |
|
c100331c727c
Forgot to fix the pluralizable messages, regarding #28.
palgarvio
parents:
172
diff
changeset
|
179 if not string: |
|
330
465a0582d308
Fix for #97, compilation of message catalogs for locales with more than two plural forms where the translations were empty was failing.
cmlenz
parents:
248
diff
changeset
|
180 msgstrs.append(message.id[min(int(idx), 1)]) |
|
173
c100331c727c
Forgot to fix the pluralizable messages, regarding #28.
palgarvio
parents:
172
diff
changeset
|
181 else: |
|
c100331c727c
Forgot to fix the pluralizable messages, regarding #28.
palgarvio
parents:
172
diff
changeset
|
182 msgstrs.append(string) |
| 160 | 183 msgstr = '\x00'.join([ |
|
173
c100331c727c
Forgot to fix the pluralizable messages, regarding #28.
palgarvio
parents:
172
diff
changeset
|
184 msgstr.encode(catalog.charset) for msgstr in msgstrs |
| 160 | 185 ]) |
| 186 else: | |
| 187 msgid = message.id.encode(catalog.charset) | |
| 172 | 188 if not message.string: |
| 189 msgstr = message.id.encode(catalog.charset) | |
| 190 else: | |
| 191 msgstr = message.string.encode(catalog.charset) | |
| 335 | 192 if message.context: |
| 420 | 193 msgid = '\x04'.join([message.context.encode(catalog.charset), |
| 194 msgid]) | |
| 160 | 195 offsets.append((len(ids), len(msgid), len(strs), len(msgstr))) |
| 196 ids += msgid + '\x00' | |
| 197 strs += msgstr + '\x00' | |
| 198 | |
| 199 # The header is 7 32-bit unsigned integers. We don't use hash tables, so | |
| 200 # the keys start right after the index tables. | |
| 201 keystart = 7 * 4 + 16 * len(messages) | |
| 202 valuestart = keystart + len(ids) | |
| 203 | |
| 204 # The string table first has the list of keys, then the list of values. | |
| 205 # Each entry has first the size of the string, then the file offset. | |
| 206 koffsets = [] | |
| 207 voffsets = [] | |
| 208 for o1, l1, o2, l2 in offsets: | |
| 209 koffsets += [l1, o1 + keystart] | |
| 210 voffsets += [l2, o2 + valuestart] | |
| 211 offsets = koffsets + voffsets | |
| 212 | |
| 213 fileobj.write(struct.pack('Iiiiiii', | |
| 334 | 214 LE_MAGIC, # magic |
| 160 | 215 0, # version |
| 216 len(messages), # number of entries | |
| 217 7 * 4, # start of key index | |
| 218 7 * 4 + len(messages) * 8, # start of value index | |
| 219 0, 0 # size and offset of hash table | |
| 220 ) + array.array("i", offsets).tostring() + ids + strs) |