Mercurial > babel > old > mirror

comparison 0.8.x/scripts/import_cldr.py @ 142:4a7af44e6695 stable

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Create branch for 0.8.x releases.
author cmlenz
date Wed, 20 Jun 2007 10:09:07 +0000
parents
children
comparison
equal deleted inserted replaced
1:bf36ec5f5e50 142:4a7af44e6695
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Copyright (C) 2007 Edgewall Software
5 # All rights reserved.
6 #
7 # This software is licensed as described in the file COPYING, which
8 # you should have received as part of this distribution. The terms
9 # are also available at http://babel.edgewall.org/wiki/License.
10 #
11 # This software consists of voluntary contributions made by many
12 # individuals. For the exact contribution history, see the revision
13 # history and logs, available at http://babel.edgewall.org/log/.
14
15 import copy
16 from optparse import OptionParser
17 import os
18 import pickle
19 import sys
20 try:
21 from xml.etree.ElementTree import parse
22 except ImportError:
23 from elementtree.ElementTree import parse
24
25 # Make sure we're using Babel source, and not some previously installed version
26 sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..'))
27
28 from babel import dates, numbers
29
30 weekdays = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5,
31 'sun': 6}
32
33 try:
34 any
35 except NameError:
36 def any(iterable):
37 return filter(None, list(iterable))
38
39 def _text(elem):
40 buf = [elem.text or '']
41 for child in elem:
42 buf.append(_text(child))
43 buf.append(elem.tail or '')
44 return u''.join(filter(None, buf)).strip()
45
46 def main():
47 parser = OptionParser(usage='%prog path/to/cldr')
48 options, args = parser.parse_args()
49 if len(args) != 1:
50 parser.error('incorrect number of arguments')
51
52 srcdir = args[0]
53 destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
54 '..', 'babel', 'localedata')
55
56 sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml'))
57
58 # build a territory containment mapping for inheritance
59 regions = {}
60 for elem in sup.findall('//territoryContainment/group'):
61 regions[elem.attrib['type']] = elem.attrib['contains'].split()
62
63 # Resolve territory containment
64 territory_containment = {}
65 region_items = regions.items()
66 region_items.sort()
67 for group, territory_list in region_items:
68 for territory in territory_list:
69 containers = territory_containment.setdefault(territory, set([]))
70 if group in territory_containment:
71 containers |= territory_containment[group]
72 containers.add(group)
73
74 filenames = os.listdir(os.path.join(srcdir, 'main'))
75 filenames.remove('root.xml')
76 filenames.sort(lambda a,b: len(a)-len(b))
77 filenames.insert(0, 'root.xml')
78
79 dicts = {}
80
81 for filename in filenames:
82 print>>sys.stderr, 'Processing input file %r' % filename
83 stem, ext = os.path.splitext(filename)
84 if ext != '.xml':
85 continue
86
87 tree = parse(os.path.join(srcdir, 'main', filename))
88 data = {}
89
90 language = None
91 elem = tree.find('//identity/language')
92 if elem is not None:
93 language = elem.attrib['type']
94 print>>sys.stderr, ' Language: %r' % language
95
96 territory = None
97 elem = tree.find('//identity/territory')
98 if elem is not None:
99 territory = elem.attrib['type']
100 else:
101 territory = '001' # world
102 print>>sys.stderr, ' Territory: %r' % territory
103 regions = territory_containment.get(territory, [])
104 print>>sys.stderr, ' Regions: %r' % regions
105
106 # <localeDisplayNames>
107
108 territories = data.setdefault('territories', {})
109 for elem in tree.findall('//territories/territory'):
110 if 'draft' in elem.attrib and elem.attrib['type'] in territories:
111 continue
112 territories[elem.attrib['type']] = _text(elem)
113
114 languages = data.setdefault('languages', {})
115 for elem in tree.findall('//languages/language'):
116 if 'draft' in elem.attrib and elem.attrib['type'] in languages:
117 continue
118 languages[elem.attrib['type']] = _text(elem)
119
120 variants = data.setdefault('variants', {})
121 for elem in tree.findall('//variants/variant'):
122 if 'draft' in elem.attrib and elem.attrib['type'] in variants:
123 continue
124 variants[elem.attrib['type']] = _text(elem)
125
126 scripts = data.setdefault('scripts', {})
127 for elem in tree.findall('//scripts/script'):
128 if 'draft' in elem.attrib and elem.attrib['type'] in scripts:
129 continue
130 scripts[elem.attrib['type']] = _text(elem)
131
132 # <dates>
133
134 week_data = data.setdefault('week_data', {})
135 supelem = sup.find('//weekData')
136
137 for elem in supelem.findall('minDays'):
138 territories = elem.attrib['territories'].split()
139 if territory in territories or any([r in territories for r in regions]):
140 week_data['min_days'] = int(elem.attrib['count'])
141
142 for elem in supelem.findall('firstDay'):
143 territories = elem.attrib['territories'].split()
144 if territory in territories or any([r in territories for r in regions]):
145 week_data['first_day'] = weekdays[elem.attrib['day']]
146
147 for elem in supelem.findall('weekendStart'):
148 territories = elem.attrib['territories'].split()
149 if territory in territories or any([r in territories for r in regions]):
150 week_data['weekend_start'] = weekdays[elem.attrib['day']]
151
152 for elem in supelem.findall('weekendEnd'):
153 territories = elem.attrib['territories'].split()
154 if territory in territories or any([r in territories for r in regions]):
155 week_data['weekend_end'] = weekdays[elem.attrib['day']]
156
157 time_zones = data.setdefault('time_zones', {})
158 for elem in tree.findall('//timeZoneNames/zone'):
159 info = {}
160 city = elem.findtext('exemplarCity')
161 if city:
162 info['city'] = unicode(city)
163 for child in elem.findall('long/*'):
164 info.setdefault('long', {})[child.tag] = unicode(child.text)
165 for child in elem.findall('short/*'):
166 info.setdefault('short', {})[child.tag] = unicode(child.text)
167 time_zones[elem.attrib['type']] = info
168
169 zone_aliases = data.setdefault('zone_aliases', {})
170 if stem == 'root':
171 for elem in sup.findall('//timezoneData/zoneFormatting/zoneItem'):
172 if 'aliases' in elem.attrib:
173 canonical_id = elem.attrib['type']
174 for alias in elem.attrib['aliases'].split():
175 zone_aliases[alias] = canonical_id
176
177 for calendar in tree.findall('//calendars/calendar'):
178 if calendar.attrib['type'] != 'gregorian':
179 # TODO: support other calendar types
180 continue
181
182 months = data.setdefault('months', {})
183 for ctxt in calendar.findall('months/monthContext'):
184 ctxts = months.setdefault(ctxt.attrib['type'], {})
185 for width in ctxt.findall('monthWidth'):
186 widths = ctxts.setdefault(width.attrib['type'], {})
187 for elem in width.findall('month'):
188 if 'draft' in elem.attrib and int(elem.attrib['type']) in widths:
189 continue
190 widths[int(elem.attrib.get('type'))] = unicode(elem.text)
191
192 days = data.setdefault('days', {})
193 for ctxt in calendar.findall('days/dayContext'):
194 ctxts = days.setdefault(ctxt.attrib['type'], {})
195 for width in ctxt.findall('dayWidth'):
196 widths = ctxts.setdefault(width.attrib['type'], {})
197 for elem in width.findall('day'):
198 dtype = weekdays[elem.attrib['type']]
199 if 'draft' in elem.attrib and dtype in widths:
200 continue
201 widths[dtype] = unicode(elem.text)
202
203 quarters = data.setdefault('quarters', {})
204 for ctxt in calendar.findall('quarters/quarterContext'):
205 ctxts = quarters.setdefault(ctxt.attrib['type'], {})
206 for width in ctxt.findall('quarterWidth'):
207 widths = ctxts.setdefault(width.attrib['type'], {})
208 for elem in width.findall('quarter'):
209 if 'draft' in elem.attrib and int(elem.attrib['type']) in widths:
210 continue
211 widths[int(elem.attrib.get('type'))] = unicode(elem.text)
212
213 eras = data.setdefault('eras', {})
214 for width in calendar.findall('eras/*'):
215 ewidth = {'eraNames': 'wide', 'eraAbbr': 'abbreviated'}[width.tag]
216 widths = eras.setdefault(ewidth, {})
217 for elem in width.findall('era'):
218 if 'draft' in elem.attrib and int(elem.attrib['type']) in widths:
219 continue
220 widths[int(elem.attrib.get('type'))] = unicode(elem.text)
221
222 # AM/PM
223 periods = data.setdefault('periods', {})
224 for elem in calendar.findall('am'):
225 if 'draft' in elem.attrib and elem.tag in periods:
226 continue
227 periods[elem.tag] = unicode(elem.text)
228 for elem in calendar.findall('pm'):
229 if 'draft' in elem.attrib and elem.tag in periods:
230 continue
231 periods[elem.tag] = unicode(elem.text)
232
233 date_formats = data.setdefault('date_formats', {})
234 for elem in calendar.findall('dateFormats/dateFormatLength'):
235 if 'draft' in elem.attrib and elem.attrib.get('type') in date_formats:
236 continue
237 try:
238 date_formats[elem.attrib.get('type')] = \
239 dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern')))
240 except ValueError, e:
241 print>>sys.stderr, 'ERROR: %s' % e
242
243 time_formats = data.setdefault('time_formats', {})
244 for elem in calendar.findall('timeFormats/timeFormatLength'):
245 if 'draft' in elem.attrib and elem.attrib.get('type') in time_formats:
246 continue
247 try:
248 time_formats[elem.attrib.get('type')] = \
249 dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern')))
250 except ValueError, e:
251 print>>sys.stderr, 'ERROR: %s' % e
252
253 datetime_formats = data.setdefault('datetime_formats', {})
254 for elem in calendar.findall('dateTimeFormats/dateTimeFormatLength'):
255 if 'draft' in elem.attrib and elem.attrib.get('type') in datetime_formats:
256 continue
257 try:
258 datetime_formats[elem.attrib.get('type')] = \
259 unicode(elem.findtext('dateTimeFormat/pattern'))
260 except ValueError, e:
261 print>>sys.stderr, 'ERROR: %s' % e
262
263 # <numbers>
264
265 number_symbols = data.setdefault('number_symbols', {})
266 for elem in tree.findall('//numbers/symbols/*'):
267 number_symbols[elem.tag] = unicode(elem.text)
268
269 decimal_formats = data.setdefault('decimal_formats', {})
270 for elem in tree.findall('//decimalFormats/decimalFormatLength'):
271 if 'draft' in elem.attrib and elem.attrib.get('type') in decimal_formats:
272 continue
273 pattern = unicode(elem.findtext('decimalFormat/pattern'))
274 decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
275
276 scientific_formats = data.setdefault('scientific_formats', {})
277 for elem in tree.findall('//scientificFormats/scientificFormatLength'):
278 if 'draft' in elem.attrib and elem.attrib.get('type') in scientific_formats:
279 continue
280 pattern = unicode(elem.findtext('scientificFormat/pattern'))
281 scientific_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
282
283 currency_formats = data.setdefault('currency_formats', {})
284 for elem in tree.findall('//currencyFormats/currencyFormatLength'):
285 if 'draft' in elem.attrib and elem.attrib.get('type') in currency_formats:
286 continue
287 pattern = unicode(elem.findtext('currencyFormat/pattern'))
288 currency_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
289
290 percent_formats = data.setdefault('percent_formats', {})
291 for elem in tree.findall('//percentFormats/percentFormatLength'):
292 if 'draft' in elem.attrib and elem.attrib.get('type') in percent_formats:
293 continue
294 pattern = unicode(elem.findtext('percentFormat/pattern'))
295 percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
296
297 currency_names = data.setdefault('currency_names', {})
298 currency_symbols = data.setdefault('currency_symbols', {})
299 for elem in tree.findall('//currencies/currency'):
300 name = elem.findtext('displayName')
301 if name:
302 currency_names[elem.attrib['type']] = unicode(name)
303 symbol = elem.findtext('symbol')
304 if symbol:
305 currency_symbols[elem.attrib['type']] = unicode(symbol)
306
307 dicts[stem] = data
308 outfile = open(os.path.join(destdir, stem + '.dat'), 'wb')
309 try:
310 pickle.dump(data, outfile, 2)
311 finally:
312 outfile.close()
313
314 if __name__ == '__main__':
315 main()
Copyright (C) 2012-2017 Edgewall Software