Mercurial > babel > mirror
annotate scripts/import_cldr.py @ 346:5e58ea360a5c stable-0.9.x
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
author | cmlenz |
---|---|
date | Mon, 16 Jun 2008 12:48:43 +0000 |
parents | da97a3138239 |
children | 4eca63af0a12 |
rev | line source |
---|---|
1 | 1 #!/usr/bin/env python |
2 # -*- coding: utf-8 -*- | |
3 # | |
4 # Copyright (C) 2007 Edgewall Software | |
5 # All rights reserved. | |
6 # | |
7 # This software is licensed as described in the file COPYING, which | |
8 # you should have received as part of this distribution. The terms | |
9 # are also available at http://babel.edgewall.org/wiki/License. | |
10 # | |
11 # This software consists of voluntary contributions made by many | |
12 # individuals. For the exact contribution history, see the revision | |
13 # history and logs, available at http://babel.edgewall.org/log/. | |
14 | |
15 import copy | |
16 from optparse import OptionParser | |
17 import os | |
18 import pickle | |
19 import sys | |
20 try: | |
21 from xml.etree.ElementTree import parse | |
22 except ImportError: | |
23 from elementtree.ElementTree import parse | |
24 | |
65
75fe8369ed3b
Add Babel soruce path to CLDR import script automatically for asmodai ;-).
cmlenz
parents:
34
diff
changeset
|
25 # Make sure we're using Babel source, and not some previously installed version |
75fe8369ed3b
Add Babel soruce path to CLDR import script automatically for asmodai ;-).
cmlenz
parents:
34
diff
changeset
|
26 sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..')) |
75fe8369ed3b
Add Babel soruce path to CLDR import script automatically for asmodai ;-).
cmlenz
parents:
34
diff
changeset
|
27 |
9 | 28 from babel import dates, numbers |
1 | 29 |
15 | 30 weekdays = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5, |
31 'sun': 6} | |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
32 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
33 try: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
34 any |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
35 except NameError: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
36 def any(iterable): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
37 return filter(None, list(iterable)) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
38 |
1 | 39 def _text(elem): |
40 buf = [elem.text or ''] | |
41 for child in elem: | |
42 buf.append(_text(child)) | |
43 buf.append(elem.tail or '') | |
44 return u''.join(filter(None, buf)).strip() | |
45 | |
46 def main(): | |
47 parser = OptionParser(usage='%prog path/to/cldr') | |
48 options, args = parser.parse_args() | |
49 if len(args) != 1: | |
50 parser.error('incorrect number of arguments') | |
51 | |
52 srcdir = args[0] | |
53 destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), | |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
54 '..', 'babel') |
1 | 55 |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
56 sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml')) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
57 |
346
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
58 # Import global data from the supplemental files |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
59 global_data = {} |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
60 |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
61 territory_zones = global_data.setdefault('territory_zones', {}) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
62 zone_aliases = global_data.setdefault('zone_aliases', {}) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
63 zone_territories = global_data.setdefault('zone_territories', {}) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
64 for elem in sup.findall('//timezoneData/zoneFormatting/zoneItem'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
65 tzid = elem.attrib['type'] |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
66 territory_zones.setdefault(elem.attrib['territory'], []).append(tzid) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
67 zone_territories[tzid] = elem.attrib['territory'] |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
68 if 'aliases' in elem.attrib: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
69 for alias in elem.attrib['aliases'].split(): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
70 zone_aliases[alias] = tzid |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
71 |
346
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
72 # Import Metazone mapping |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
73 meta_zones = global_data.setdefault('meta_zones', {}) |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
74 tzsup = parse(os.path.join(srcdir, 'supplemental', 'metazoneInfo.xml')) |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
75 for elem in tzsup.findall('//timezone'): |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
76 for child in elem.findall('usesMetazone'): |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
77 if 'to' not in child.attrib: # FIXME: support old mappings |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
78 meta_zones[elem.attrib['type']] = child.attrib['mzone'] |
5e58ea360a5c
Merged revisions [358:360], [364:370], [373:378], [380:382] from [source:trunk].
cmlenz
parents:
233
diff
changeset
|
79 |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
80 outfile = open(os.path.join(destdir, 'global.dat'), 'wb') |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
81 try: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
82 pickle.dump(global_data, outfile, 2) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
83 finally: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
84 outfile.close() |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
85 |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
86 # build a territory containment mapping for inheritance |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
87 regions = {} |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
88 for elem in sup.findall('//territoryContainment/group'): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
89 regions[elem.attrib['type']] = elem.attrib['contains'].split() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
90 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
91 # Resolve territory containment |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
92 territory_containment = {} |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
93 region_items = regions.items() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
94 region_items.sort() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
95 for group, territory_list in region_items: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
96 for territory in territory_list: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
97 containers = territory_containment.setdefault(territory, set([])) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
98 if group in territory_containment: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
99 containers |= territory_containment[group] |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
100 containers.add(group) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
101 |
1 | 102 filenames = os.listdir(os.path.join(srcdir, 'main')) |
103 filenames.remove('root.xml') | |
104 filenames.sort(lambda a,b: len(a)-len(b)) | |
105 filenames.insert(0, 'root.xml') | |
106 | |
107 for filename in filenames: | |
108 print>>sys.stderr, 'Processing input file %r' % filename | |
109 stem, ext = os.path.splitext(filename) | |
110 if ext != '.xml': | |
111 continue | |
112 | |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
113 tree = parse(os.path.join(srcdir, 'main', filename)) |
1 | 114 data = {} |
115 | |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
116 language = None |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
117 elem = tree.find('//identity/language') |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
118 if elem is not None: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
119 language = elem.attrib['type'] |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
120 print>>sys.stderr, ' Language: %r' % language |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
121 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
122 territory = None |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
123 elem = tree.find('//identity/territory') |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
124 if elem is not None: |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
125 territory = elem.attrib['type'] |
13 | 126 else: |
127 territory = '001' # world | |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
128 print>>sys.stderr, ' Territory: %r' % territory |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
129 regions = territory_containment.get(territory, []) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
130 print>>sys.stderr, ' Regions: %r' % regions |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
131 |
1 | 132 # <localeDisplayNames> |
133 | |
134 territories = data.setdefault('territories', {}) | |
135 for elem in tree.findall('//territories/territory'): | |
136 if 'draft' in elem.attrib and elem.attrib['type'] in territories: | |
137 continue | |
138 territories[elem.attrib['type']] = _text(elem) | |
139 | |
140 languages = data.setdefault('languages', {}) | |
141 for elem in tree.findall('//languages/language'): | |
142 if 'draft' in elem.attrib and elem.attrib['type'] in languages: | |
143 continue | |
144 languages[elem.attrib['type']] = _text(elem) | |
145 | |
146 variants = data.setdefault('variants', {}) | |
147 for elem in tree.findall('//variants/variant'): | |
148 if 'draft' in elem.attrib and elem.attrib['type'] in variants: | |
149 continue | |
150 variants[elem.attrib['type']] = _text(elem) | |
151 | |
152 scripts = data.setdefault('scripts', {}) | |
153 for elem in tree.findall('//scripts/script'): | |
154 if 'draft' in elem.attrib and elem.attrib['type'] in scripts: | |
155 continue | |
156 scripts[elem.attrib['type']] = _text(elem) | |
157 | |
158 # <dates> | |
159 | |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
160 week_data = data.setdefault('week_data', {}) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
161 supelem = sup.find('//weekData') |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
162 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
163 for elem in supelem.findall('minDays'): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
164 territories = elem.attrib['territories'].split() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
165 if territory in territories or any([r in territories for r in regions]): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
166 week_data['min_days'] = int(elem.attrib['count']) |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
167 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
168 for elem in supelem.findall('firstDay'): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
169 territories = elem.attrib['territories'].split() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
170 if territory in territories or any([r in territories for r in regions]): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
171 week_data['first_day'] = weekdays[elem.attrib['day']] |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
172 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
173 for elem in supelem.findall('weekendStart'): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
174 territories = elem.attrib['territories'].split() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
175 if territory in territories or any([r in territories for r in regions]): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
176 week_data['weekend_start'] = weekdays[elem.attrib['day']] |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
177 |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
178 for elem in supelem.findall('weekendEnd'): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
179 territories = elem.attrib['territories'].split() |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
180 if territory in territories or any([r in territories for r in regions]): |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
181 week_data['weekend_end'] = weekdays[elem.attrib['day']] |
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
182 |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
183 zone_formats = data.setdefault('zone_formats', {}) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
184 for elem in tree.findall('//timeZoneNames/gmtFormat'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
185 if 'draft' not in elem.attrib: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
186 zone_formats['gmt'] = unicode(elem.text).replace('{0}', '%s') |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
187 break |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
188 for elem in tree.findall('//timeZoneNames/regionFormat'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
189 if 'draft' not in elem.attrib: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
190 zone_formats['region'] = unicode(elem.text).replace('{0}', '%s') |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
191 break |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
192 for elem in tree.findall('//timeZoneNames/fallbackFormat'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
193 if 'draft' not in elem.attrib: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
194 zone_formats['fallback'] = unicode(elem.text) \ |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
195 .replace('{0}', '%(0)s').replace('{1}', '%(1)s') |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
196 break |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
197 |
1 | 198 time_zones = data.setdefault('time_zones', {}) |
199 for elem in tree.findall('//timeZoneNames/zone'): | |
28
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
200 info = {} |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
201 city = elem.findtext('exemplarCity') |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
202 if city: |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
203 info['city'] = unicode(city) |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
204 for child in elem.findall('long/*'): |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
205 info.setdefault('long', {})[child.tag] = unicode(child.text) |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
206 for child in elem.findall('short/*'): |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
207 info.setdefault('short', {})[child.tag] = unicode(child.text) |
b00b06e5ace8
Import basic timezone info from CLDR (see #3). Still missing a couple other pieces in the puzzle.
cmlenz
parents:
26
diff
changeset
|
208 time_zones[elem.attrib['type']] = info |
1 | 209 |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
210 meta_zones = data.setdefault('meta_zones', {}) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
211 for elem in tree.findall('//timeZoneNames/metazone'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
212 info = {} |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
213 city = elem.findtext('exemplarCity') |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
214 if city: |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
215 info['city'] = unicode(city) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
216 for child in elem.findall('long/*'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
217 info.setdefault('long', {})[child.tag] = unicode(child.text) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
218 for child in elem.findall('short/*'): |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
219 info.setdefault('short', {})[child.tag] = unicode(child.text) |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
220 info['common'] = elem.findtext('commonlyUsed') == 'true' |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
221 meta_zones[elem.attrib['type']] = info |
34 | 222 |
1 | 223 for calendar in tree.findall('//calendars/calendar'): |
224 if calendar.attrib['type'] != 'gregorian': | |
225 # TODO: support other calendar types | |
226 continue | |
227 | |
228 months = data.setdefault('months', {}) | |
229 for ctxt in calendar.findall('months/monthContext'): | |
230 ctxts = months.setdefault(ctxt.attrib['type'], {}) | |
231 for width in ctxt.findall('monthWidth'): | |
232 widths = ctxts.setdefault(width.attrib['type'], {}) | |
233 for elem in width.findall('month'): | |
234 if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: | |
235 continue | |
236 widths[int(elem.attrib.get('type'))] = unicode(elem.text) | |
237 | |
238 days = data.setdefault('days', {}) | |
239 for ctxt in calendar.findall('days/dayContext'): | |
240 ctxts = days.setdefault(ctxt.attrib['type'], {}) | |
241 for width in ctxt.findall('dayWidth'): | |
242 widths = ctxts.setdefault(width.attrib['type'], {}) | |
243 for elem in width.findall('day'): | |
8
29f6f9a90f14
Pull in some supplemental data from the CLDR, for things like the first day of the week.
cmlenz
parents:
1
diff
changeset
|
244 dtype = weekdays[elem.attrib['type']] |
1 | 245 if 'draft' in elem.attrib and dtype in widths: |
246 continue | |
247 widths[dtype] = unicode(elem.text) | |
248 | |
249 quarters = data.setdefault('quarters', {}) | |
250 for ctxt in calendar.findall('quarters/quarterContext'): | |
251 ctxts = quarters.setdefault(ctxt.attrib['type'], {}) | |
252 for width in ctxt.findall('quarterWidth'): | |
253 widths = ctxts.setdefault(width.attrib['type'], {}) | |
254 for elem in width.findall('quarter'): | |
255 if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: | |
256 continue | |
257 widths[int(elem.attrib.get('type'))] = unicode(elem.text) | |
258 | |
259 eras = data.setdefault('eras', {}) | |
260 for width in calendar.findall('eras/*'): | |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
261 ewidth = { |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
262 'eraAbbr': 'abbreviated', |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
263 'eraNames': 'wide', |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
264 'eraNarrow': 'narrow', |
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
265 }[width.tag] |
1 | 266 widths = eras.setdefault(ewidth, {}) |
267 for elem in width.findall('era'): | |
268 if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: | |
269 continue | |
270 widths[int(elem.attrib.get('type'))] = unicode(elem.text) | |
271 | |
272 # AM/PM | |
273 periods = data.setdefault('periods', {}) | |
274 for elem in calendar.findall('am'): | |
275 if 'draft' in elem.attrib and elem.tag in periods: | |
276 continue | |
277 periods[elem.tag] = unicode(elem.text) | |
278 for elem in calendar.findall('pm'): | |
279 if 'draft' in elem.attrib and elem.tag in periods: | |
280 continue | |
281 periods[elem.tag] = unicode(elem.text) | |
282 | |
283 date_formats = data.setdefault('date_formats', {}) | |
284 for elem in calendar.findall('dateFormats/dateFormatLength'): | |
285 if 'draft' in elem.attrib and elem.attrib.get('type') in date_formats: | |
286 continue | |
287 try: | |
288 date_formats[elem.attrib.get('type')] = \ | |
9 | 289 dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern'))) |
1 | 290 except ValueError, e: |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
291 print>>sys.stderr, 'ERROR: %s' % e |
1 | 292 |
293 time_formats = data.setdefault('time_formats', {}) | |
294 for elem in calendar.findall('timeFormats/timeFormatLength'): | |
295 if 'draft' in elem.attrib and elem.attrib.get('type') in time_formats: | |
296 continue | |
297 try: | |
298 time_formats[elem.attrib.get('type')] = \ | |
9 | 299 dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern'))) |
1 | 300 except ValueError, e: |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
301 print>>sys.stderr, 'ERROR: %s' % e |
1 | 302 |
33 | 303 datetime_formats = data.setdefault('datetime_formats', {}) |
304 for elem in calendar.findall('dateTimeFormats/dateTimeFormatLength'): | |
305 if 'draft' in elem.attrib and elem.attrib.get('type') in datetime_formats: | |
306 continue | |
307 try: | |
308 datetime_formats[elem.attrib.get('type')] = \ | |
309 unicode(elem.findtext('dateTimeFormat/pattern')) | |
310 except ValueError, e: | |
311 print>>sys.stderr, 'ERROR: %s' % e | |
312 | |
1 | 313 # <numbers> |
314 | |
315 number_symbols = data.setdefault('number_symbols', {}) | |
316 for elem in tree.findall('//numbers/symbols/*'): | |
317 number_symbols[elem.tag] = unicode(elem.text) | |
318 | |
319 decimal_formats = data.setdefault('decimal_formats', {}) | |
320 for elem in tree.findall('//decimalFormats/decimalFormatLength'): | |
321 if 'draft' in elem.attrib and elem.attrib.get('type') in decimal_formats: | |
322 continue | |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
323 pattern = unicode(elem.findtext('decimalFormat/pattern')) |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
324 decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) |
1 | 325 |
326 scientific_formats = data.setdefault('scientific_formats', {}) | |
327 for elem in tree.findall('//scientificFormats/scientificFormatLength'): | |
328 if 'draft' in elem.attrib and elem.attrib.get('type') in scientific_formats: | |
329 continue | |
125 | 330 pattern = unicode(elem.findtext('scientificFormat/pattern')) |
331 scientific_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) | |
1 | 332 |
333 currency_formats = data.setdefault('currency_formats', {}) | |
334 for elem in tree.findall('//currencyFormats/currencyFormatLength'): | |
335 if 'draft' in elem.attrib and elem.attrib.get('type') in currency_formats: | |
336 continue | |
125 | 337 pattern = unicode(elem.findtext('currencyFormat/pattern')) |
338 currency_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) | |
1 | 339 |
340 percent_formats = data.setdefault('percent_formats', {}) | |
341 for elem in tree.findall('//percentFormats/percentFormatLength'): | |
342 if 'draft' in elem.attrib and elem.attrib.get('type') in percent_formats: | |
343 continue | |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
344 pattern = unicode(elem.findtext('percentFormat/pattern')) |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
345 percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) |
1 | 346 |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
347 currency_names = data.setdefault('currency_names', {}) |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
348 currency_symbols = data.setdefault('currency_symbols', {}) |
1 | 349 for elem in tree.findall('//currencies/currency'): |
26
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
350 name = elem.findtext('displayName') |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
351 if name: |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
352 currency_names[elem.attrib['type']] = unicode(name) |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
353 symbol = elem.findtext('symbol') |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
354 if symbol: |
6041782ea677
* Reduce size of locale data pickles by only storing the data provided by each locale itself, and merging inherited data at runtime.
cmlenz
parents:
22
diff
changeset
|
355 currency_symbols[elem.attrib['type']] = unicode(symbol) |
1 | 356 |
233
da97a3138239
Upgraded to CLDR 1.5 and improved timezone formatting.
cmlenz
parents:
125
diff
changeset
|
357 outfile = open(os.path.join(destdir, 'localedata', stem + '.dat'), 'wb') |
1 | 358 try: |
359 pickle.dump(data, outfile, 2) | |
360 finally: | |
361 outfile.close() | |
362 | |
363 if __name__ == '__main__': | |
364 main() |