cmlenz@3: #!/usr/bin/env python cmlenz@3: # -*- coding: utf-8 -*- cmlenz@3: # cmlenz@3: # Copyright (C) 2007 Edgewall Software cmlenz@3: # All rights reserved. cmlenz@3: # cmlenz@3: # This software is licensed as described in the file COPYING, which cmlenz@3: # you should have received as part of this distribution. The terms cmlenz@3: # are also available at http://babel.edgewall.org/wiki/License. cmlenz@3: # cmlenz@3: # This software consists of voluntary contributions made by many cmlenz@3: # individuals. For the exact contribution history, see the revision cmlenz@3: # history and logs, available at http://babel.edgewall.org/log/. cmlenz@3: cmlenz@3: import copy cmlenz@3: from optparse import OptionParser cmlenz@3: import os cmlenz@3: import pickle cmlenz@377: import re cmlenz@3: import sys cmlenz@3: try: cmlenz@3: from xml.etree.ElementTree import parse cmlenz@3: except ImportError: cmlenz@3: from elementtree.ElementTree import parse cmlenz@3: cmlenz@67: # Make sure we're using Babel source, and not some previously installed version cmlenz@67: sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..')) cmlenz@67: jonas@11: from babel import dates, numbers cmlenz@377: from babel.localedata import Alias cmlenz@3: cmlenz@17: weekdays = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5, cmlenz@17: 'sun': 6} cmlenz@10: cmlenz@10: try: cmlenz@10: any cmlenz@10: except NameError: cmlenz@10: def any(iterable): cmlenz@10: return filter(None, list(iterable)) cmlenz@10: cmlenz@377: cmlenz@3: def _text(elem): cmlenz@3: buf = [elem.text or ''] cmlenz@3: for child in elem: cmlenz@3: buf.append(_text(child)) cmlenz@3: buf.append(elem.tail or '') cmlenz@3: return u''.join(filter(None, buf)).strip() cmlenz@3: cmlenz@377: cmlenz@377: NAME_RE = re.compile(r"^\w+$") cmlenz@377: TYPE_ATTR_RE = re.compile(r"^\w+\[@type='(.*?)'\]$") cmlenz@377: cmlenz@377: NAME_MAP = { cmlenz@377: 'dateFormats': 'date_formats', cmlenz@377: 'dateTimeFormats': 'datetime_formats', cmlenz@377: 'eraAbbr': 'abbreviated', cmlenz@377: 'eraNames': 'wide', cmlenz@377: 'eraNarrow': 'narrow', cmlenz@377: 'timeFormats': 'time_formats' cmlenz@377: } cmlenz@377: cmlenz@377: def _translate_alias(ctxt, path): cmlenz@377: parts = path.split('/') cmlenz@377: keys = ctxt[:] cmlenz@377: for part in parts: cmlenz@377: if part == '..': cmlenz@377: keys.pop() cmlenz@377: else: cmlenz@377: match = TYPE_ATTR_RE.match(part) cmlenz@377: if match: cmlenz@377: keys.append(match.group(1)) cmlenz@377: else: cmlenz@377: assert NAME_RE.match(part) cmlenz@377: keys.append(NAME_MAP.get(part, part)) cmlenz@377: return keys cmlenz@377: cmlenz@377: cmlenz@3: def main(): cmlenz@3: parser = OptionParser(usage='%prog path/to/cldr') cmlenz@3: options, args = parser.parse_args() cmlenz@3: if len(args) != 1: cmlenz@3: parser.error('incorrect number of arguments') cmlenz@3: cmlenz@3: srcdir = args[0] cmlenz@3: destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), cmlenz@235: '..', 'babel') cmlenz@3: cmlenz@10: sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml')) cmlenz@10: cmlenz@347: # Import global data from the supplemental files cmlenz@235: global_data = {} cmlenz@235: cmlenz@235: territory_zones = global_data.setdefault('territory_zones', {}) cmlenz@235: zone_aliases = global_data.setdefault('zone_aliases', {}) cmlenz@235: zone_territories = global_data.setdefault('zone_territories', {}) cmlenz@235: for elem in sup.findall('//timezoneData/zoneFormatting/zoneItem'): cmlenz@235: tzid = elem.attrib['type'] cmlenz@235: territory_zones.setdefault(elem.attrib['territory'], []).append(tzid) cmlenz@235: zone_territories[tzid] = elem.attrib['territory'] cmlenz@235: if 'aliases' in elem.attrib: cmlenz@235: for alias in elem.attrib['aliases'].split(): cmlenz@235: zone_aliases[alias] = tzid cmlenz@235: cmlenz@347: # Import Metazone mapping cmlenz@347: meta_zones = global_data.setdefault('meta_zones', {}) cmlenz@347: tzsup = parse(os.path.join(srcdir, 'supplemental', 'metazoneInfo.xml')) cmlenz@347: for elem in tzsup.findall('//timezone'): cmlenz@347: for child in elem.findall('usesMetazone'): cmlenz@347: if 'to' not in child.attrib: # FIXME: support old mappings cmlenz@347: meta_zones[elem.attrib['type']] = child.attrib['mzone'] cmlenz@347: cmlenz@235: outfile = open(os.path.join(destdir, 'global.dat'), 'wb') cmlenz@235: try: cmlenz@235: pickle.dump(global_data, outfile, 2) cmlenz@235: finally: cmlenz@235: outfile.close() cmlenz@235: cmlenz@10: # build a territory containment mapping for inheritance cmlenz@10: regions = {} cmlenz@10: for elem in sup.findall('//territoryContainment/group'): cmlenz@10: regions[elem.attrib['type']] = elem.attrib['contains'].split() cmlenz@10: cmlenz@10: # Resolve territory containment cmlenz@10: territory_containment = {} cmlenz@10: region_items = regions.items() cmlenz@10: region_items.sort() cmlenz@10: for group, territory_list in region_items: cmlenz@10: for territory in territory_list: cmlenz@10: containers = territory_containment.setdefault(territory, set([])) cmlenz@10: if group in territory_containment: cmlenz@10: containers |= territory_containment[group] cmlenz@10: containers.add(group) cmlenz@10: cmlenz@3: filenames = os.listdir(os.path.join(srcdir, 'main')) cmlenz@3: filenames.remove('root.xml') cmlenz@3: filenames.sort(lambda a,b: len(a)-len(b)) cmlenz@3: filenames.insert(0, 'root.xml') cmlenz@3: cmlenz@3: for filename in filenames: cmlenz@3: stem, ext = os.path.splitext(filename) cmlenz@3: if ext != '.xml': cmlenz@3: continue cmlenz@3: cmlenz@387: print>>sys.stderr, 'Processing input file %r' % filename cmlenz@28: tree = parse(os.path.join(srcdir, 'main', filename)) cmlenz@3: data = {} cmlenz@3: cmlenz@10: language = None cmlenz@10: elem = tree.find('//identity/language') cmlenz@10: if elem is not None: cmlenz@10: language = elem.attrib['type'] cmlenz@10: print>>sys.stderr, ' Language: %r' % language cmlenz@10: cmlenz@10: territory = None cmlenz@10: elem = tree.find('//identity/territory') cmlenz@10: if elem is not None: cmlenz@10: territory = elem.attrib['type'] cmlenz@15: else: cmlenz@15: territory = '001' # world cmlenz@10: print>>sys.stderr, ' Territory: %r' % territory cmlenz@10: regions = territory_containment.get(territory, []) cmlenz@10: print>>sys.stderr, ' Regions: %r' % regions cmlenz@10: cmlenz@3: # cmlenz@3: cmlenz@3: territories = data.setdefault('territories', {}) cmlenz@3: for elem in tree.findall('//territories/territory'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib['type'] in territories: cmlenz@3: continue cmlenz@3: territories[elem.attrib['type']] = _text(elem) cmlenz@3: cmlenz@3: languages = data.setdefault('languages', {}) cmlenz@3: for elem in tree.findall('//languages/language'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib['type'] in languages: cmlenz@3: continue cmlenz@3: languages[elem.attrib['type']] = _text(elem) cmlenz@3: cmlenz@3: variants = data.setdefault('variants', {}) cmlenz@3: for elem in tree.findall('//variants/variant'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib['type'] in variants: cmlenz@3: continue cmlenz@3: variants[elem.attrib['type']] = _text(elem) cmlenz@3: cmlenz@3: scripts = data.setdefault('scripts', {}) cmlenz@3: for elem in tree.findall('//scripts/script'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib['type'] in scripts: cmlenz@3: continue cmlenz@3: scripts[elem.attrib['type']] = _text(elem) cmlenz@3: cmlenz@3: # cmlenz@3: cmlenz@10: week_data = data.setdefault('week_data', {}) cmlenz@10: supelem = sup.find('//weekData') cmlenz@10: cmlenz@10: for elem in supelem.findall('minDays'): cmlenz@10: territories = elem.attrib['territories'].split() cmlenz@10: if territory in territories or any([r in territories for r in regions]): cmlenz@10: week_data['min_days'] = int(elem.attrib['count']) cmlenz@10: cmlenz@10: for elem in supelem.findall('firstDay'): cmlenz@10: territories = elem.attrib['territories'].split() cmlenz@10: if territory in territories or any([r in territories for r in regions]): cmlenz@10: week_data['first_day'] = weekdays[elem.attrib['day']] cmlenz@10: cmlenz@10: for elem in supelem.findall('weekendStart'): cmlenz@10: territories = elem.attrib['territories'].split() cmlenz@10: if territory in territories or any([r in territories for r in regions]): cmlenz@10: week_data['weekend_start'] = weekdays[elem.attrib['day']] cmlenz@10: cmlenz@10: for elem in supelem.findall('weekendEnd'): cmlenz@10: territories = elem.attrib['territories'].split() cmlenz@10: if territory in territories or any([r in territories for r in regions]): cmlenz@10: week_data['weekend_end'] = weekdays[elem.attrib['day']] cmlenz@10: cmlenz@235: zone_formats = data.setdefault('zone_formats', {}) cmlenz@235: for elem in tree.findall('//timeZoneNames/gmtFormat'): cmlenz@377: if 'draft' not in elem.attrib and 'alt' not in elem.attrib: cmlenz@235: zone_formats['gmt'] = unicode(elem.text).replace('{0}', '%s') cmlenz@235: break cmlenz@235: for elem in tree.findall('//timeZoneNames/regionFormat'): cmlenz@377: if 'draft' not in elem.attrib and 'alt' not in elem.attrib: cmlenz@235: zone_formats['region'] = unicode(elem.text).replace('{0}', '%s') cmlenz@235: break cmlenz@235: for elem in tree.findall('//timeZoneNames/fallbackFormat'): cmlenz@377: if 'draft' not in elem.attrib and 'alt' not in elem.attrib: cmlenz@235: zone_formats['fallback'] = unicode(elem.text) \ cmlenz@235: .replace('{0}', '%(0)s').replace('{1}', '%(1)s') cmlenz@235: break cmlenz@235: cmlenz@3: time_zones = data.setdefault('time_zones', {}) cmlenz@3: for elem in tree.findall('//timeZoneNames/zone'): cmlenz@30: info = {} cmlenz@30: city = elem.findtext('exemplarCity') cmlenz@30: if city: cmlenz@30: info['city'] = unicode(city) cmlenz@30: for child in elem.findall('long/*'): cmlenz@30: info.setdefault('long', {})[child.tag] = unicode(child.text) cmlenz@30: for child in elem.findall('short/*'): cmlenz@30: info.setdefault('short', {})[child.tag] = unicode(child.text) cmlenz@30: time_zones[elem.attrib['type']] = info cmlenz@3: cmlenz@235: meta_zones = data.setdefault('meta_zones', {}) cmlenz@235: for elem in tree.findall('//timeZoneNames/metazone'): cmlenz@235: info = {} cmlenz@235: city = elem.findtext('exemplarCity') cmlenz@235: if city: cmlenz@235: info['city'] = unicode(city) cmlenz@235: for child in elem.findall('long/*'): cmlenz@235: info.setdefault('long', {})[child.tag] = unicode(child.text) cmlenz@235: for child in elem.findall('short/*'): cmlenz@235: info.setdefault('short', {})[child.tag] = unicode(child.text) cmlenz@235: info['common'] = elem.findtext('commonlyUsed') == 'true' cmlenz@235: meta_zones[elem.attrib['type']] = info cmlenz@36: cmlenz@3: for calendar in tree.findall('//calendars/calendar'): cmlenz@3: if calendar.attrib['type'] != 'gregorian': cmlenz@3: # TODO: support other calendar types cmlenz@3: continue cmlenz@3: cmlenz@3: months = data.setdefault('months', {}) cmlenz@3: for ctxt in calendar.findall('months/monthContext'): cmlenz@377: ctxt_type = ctxt.attrib['type'] cmlenz@377: ctxts = months.setdefault(ctxt_type, {}) cmlenz@3: for width in ctxt.findall('monthWidth'): cmlenz@377: width_type = width.attrib['type'] cmlenz@377: widths = ctxts.setdefault(width_type, {}) cmlenz@377: for elem in width.getiterator(): cmlenz@377: if elem.tag == 'month': cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and int(elem.attrib['type']) in widths: cmlenz@377: continue cmlenz@377: widths[int(elem.attrib.get('type'))] = unicode(elem.text) cmlenz@377: elif elem.tag == 'alias': cmlenz@377: ctxts[width_type] = Alias( cmlenz@377: _translate_alias(['months', ctxt_type, width_type], cmlenz@377: elem.attrib['path']) cmlenz@377: ) cmlenz@3: cmlenz@3: days = data.setdefault('days', {}) cmlenz@3: for ctxt in calendar.findall('days/dayContext'): cmlenz@377: ctxt_type = ctxt.attrib['type'] cmlenz@377: ctxts = days.setdefault(ctxt_type, {}) cmlenz@3: for width in ctxt.findall('dayWidth'): cmlenz@377: width_type = width.attrib['type'] cmlenz@377: widths = ctxts.setdefault(width_type, {}) cmlenz@377: for elem in width.getiterator(): cmlenz@377: if elem.tag == 'day': cmlenz@377: dtype = weekdays[elem.attrib['type']] cmlenz@377: if ('draft' in elem.attrib or 'alt' not in elem.attrib) \ cmlenz@377: and dtype in widths: cmlenz@377: continue cmlenz@377: widths[dtype] = unicode(elem.text) cmlenz@377: elif elem.tag == 'alias': cmlenz@377: ctxts[width_type] = Alias( cmlenz@377: _translate_alias(['days', ctxt_type, width_type], cmlenz@377: elem.attrib['path']) cmlenz@377: ) cmlenz@3: cmlenz@3: quarters = data.setdefault('quarters', {}) cmlenz@3: for ctxt in calendar.findall('quarters/quarterContext'): cmlenz@377: ctxt_type = ctxt.attrib['type'] cmlenz@3: ctxts = quarters.setdefault(ctxt.attrib['type'], {}) cmlenz@3: for width in ctxt.findall('quarterWidth'): cmlenz@377: width_type = width.attrib['type'] cmlenz@377: widths = ctxts.setdefault(width_type, {}) cmlenz@377: for elem in width.getiterator(): cmlenz@377: if elem.tag == 'quarter': cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and int(elem.attrib['type']) in widths: cmlenz@377: continue cmlenz@377: widths[int(elem.attrib['type'])] = unicode(elem.text) cmlenz@377: elif elem.tag == 'alias': cmlenz@377: ctxts[width_type] = Alias( cmlenz@377: _translate_alias(['quarters', ctxt_type, width_type], cmlenz@377: elem.attrib['path']) cmlenz@377: ) cmlenz@3: cmlenz@3: eras = data.setdefault('eras', {}) cmlenz@3: for width in calendar.findall('eras/*'): cmlenz@377: width_type = NAME_MAP[width.tag] cmlenz@377: widths = eras.setdefault(width_type, {}) cmlenz@377: for elem in width.getiterator(): cmlenz@377: if elem.tag == 'era': cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and int(elem.attrib['type']) in widths: cmlenz@377: continue cmlenz@377: widths[int(elem.attrib.get('type'))] = unicode(elem.text) cmlenz@377: elif elem.tag == 'alias': cmlenz@377: eras[width_type] = Alias( cmlenz@377: _translate_alias(['eras', width_type], cmlenz@377: elem.attrib['path']) cmlenz@377: ) cmlenz@3: cmlenz@3: # AM/PM cmlenz@3: periods = data.setdefault('periods', {}) cmlenz@3: for elem in calendar.findall('am'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.tag in periods: cmlenz@3: continue cmlenz@3: periods[elem.tag] = unicode(elem.text) cmlenz@3: for elem in calendar.findall('pm'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.tag in periods: cmlenz@3: continue cmlenz@3: periods[elem.tag] = unicode(elem.text) cmlenz@3: cmlenz@3: date_formats = data.setdefault('date_formats', {}) cmlenz@377: for format in calendar.findall('dateFormats'): cmlenz@377: for elem in format.getiterator(): cmlenz@377: if elem.tag == 'dateFormatLength': cmlenz@377: if 'draft' in elem.attrib and \ cmlenz@377: elem.attrib.get('type') in date_formats: cmlenz@377: continue cmlenz@377: try: cmlenz@377: date_formats[elem.attrib.get('type')] = \ cmlenz@377: dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern'))) cmlenz@377: except ValueError, e: cmlenz@377: print>>sys.stderr, 'ERROR: %s' % e cmlenz@377: elif elem.tag == 'alias': cmlenz@377: date_formats = Alias(_translate_alias( cmlenz@377: ['date_formats'], elem.attrib['path']) cmlenz@377: ) cmlenz@3: cmlenz@3: time_formats = data.setdefault('time_formats', {}) cmlenz@377: for format in calendar.findall('timeFormats'): cmlenz@377: for elem in format.getiterator(): cmlenz@377: if elem.tag == 'timeFormatLength': cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib.get('type') in time_formats: cmlenz@377: continue cmlenz@377: try: cmlenz@377: time_formats[elem.attrib.get('type')] = \ cmlenz@377: dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern'))) cmlenz@377: except ValueError, e: cmlenz@377: print>>sys.stderr, 'ERROR: %s' % e cmlenz@377: elif elem.tag == 'alias': cmlenz@377: time_formats = Alias(_translate_alias( cmlenz@377: ['time_formats'], elem.attrib['path']) cmlenz@377: ) cmlenz@3: cmlenz@35: datetime_formats = data.setdefault('datetime_formats', {}) cmlenz@377: for format in calendar.findall('dateTimeFormats'): cmlenz@377: for elem in format.getiterator(): cmlenz@377: if elem.tag == 'dateTimeFormatLength': cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib.get('type') in datetime_formats: cmlenz@377: continue cmlenz@377: try: cmlenz@377: datetime_formats[elem.attrib.get('type')] = \ cmlenz@377: unicode(elem.findtext('dateTimeFormat/pattern')) cmlenz@377: except ValueError, e: cmlenz@377: print>>sys.stderr, 'ERROR: %s' % e cmlenz@377: elif elem.tag == 'alias': cmlenz@377: datetime_formats = Alias(_translate_alias( cmlenz@377: ['datetime_formats'], elem.attrib['path']) cmlenz@377: ) cmlenz@35: cmlenz@3: # cmlenz@3: cmlenz@3: number_symbols = data.setdefault('number_symbols', {}) cmlenz@3: for elem in tree.findall('//numbers/symbols/*'): cmlenz@3: number_symbols[elem.tag] = unicode(elem.text) cmlenz@3: cmlenz@3: decimal_formats = data.setdefault('decimal_formats', {}) cmlenz@3: for elem in tree.findall('//decimalFormats/decimalFormatLength'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib.get('type') in decimal_formats: cmlenz@3: continue cmlenz@28: pattern = unicode(elem.findtext('decimalFormat/pattern')) cmlenz@28: decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@3: cmlenz@3: scientific_formats = data.setdefault('scientific_formats', {}) cmlenz@3: for elem in tree.findall('//scientificFormats/scientificFormatLength'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib.get('type') in scientific_formats: cmlenz@3: continue cmlenz@127: pattern = unicode(elem.findtext('scientificFormat/pattern')) cmlenz@127: scientific_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@3: cmlenz@3: currency_formats = data.setdefault('currency_formats', {}) cmlenz@3: for elem in tree.findall('//currencyFormats/currencyFormatLength'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib.get('type') in currency_formats: cmlenz@3: continue cmlenz@127: pattern = unicode(elem.findtext('currencyFormat/pattern')) cmlenz@127: currency_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@3: cmlenz@3: percent_formats = data.setdefault('percent_formats', {}) cmlenz@3: for elem in tree.findall('//percentFormats/percentFormatLength'): cmlenz@377: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@377: and elem.attrib.get('type') in percent_formats: cmlenz@3: continue cmlenz@28: pattern = unicode(elem.findtext('percentFormat/pattern')) cmlenz@28: percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@3: cmlenz@28: currency_names = data.setdefault('currency_names', {}) cmlenz@28: currency_symbols = data.setdefault('currency_symbols', {}) cmlenz@3: for elem in tree.findall('//currencies/currency'): cmlenz@387: code = elem.attrib['type'] cmlenz@387: # TODO: support plural rules for currency name selection cmlenz@387: for name in elem.findall('displayName'): cmlenz@387: if ('draft' in name.attrib or 'count' in name.attrib) \ cmlenz@387: and code in currency_names: cmlenz@387: continue cmlenz@387: currency_names[code] = unicode(name.text) cmlenz@387: # TODO: support choice patterns for currency symbol selection cmlenz@387: symbol = elem.find('symbol') cmlenz@387: if symbol is not None and 'draft' not in symbol.attrib \ cmlenz@387: and 'choice' not in symbol.attrib: cmlenz@387: currency_symbols[code] = unicode(symbol.text) cmlenz@3: cmlenz@235: outfile = open(os.path.join(destdir, 'localedata', stem + '.dat'), 'wb') cmlenz@3: try: cmlenz@3: pickle.dump(data, outfile, 2) cmlenz@3: finally: cmlenz@3: outfile.close() cmlenz@3: cmlenz@377: cmlenz@3: if __name__ == '__main__': cmlenz@3: main()