cmlenz@1: #!/usr/bin/env python cmlenz@1: # -*- coding: utf-8 -*- cmlenz@1: # cmlenz@1: # Copyright (C) 2007 Edgewall Software cmlenz@1: # All rights reserved. cmlenz@1: # cmlenz@1: # This software is licensed as described in the file COPYING, which cmlenz@1: # you should have received as part of this distribution. The terms cmlenz@1: # are also available at http://babel.edgewall.org/wiki/License. cmlenz@1: # cmlenz@1: # This software consists of voluntary contributions made by many cmlenz@1: # individuals. For the exact contribution history, see the revision cmlenz@1: # history and logs, available at http://babel.edgewall.org/log/. cmlenz@1: cmlenz@1: import copy cmlenz@1: from optparse import OptionParser cmlenz@1: import os cmlenz@1: import pickle cmlenz@379: import re cmlenz@1: import sys cmlenz@1: try: cmlenz@1: from xml.etree.ElementTree import parse cmlenz@1: except ImportError: cmlenz@1: from elementtree.ElementTree import parse cmlenz@1: cmlenz@65: # Make sure we're using Babel source, and not some previously installed version cmlenz@65: sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..')) cmlenz@65: jonas@9: from babel import dates, numbers cmlenz@379: from babel.localedata import Alias cmlenz@1: cmlenz@15: weekdays = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5, cmlenz@15: 'sun': 6} cmlenz@8: cmlenz@8: try: cmlenz@8: any cmlenz@8: except NameError: cmlenz@8: def any(iterable): cmlenz@8: return filter(None, list(iterable)) cmlenz@8: cmlenz@379: cmlenz@1: def _text(elem): cmlenz@1: buf = [elem.text or ''] cmlenz@1: for child in elem: cmlenz@1: buf.append(_text(child)) cmlenz@1: buf.append(elem.tail or '') cmlenz@1: return u''.join(filter(None, buf)).strip() cmlenz@1: cmlenz@379: cmlenz@379: NAME_RE = re.compile(r"^\w+$") cmlenz@379: TYPE_ATTR_RE = re.compile(r"^\w+\[@type='(.*?)'\]$") cmlenz@379: cmlenz@379: NAME_MAP = { cmlenz@379: 'dateFormats': 'date_formats', cmlenz@379: 'dateTimeFormats': 'datetime_formats', cmlenz@379: 'eraAbbr': 'abbreviated', cmlenz@379: 'eraNames': 'wide', cmlenz@379: 'eraNarrow': 'narrow', cmlenz@379: 'timeFormats': 'time_formats' cmlenz@379: } cmlenz@379: cmlenz@379: def _translate_alias(ctxt, path): cmlenz@379: parts = path.split('/') cmlenz@379: keys = ctxt[:] cmlenz@379: for part in parts: cmlenz@379: if part == '..': cmlenz@379: keys.pop() cmlenz@379: else: cmlenz@379: match = TYPE_ATTR_RE.match(part) cmlenz@379: if match: cmlenz@379: keys.append(match.group(1)) cmlenz@379: else: cmlenz@379: assert NAME_RE.match(part) cmlenz@379: keys.append(NAME_MAP.get(part, part)) cmlenz@379: return keys cmlenz@379: cmlenz@379: cmlenz@1: def main(): cmlenz@1: parser = OptionParser(usage='%prog path/to/cldr') cmlenz@1: options, args = parser.parse_args() cmlenz@1: if len(args) != 1: cmlenz@1: parser.error('incorrect number of arguments') cmlenz@1: cmlenz@1: srcdir = args[0] cmlenz@1: destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), cmlenz@233: '..', 'babel') cmlenz@1: cmlenz@8: sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml')) cmlenz@8: cmlenz@346: # Import global data from the supplemental files cmlenz@233: global_data = {} cmlenz@233: cmlenz@233: territory_zones = global_data.setdefault('territory_zones', {}) cmlenz@233: zone_aliases = global_data.setdefault('zone_aliases', {}) cmlenz@233: zone_territories = global_data.setdefault('zone_territories', {}) cmlenz@233: for elem in sup.findall('//timezoneData/zoneFormatting/zoneItem'): cmlenz@233: tzid = elem.attrib['type'] cmlenz@233: territory_zones.setdefault(elem.attrib['territory'], []).append(tzid) cmlenz@233: zone_territories[tzid] = elem.attrib['territory'] cmlenz@233: if 'aliases' in elem.attrib: cmlenz@233: for alias in elem.attrib['aliases'].split(): cmlenz@233: zone_aliases[alias] = tzid cmlenz@233: cmlenz@346: # Import Metazone mapping cmlenz@346: meta_zones = global_data.setdefault('meta_zones', {}) cmlenz@346: tzsup = parse(os.path.join(srcdir, 'supplemental', 'metazoneInfo.xml')) cmlenz@346: for elem in tzsup.findall('//timezone'): cmlenz@346: for child in elem.findall('usesMetazone'): cmlenz@346: if 'to' not in child.attrib: # FIXME: support old mappings cmlenz@346: meta_zones[elem.attrib['type']] = child.attrib['mzone'] cmlenz@346: cmlenz@233: outfile = open(os.path.join(destdir, 'global.dat'), 'wb') cmlenz@233: try: cmlenz@233: pickle.dump(global_data, outfile, 2) cmlenz@233: finally: cmlenz@233: outfile.close() cmlenz@233: cmlenz@8: # build a territory containment mapping for inheritance cmlenz@8: regions = {} cmlenz@8: for elem in sup.findall('//territoryContainment/group'): cmlenz@8: regions[elem.attrib['type']] = elem.attrib['contains'].split() cmlenz@8: cmlenz@8: # Resolve territory containment cmlenz@8: territory_containment = {} cmlenz@8: region_items = regions.items() cmlenz@8: region_items.sort() cmlenz@8: for group, territory_list in region_items: cmlenz@8: for territory in territory_list: cmlenz@8: containers = territory_containment.setdefault(territory, set([])) cmlenz@8: if group in territory_containment: cmlenz@8: containers |= territory_containment[group] cmlenz@8: containers.add(group) cmlenz@8: cmlenz@1: filenames = os.listdir(os.path.join(srcdir, 'main')) cmlenz@1: filenames.remove('root.xml') cmlenz@1: filenames.sort(lambda a,b: len(a)-len(b)) cmlenz@1: filenames.insert(0, 'root.xml') cmlenz@1: cmlenz@1: for filename in filenames: cmlenz@1: print>>sys.stderr, 'Processing input file %r' % filename cmlenz@1: stem, ext = os.path.splitext(filename) cmlenz@1: if ext != '.xml': cmlenz@1: continue cmlenz@379: #if stem != 'root': cmlenz@379: # break cmlenz@1: cmlenz@26: tree = parse(os.path.join(srcdir, 'main', filename)) cmlenz@1: data = {} cmlenz@1: cmlenz@8: language = None cmlenz@8: elem = tree.find('//identity/language') cmlenz@8: if elem is not None: cmlenz@8: language = elem.attrib['type'] cmlenz@8: print>>sys.stderr, ' Language: %r' % language cmlenz@8: cmlenz@8: territory = None cmlenz@8: elem = tree.find('//identity/territory') cmlenz@8: if elem is not None: cmlenz@8: territory = elem.attrib['type'] cmlenz@13: else: cmlenz@13: territory = '001' # world cmlenz@8: print>>sys.stderr, ' Territory: %r' % territory cmlenz@8: regions = territory_containment.get(territory, []) cmlenz@8: print>>sys.stderr, ' Regions: %r' % regions cmlenz@8: cmlenz@1: # cmlenz@1: cmlenz@1: territories = data.setdefault('territories', {}) cmlenz@1: for elem in tree.findall('//territories/territory'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib['type'] in territories: cmlenz@1: continue cmlenz@1: territories[elem.attrib['type']] = _text(elem) cmlenz@1: cmlenz@1: languages = data.setdefault('languages', {}) cmlenz@1: for elem in tree.findall('//languages/language'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib['type'] in languages: cmlenz@1: continue cmlenz@1: languages[elem.attrib['type']] = _text(elem) cmlenz@1: cmlenz@1: variants = data.setdefault('variants', {}) cmlenz@1: for elem in tree.findall('//variants/variant'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib['type'] in variants: cmlenz@1: continue cmlenz@1: variants[elem.attrib['type']] = _text(elem) cmlenz@1: cmlenz@1: scripts = data.setdefault('scripts', {}) cmlenz@1: for elem in tree.findall('//scripts/script'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib['type'] in scripts: cmlenz@1: continue cmlenz@1: scripts[elem.attrib['type']] = _text(elem) cmlenz@1: cmlenz@1: # cmlenz@1: cmlenz@8: week_data = data.setdefault('week_data', {}) cmlenz@8: supelem = sup.find('//weekData') cmlenz@8: cmlenz@8: for elem in supelem.findall('minDays'): cmlenz@8: territories = elem.attrib['territories'].split() cmlenz@8: if territory in territories or any([r in territories for r in regions]): cmlenz@8: week_data['min_days'] = int(elem.attrib['count']) cmlenz@8: cmlenz@8: for elem in supelem.findall('firstDay'): cmlenz@8: territories = elem.attrib['territories'].split() cmlenz@8: if territory in territories or any([r in territories for r in regions]): cmlenz@8: week_data['first_day'] = weekdays[elem.attrib['day']] cmlenz@8: cmlenz@8: for elem in supelem.findall('weekendStart'): cmlenz@8: territories = elem.attrib['territories'].split() cmlenz@8: if territory in territories or any([r in territories for r in regions]): cmlenz@8: week_data['weekend_start'] = weekdays[elem.attrib['day']] cmlenz@8: cmlenz@8: for elem in supelem.findall('weekendEnd'): cmlenz@8: territories = elem.attrib['territories'].split() cmlenz@8: if territory in territories or any([r in territories for r in regions]): cmlenz@8: week_data['weekend_end'] = weekdays[elem.attrib['day']] cmlenz@8: cmlenz@233: zone_formats = data.setdefault('zone_formats', {}) cmlenz@233: for elem in tree.findall('//timeZoneNames/gmtFormat'): cmlenz@379: if 'draft' not in elem.attrib and 'alt' not in elem.attrib: cmlenz@233: zone_formats['gmt'] = unicode(elem.text).replace('{0}', '%s') cmlenz@233: break cmlenz@233: for elem in tree.findall('//timeZoneNames/regionFormat'): cmlenz@379: if 'draft' not in elem.attrib and 'alt' not in elem.attrib: cmlenz@233: zone_formats['region'] = unicode(elem.text).replace('{0}', '%s') cmlenz@233: break cmlenz@233: for elem in tree.findall('//timeZoneNames/fallbackFormat'): cmlenz@379: if 'draft' not in elem.attrib and 'alt' not in elem.attrib: cmlenz@233: zone_formats['fallback'] = unicode(elem.text) \ cmlenz@233: .replace('{0}', '%(0)s').replace('{1}', '%(1)s') cmlenz@233: break cmlenz@233: cmlenz@1: time_zones = data.setdefault('time_zones', {}) cmlenz@1: for elem in tree.findall('//timeZoneNames/zone'): cmlenz@28: info = {} cmlenz@28: city = elem.findtext('exemplarCity') cmlenz@28: if city: cmlenz@28: info['city'] = unicode(city) cmlenz@28: for child in elem.findall('long/*'): cmlenz@28: info.setdefault('long', {})[child.tag] = unicode(child.text) cmlenz@28: for child in elem.findall('short/*'): cmlenz@28: info.setdefault('short', {})[child.tag] = unicode(child.text) cmlenz@28: time_zones[elem.attrib['type']] = info cmlenz@1: cmlenz@233: meta_zones = data.setdefault('meta_zones', {}) cmlenz@233: for elem in tree.findall('//timeZoneNames/metazone'): cmlenz@233: info = {} cmlenz@233: city = elem.findtext('exemplarCity') cmlenz@233: if city: cmlenz@233: info['city'] = unicode(city) cmlenz@233: for child in elem.findall('long/*'): cmlenz@233: info.setdefault('long', {})[child.tag] = unicode(child.text) cmlenz@233: for child in elem.findall('short/*'): cmlenz@233: info.setdefault('short', {})[child.tag] = unicode(child.text) cmlenz@233: info['common'] = elem.findtext('commonlyUsed') == 'true' cmlenz@233: meta_zones[elem.attrib['type']] = info cmlenz@34: cmlenz@1: for calendar in tree.findall('//calendars/calendar'): cmlenz@1: if calendar.attrib['type'] != 'gregorian': cmlenz@1: # TODO: support other calendar types cmlenz@1: continue cmlenz@1: cmlenz@1: months = data.setdefault('months', {}) cmlenz@1: for ctxt in calendar.findall('months/monthContext'): cmlenz@379: ctxt_type = ctxt.attrib['type'] cmlenz@379: ctxts = months.setdefault(ctxt_type, {}) cmlenz@1: for width in ctxt.findall('monthWidth'): cmlenz@379: width_type = width.attrib['type'] cmlenz@379: widths = ctxts.setdefault(width_type, {}) cmlenz@379: for elem in width.getiterator(): cmlenz@379: if elem.tag == 'month': cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and int(elem.attrib['type']) in widths: cmlenz@379: continue cmlenz@379: widths[int(elem.attrib.get('type'))] = unicode(elem.text) cmlenz@379: elif elem.tag == 'alias': cmlenz@379: ctxts[width_type] = Alias( cmlenz@379: _translate_alias(['months', ctxt_type, width_type], cmlenz@379: elem.attrib['path']) cmlenz@379: ) cmlenz@1: cmlenz@1: days = data.setdefault('days', {}) cmlenz@1: for ctxt in calendar.findall('days/dayContext'): cmlenz@379: ctxt_type = ctxt.attrib['type'] cmlenz@379: ctxts = days.setdefault(ctxt_type, {}) cmlenz@1: for width in ctxt.findall('dayWidth'): cmlenz@379: width_type = width.attrib['type'] cmlenz@379: widths = ctxts.setdefault(width_type, {}) cmlenz@379: for elem in width.getiterator(): cmlenz@379: if elem.tag == 'day': cmlenz@379: dtype = weekdays[elem.attrib['type']] cmlenz@379: if ('draft' in elem.attrib or 'alt' not in elem.attrib) \ cmlenz@379: and dtype in widths: cmlenz@379: continue cmlenz@379: widths[dtype] = unicode(elem.text) cmlenz@379: elif elem.tag == 'alias': cmlenz@379: ctxts[width_type] = Alias( cmlenz@379: _translate_alias(['days', ctxt_type, width_type], cmlenz@379: elem.attrib['path']) cmlenz@379: ) cmlenz@1: cmlenz@1: quarters = data.setdefault('quarters', {}) cmlenz@1: for ctxt in calendar.findall('quarters/quarterContext'): cmlenz@379: ctxt_type = ctxt.attrib['type'] cmlenz@1: ctxts = quarters.setdefault(ctxt.attrib['type'], {}) cmlenz@1: for width in ctxt.findall('quarterWidth'): cmlenz@379: width_type = width.attrib['type'] cmlenz@379: widths = ctxts.setdefault(width_type, {}) cmlenz@379: for elem in width.getiterator(): cmlenz@379: if elem.tag == 'quarter': cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and int(elem.attrib['type']) in widths: cmlenz@379: continue cmlenz@379: widths[int(elem.attrib['type'])] = unicode(elem.text) cmlenz@379: elif elem.tag == 'alias': cmlenz@379: ctxts[width_type] = Alias( cmlenz@379: _translate_alias(['quarters', ctxt_type, width_type], cmlenz@379: elem.attrib['path']) cmlenz@379: ) cmlenz@1: cmlenz@1: eras = data.setdefault('eras', {}) cmlenz@1: for width in calendar.findall('eras/*'): cmlenz@379: width_type = NAME_MAP[width.tag] cmlenz@379: widths = eras.setdefault(width_type, {}) cmlenz@379: for elem in width.getiterator(): cmlenz@379: if elem.tag == 'era': cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and int(elem.attrib['type']) in widths: cmlenz@379: continue cmlenz@379: widths[int(elem.attrib.get('type'))] = unicode(elem.text) cmlenz@379: elif elem.tag == 'alias': cmlenz@379: eras[width_type] = Alias( cmlenz@379: _translate_alias(['eras', width_type], cmlenz@379: elem.attrib['path']) cmlenz@379: ) cmlenz@1: cmlenz@1: # AM/PM cmlenz@1: periods = data.setdefault('periods', {}) cmlenz@1: for elem in calendar.findall('am'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.tag in periods: cmlenz@1: continue cmlenz@1: periods[elem.tag] = unicode(elem.text) cmlenz@1: for elem in calendar.findall('pm'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.tag in periods: cmlenz@1: continue cmlenz@1: periods[elem.tag] = unicode(elem.text) cmlenz@1: cmlenz@1: date_formats = data.setdefault('date_formats', {}) cmlenz@379: for format in calendar.findall('dateFormats'): cmlenz@379: for elem in format.getiterator(): cmlenz@379: if elem.tag == 'dateFormatLength': cmlenz@379: if 'draft' in elem.attrib and \ cmlenz@379: elem.attrib.get('type') in date_formats: cmlenz@379: continue cmlenz@379: try: cmlenz@379: date_formats[elem.attrib.get('type')] = \ cmlenz@379: dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern'))) cmlenz@379: except ValueError, e: cmlenz@379: print>>sys.stderr, 'ERROR: %s' % e cmlenz@379: elif elem.tag == 'alias': cmlenz@379: date_formats = Alias(_translate_alias( cmlenz@379: ['date_formats'], elem.attrib['path']) cmlenz@379: ) cmlenz@1: cmlenz@1: time_formats = data.setdefault('time_formats', {}) cmlenz@379: for format in calendar.findall('timeFormats'): cmlenz@379: for elem in format.getiterator(): cmlenz@379: if elem.tag == 'timeFormatLength': cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib.get('type') in time_formats: cmlenz@379: continue cmlenz@379: try: cmlenz@379: time_formats[elem.attrib.get('type')] = \ cmlenz@379: dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern'))) cmlenz@379: except ValueError, e: cmlenz@379: print>>sys.stderr, 'ERROR: %s' % e cmlenz@379: elif elem.tag == 'alias': cmlenz@379: time_formats = Alias(_translate_alias( cmlenz@379: ['time_formats'], elem.attrib['path']) cmlenz@379: ) cmlenz@1: cmlenz@33: datetime_formats = data.setdefault('datetime_formats', {}) cmlenz@379: for format in calendar.findall('dateTimeFormats'): cmlenz@379: for elem in format.getiterator(): cmlenz@379: if elem.tag == 'dateTimeFormatLength': cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib.get('type') in datetime_formats: cmlenz@379: continue cmlenz@379: try: cmlenz@379: datetime_formats[elem.attrib.get('type')] = \ cmlenz@379: unicode(elem.findtext('dateTimeFormat/pattern')) cmlenz@379: except ValueError, e: cmlenz@379: print>>sys.stderr, 'ERROR: %s' % e cmlenz@379: elif elem.tag == 'alias': cmlenz@379: datetime_formats = Alias(_translate_alias( cmlenz@379: ['datetime_formats'], elem.attrib['path']) cmlenz@379: ) cmlenz@33: cmlenz@1: # cmlenz@1: cmlenz@1: number_symbols = data.setdefault('number_symbols', {}) cmlenz@1: for elem in tree.findall('//numbers/symbols/*'): cmlenz@1: number_symbols[elem.tag] = unicode(elem.text) cmlenz@1: cmlenz@1: decimal_formats = data.setdefault('decimal_formats', {}) cmlenz@1: for elem in tree.findall('//decimalFormats/decimalFormatLength'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib.get('type') in decimal_formats: cmlenz@1: continue cmlenz@26: pattern = unicode(elem.findtext('decimalFormat/pattern')) cmlenz@26: decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@1: cmlenz@1: scientific_formats = data.setdefault('scientific_formats', {}) cmlenz@1: for elem in tree.findall('//scientificFormats/scientificFormatLength'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib.get('type') in scientific_formats: cmlenz@1: continue cmlenz@125: pattern = unicode(elem.findtext('scientificFormat/pattern')) cmlenz@125: scientific_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@1: cmlenz@1: currency_formats = data.setdefault('currency_formats', {}) cmlenz@1: for elem in tree.findall('//currencyFormats/currencyFormatLength'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib.get('type') in currency_formats: cmlenz@1: continue cmlenz@125: pattern = unicode(elem.findtext('currencyFormat/pattern')) cmlenz@125: currency_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@1: cmlenz@1: percent_formats = data.setdefault('percent_formats', {}) cmlenz@1: for elem in tree.findall('//percentFormats/percentFormatLength'): cmlenz@379: if ('draft' in elem.attrib or 'alt' in elem.attrib) \ cmlenz@379: and elem.attrib.get('type') in percent_formats: cmlenz@1: continue cmlenz@26: pattern = unicode(elem.findtext('percentFormat/pattern')) cmlenz@26: percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern) cmlenz@1: cmlenz@26: currency_names = data.setdefault('currency_names', {}) cmlenz@26: currency_symbols = data.setdefault('currency_symbols', {}) cmlenz@1: for elem in tree.findall('//currencies/currency'): cmlenz@26: name = elem.findtext('displayName') cmlenz@26: if name: cmlenz@26: currency_names[elem.attrib['type']] = unicode(name) cmlenz@26: symbol = elem.findtext('symbol') cmlenz@26: if symbol: cmlenz@26: currency_symbols[elem.attrib['type']] = unicode(symbol) cmlenz@1: cmlenz@233: outfile = open(os.path.join(destdir, 'localedata', stem + '.dat'), 'wb') cmlenz@1: try: cmlenz@1: pickle.dump(data, outfile, 2) cmlenz@1: finally: cmlenz@1: outfile.close() cmlenz@1: cmlenz@379: cmlenz@1: if __name__ == '__main__': cmlenz@1: main()