cmlenz@3: #!/usr/bin/env python
cmlenz@3: # -*- coding: utf-8 -*-
cmlenz@3: #
cmlenz@3: # Copyright (C) 2007 Edgewall Software
cmlenz@3: # All rights reserved.
cmlenz@3: #
cmlenz@3: # This software is licensed as described in the file COPYING, which
cmlenz@3: # you should have received as part of this distribution. The terms
cmlenz@3: # are also available at http://babel.edgewall.org/wiki/License.
cmlenz@3: #
cmlenz@3: # This software consists of voluntary contributions made by many
cmlenz@3: # individuals. For the exact contribution history, see the revision
cmlenz@3: # history and logs, available at http://babel.edgewall.org/log/.
cmlenz@3: 
cmlenz@3: import copy
cmlenz@3: from optparse import OptionParser
cmlenz@3: import os
cmlenz@3: import pickle
cmlenz@3: import sys
cmlenz@3: try:
cmlenz@3:     from xml.etree.ElementTree import parse
cmlenz@3: except ImportError:
cmlenz@3:     from elementtree.ElementTree import parse
cmlenz@3: 
jonas@11: from babel import dates, numbers
cmlenz@3: 
cmlenz@17: weekdays = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5,
cmlenz@17:             'sun': 6}
cmlenz@10: 
cmlenz@10: try:
cmlenz@10:     any
cmlenz@10: except NameError:
cmlenz@10:     def any(iterable):
cmlenz@10:         return filter(None, list(iterable))
cmlenz@10: 
cmlenz@3: def _text(elem):
cmlenz@3:     buf = [elem.text or '']
cmlenz@3:     for child in elem:
cmlenz@3:         buf.append(_text(child))
cmlenz@3:     buf.append(elem.tail or '')
cmlenz@3:     return u''.join(filter(None, buf)).strip()
cmlenz@3: 
cmlenz@3: def main():
cmlenz@3:     parser = OptionParser(usage='%prog path/to/cldr')
cmlenz@3:     options, args = parser.parse_args()
cmlenz@3:     if len(args) != 1:
cmlenz@3:         parser.error('incorrect number of arguments')
cmlenz@3: 
cmlenz@3:     srcdir = args[0]
cmlenz@3:     destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
cmlenz@3:                            '..', 'babel', 'localedata')
cmlenz@3: 
cmlenz@10:     sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml'))
cmlenz@10: 
cmlenz@10:     # build a territory containment mapping for inheritance
cmlenz@10:     regions = {}
cmlenz@10:     for elem in sup.findall('//territoryContainment/group'):
cmlenz@10:         regions[elem.attrib['type']] = elem.attrib['contains'].split()
cmlenz@10: 
cmlenz@10:     # Resolve territory containment
cmlenz@10:     territory_containment = {}
cmlenz@10:     region_items = regions.items()
cmlenz@10:     region_items.sort()
cmlenz@10:     for group, territory_list in region_items:
cmlenz@10:         for territory in territory_list:
cmlenz@10:             containers = territory_containment.setdefault(territory, set([]))
cmlenz@10:             if group in territory_containment:
cmlenz@10:                 containers |= territory_containment[group]
cmlenz@10:             containers.add(group)
cmlenz@10: 
cmlenz@3:     filenames = os.listdir(os.path.join(srcdir, 'main'))
cmlenz@3:     filenames.remove('root.xml')
cmlenz@3:     filenames.sort(lambda a,b: len(a)-len(b))
cmlenz@3:     filenames.insert(0, 'root.xml')
cmlenz@3: 
cmlenz@3:     dicts = {}
cmlenz@3: 
cmlenz@3:     for filename in filenames:
cmlenz@3:         print>>sys.stderr, 'Processing input file %r' % filename
cmlenz@3:         stem, ext = os.path.splitext(filename)
cmlenz@3:         if ext != '.xml':
cmlenz@3:             continue
cmlenz@3: 
cmlenz@28:         tree = parse(os.path.join(srcdir, 'main', filename))
cmlenz@3:         data = {}
cmlenz@3: 
cmlenz@10:         language = None
cmlenz@10:         elem = tree.find('//identity/language')
cmlenz@10:         if elem is not None:
cmlenz@10:             language = elem.attrib['type']
cmlenz@10:         print>>sys.stderr, '  Language:  %r' % language
cmlenz@10: 
cmlenz@10:         territory = None
cmlenz@10:         elem = tree.find('//identity/territory')
cmlenz@10:         if elem is not None:
cmlenz@10:             territory = elem.attrib['type']
cmlenz@15:         else:
cmlenz@15:             territory = '001' # world
cmlenz@10:         print>>sys.stderr, '  Territory: %r' % territory
cmlenz@10:         regions = territory_containment.get(territory, [])
cmlenz@10:         print>>sys.stderr, '  Regions:    %r' % regions
cmlenz@10: 
cmlenz@3:         # <localeDisplayNames>
cmlenz@3: 
cmlenz@3:         territories = data.setdefault('territories', {})
cmlenz@3:         for elem in tree.findall('//territories/territory'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib['type'] in territories:
cmlenz@3:                 continue
cmlenz@3:             territories[elem.attrib['type']] = _text(elem)
cmlenz@3: 
cmlenz@3:         languages = data.setdefault('languages', {})
cmlenz@3:         for elem in tree.findall('//languages/language'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib['type'] in languages:
cmlenz@3:                 continue
cmlenz@3:             languages[elem.attrib['type']] = _text(elem)
cmlenz@3: 
cmlenz@3:         variants = data.setdefault('variants', {})
cmlenz@3:         for elem in tree.findall('//variants/variant'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib['type'] in variants:
cmlenz@3:                 continue
cmlenz@3:             variants[elem.attrib['type']] = _text(elem)
cmlenz@3: 
cmlenz@3:         scripts = data.setdefault('scripts', {})
cmlenz@3:         for elem in tree.findall('//scripts/script'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib['type'] in scripts:
cmlenz@3:                 continue
cmlenz@3:             scripts[elem.attrib['type']] = _text(elem)
cmlenz@3: 
cmlenz@3:         # <dates>
cmlenz@3: 
cmlenz@10:         week_data = data.setdefault('week_data', {})
cmlenz@10:         supelem = sup.find('//weekData')
cmlenz@10: 
cmlenz@10:         for elem in supelem.findall('minDays'):
cmlenz@10:             territories = elem.attrib['territories'].split()
cmlenz@10:             if territory in territories or any([r in territories for r in regions]):
cmlenz@10:                 week_data['min_days'] = int(elem.attrib['count'])
cmlenz@10: 
cmlenz@10:         for elem in supelem.findall('firstDay'):
cmlenz@10:             territories = elem.attrib['territories'].split()
cmlenz@10:             if territory in territories or any([r in territories for r in regions]):
cmlenz@10:                 week_data['first_day'] = weekdays[elem.attrib['day']]
cmlenz@10: 
cmlenz@10:         for elem in supelem.findall('weekendStart'):
cmlenz@10:             territories = elem.attrib['territories'].split()
cmlenz@10:             if territory in territories or any([r in territories for r in regions]):
cmlenz@10:                 week_data['weekend_start'] = weekdays[elem.attrib['day']]
cmlenz@10: 
cmlenz@10:         for elem in supelem.findall('weekendEnd'):
cmlenz@10:             territories = elem.attrib['territories'].split()
cmlenz@10:             if territory in territories or any([r in territories for r in regions]):
cmlenz@10:                 week_data['weekend_end'] = weekdays[elem.attrib['day']]
cmlenz@10: 
cmlenz@3:         time_zones = data.setdefault('time_zones', {})
cmlenz@3:         for elem in tree.findall('//timeZoneNames/zone'):
cmlenz@30:             info = {}
cmlenz@30:             city = elem.findtext('exemplarCity')
cmlenz@30:             if city:
cmlenz@30:                 info['city'] = unicode(city)
cmlenz@30:             for child in elem.findall('long/*'):
cmlenz@30:                 info.setdefault('long', {})[child.tag] = unicode(child.text)
cmlenz@30:             for child in elem.findall('short/*'):
cmlenz@30:                 info.setdefault('short', {})[child.tag] = unicode(child.text)
cmlenz@30:             time_zones[elem.attrib['type']] = info
cmlenz@3: 
cmlenz@3:         for calendar in tree.findall('//calendars/calendar'):
cmlenz@3:             if calendar.attrib['type'] != 'gregorian':
cmlenz@3:                 # TODO: support other calendar types
cmlenz@3:                 continue
cmlenz@3: 
cmlenz@3:             months = data.setdefault('months', {})
cmlenz@3:             for ctxt in calendar.findall('months/monthContext'):
cmlenz@3:                 ctxts = months.setdefault(ctxt.attrib['type'], {})
cmlenz@3:                 for width in ctxt.findall('monthWidth'):
cmlenz@3:                     widths = ctxts.setdefault(width.attrib['type'], {})
cmlenz@3:                     for elem in width.findall('month'):
cmlenz@3:                         if 'draft' in elem.attrib and int(elem.attrib['type']) in widths:
cmlenz@3:                             continue
cmlenz@3:                         widths[int(elem.attrib.get('type'))] = unicode(elem.text)
cmlenz@3: 
cmlenz@3:             days = data.setdefault('days', {})
cmlenz@3:             for ctxt in calendar.findall('days/dayContext'):
cmlenz@3:                 ctxts = days.setdefault(ctxt.attrib['type'], {})
cmlenz@3:                 for width in ctxt.findall('dayWidth'):
cmlenz@3:                     widths = ctxts.setdefault(width.attrib['type'], {})
cmlenz@3:                     for elem in width.findall('day'):
cmlenz@10:                         dtype = weekdays[elem.attrib['type']]
cmlenz@3:                         if 'draft' in elem.attrib and dtype in widths:
cmlenz@3:                             continue
cmlenz@3:                         widths[dtype] = unicode(elem.text)
cmlenz@3: 
cmlenz@3:             quarters = data.setdefault('quarters', {})
cmlenz@3:             for ctxt in calendar.findall('quarters/quarterContext'):
cmlenz@3:                 ctxts = quarters.setdefault(ctxt.attrib['type'], {})
cmlenz@3:                 for width in ctxt.findall('quarterWidth'):
cmlenz@3:                     widths = ctxts.setdefault(width.attrib['type'], {})
cmlenz@3:                     for elem in width.findall('quarter'):
cmlenz@3:                         if 'draft' in elem.attrib and int(elem.attrib['type']) in widths:
cmlenz@3:                             continue
cmlenz@3:                         widths[int(elem.attrib.get('type'))] = unicode(elem.text)
cmlenz@3: 
cmlenz@3:             eras = data.setdefault('eras', {})
cmlenz@3:             for width in calendar.findall('eras/*'):
cmlenz@3:                 ewidth = {'eraNames': 'wide', 'eraAbbr': 'abbreviated'}[width.tag]
cmlenz@3:                 widths = eras.setdefault(ewidth, {})
cmlenz@3:                 for elem in width.findall('era'):
cmlenz@3:                     if 'draft' in elem.attrib and int(elem.attrib['type']) in widths:
cmlenz@3:                         continue
cmlenz@3:                     widths[int(elem.attrib.get('type'))] = unicode(elem.text)
cmlenz@3: 
cmlenz@3:             # AM/PM
cmlenz@3:             periods = data.setdefault('periods', {})
cmlenz@3:             for elem in calendar.findall('am'):
cmlenz@3:                 if 'draft' in elem.attrib and elem.tag in periods:
cmlenz@3:                     continue
cmlenz@3:                 periods[elem.tag] = unicode(elem.text)
cmlenz@3:             for elem in calendar.findall('pm'):
cmlenz@3:                 if 'draft' in elem.attrib and elem.tag in periods:
cmlenz@3:                     continue
cmlenz@3:                 periods[elem.tag] = unicode(elem.text)
cmlenz@3: 
cmlenz@3:             date_formats = data.setdefault('date_formats', {})
cmlenz@3:             for elem in calendar.findall('dateFormats/dateFormatLength'):
cmlenz@3:                 if 'draft' in elem.attrib and elem.attrib.get('type') in date_formats:
cmlenz@3:                     continue
cmlenz@3:                 try:
cmlenz@3:                     date_formats[elem.attrib.get('type')] = \
jonas@11:                         dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern')))
cmlenz@3:                 except ValueError, e:
cmlenz@28:                     print>>sys.stderr, 'ERROR: %s' % e
cmlenz@3: 
cmlenz@3:             time_formats = data.setdefault('time_formats', {})
cmlenz@3:             for elem in calendar.findall('timeFormats/timeFormatLength'):
cmlenz@3:                 if 'draft' in elem.attrib and elem.attrib.get('type') in time_formats:
cmlenz@3:                     continue
cmlenz@3:                 try:
cmlenz@3:                     time_formats[elem.attrib.get('type')] = \
jonas@11:                         dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern')))
cmlenz@3:                 except ValueError, e:
cmlenz@28:                     print>>sys.stderr, 'ERROR: %s' % e
cmlenz@3: 
cmlenz@35:             datetime_formats = data.setdefault('datetime_formats', {})
cmlenz@35:             for elem in calendar.findall('dateTimeFormats/dateTimeFormatLength'):
cmlenz@35:                 if 'draft' in elem.attrib and elem.attrib.get('type') in datetime_formats:
cmlenz@35:                     continue
cmlenz@35:                 try:
cmlenz@35:                     datetime_formats[elem.attrib.get('type')] = \
cmlenz@35:                         unicode(elem.findtext('dateTimeFormat/pattern'))
cmlenz@35:                 except ValueError, e:
cmlenz@35:                     print>>sys.stderr, 'ERROR: %s' % e
cmlenz@35: 
cmlenz@3:         # <numbers>
cmlenz@3: 
cmlenz@3:         number_symbols = data.setdefault('number_symbols', {})
cmlenz@3:         for elem in tree.findall('//numbers/symbols/*'):
cmlenz@3:             number_symbols[elem.tag] = unicode(elem.text)
cmlenz@3: 
cmlenz@3:         decimal_formats = data.setdefault('decimal_formats', {})
cmlenz@3:         for elem in tree.findall('//decimalFormats/decimalFormatLength'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib.get('type') in decimal_formats:
cmlenz@3:                 continue
cmlenz@28:             pattern = unicode(elem.findtext('decimalFormat/pattern'))
cmlenz@28:             decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
cmlenz@3: 
cmlenz@3:         scientific_formats = data.setdefault('scientific_formats', {})
cmlenz@3:         for elem in tree.findall('//scientificFormats/scientificFormatLength'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib.get('type') in scientific_formats:
cmlenz@3:                 continue
cmlenz@28:             # FIXME: should use numbers.parse_pattern
cmlenz@3:             scientific_formats[elem.attrib.get('type')] = unicode(elem.findtext('scientificFormat/pattern'))
cmlenz@3: 
cmlenz@3:         currency_formats = data.setdefault('currency_formats', {})
cmlenz@3:         for elem in tree.findall('//currencyFormats/currencyFormatLength'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib.get('type') in currency_formats:
cmlenz@3:                 continue
cmlenz@28:             # FIXME: should use numbers.parse_pattern
cmlenz@3:             currency_formats[elem.attrib.get('type')] = unicode(elem.findtext('currencyFormat/pattern'))
cmlenz@3: 
cmlenz@3:         percent_formats = data.setdefault('percent_formats', {})
cmlenz@3:         for elem in tree.findall('//percentFormats/percentFormatLength'):
cmlenz@3:             if 'draft' in elem.attrib and elem.attrib.get('type') in percent_formats:
cmlenz@3:                 continue
cmlenz@28:             pattern = unicode(elem.findtext('percentFormat/pattern'))
cmlenz@28:             percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
cmlenz@3: 
cmlenz@28:         currency_names = data.setdefault('currency_names', {})
cmlenz@28:         currency_symbols = data.setdefault('currency_symbols', {})
cmlenz@3:         for elem in tree.findall('//currencies/currency'):
cmlenz@28:             name = elem.findtext('displayName')
cmlenz@28:             if name:
cmlenz@28:                 currency_names[elem.attrib['type']] = unicode(name)
cmlenz@28:             symbol = elem.findtext('symbol')
cmlenz@28:             if symbol:
cmlenz@28:                 currency_symbols[elem.attrib['type']] = unicode(symbol)
cmlenz@3: 
cmlenz@3:         dicts[stem] = data
cmlenz@3:         outfile = open(os.path.join(destdir, stem + '.dat'), 'wb')
cmlenz@3:         try:
cmlenz@3:             pickle.dump(data, outfile, 2)
cmlenz@3:         finally:
cmlenz@3:             outfile.close()
cmlenz@3: 
cmlenz@3: if __name__ == '__main__':
cmlenz@3:     main()