aronacher@339: # -*- coding: utf-8 -*-
aronacher@339: #
aronacher@339: # Copyright (C) 2008 Edgewall Software
aronacher@339: # All rights reserved.
aronacher@339: #
aronacher@339: # This software is licensed as described in the file COPYING, which
aronacher@339: # you should have received as part of this distribution. The terms
aronacher@339: # are also available at http://babel.edgewall.org/wiki/License.
aronacher@339: #
aronacher@339: # This software consists of voluntary contributions made by many
aronacher@339: # individuals. For the exact contribution history, see the revision
aronacher@339: # history and logs, available at http://babel.edgewall.org/log/.
aronacher@339: 
aronacher@339: """A simple JavaScript 1.5 lexer which is used for the JavaScript
aronacher@339: extractor.
aronacher@339: """
aronacher@339: 
jruigrok@527: from operator import itemgetter
aronacher@339: import re
pjenvey@414: 
aronacher@339: operators = [
aronacher@339:     '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=',
aronacher@339:     '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=',
aronacher@339:     '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')',
aronacher@341:     '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':'
aronacher@339: ]
aronacher@339: operators.sort(lambda a, b: cmp(-len(a), -len(b)))
aronacher@339: 
aronacher@339: escapes = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'}
aronacher@339: 
aronacher@339: rules = [
aronacher@339:     (None, re.compile(r'\s+(?u)')),
aronacher@339:     (None, re.compile(r'<!--.*')),
aronacher@339:     ('linecomment', re.compile(r'//.*')),
aronacher@339:     ('multilinecomment', re.compile(r'/\*.*?\*/(?us)')),
aronacher@339:     ('name', re.compile(r'(\$+\w*|[^\W\d]\w*)(?u)')),
aronacher@339:     ('number', re.compile(r'''(?x)(
aronacher@339:         (?:0|[1-9]\d*)
aronacher@339:         (\.\d+)?
aronacher@339:         ([eE][-+]?\d+)? |
aronacher@339:         (0x[a-fA-F0-9]+)
aronacher@339:     )''')),
aronacher@339:     ('operator', re.compile(r'(%s)' % '|'.join(map(re.escape, operators)))),
aronacher@339:     ('string', re.compile(r'''(?xs)(
aronacher@339:         '(?:[^'\\]*(?:\\.[^'\\]*)*)'  |
aronacher@339:         "(?:[^"\\]*(?:\\.[^"\\]*)*)"
aronacher@339:     )'''))
aronacher@339: ]
aronacher@339: 
aronacher@339: division_re = re.compile(r'/=?')
aronacher@404: regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*(?s)')
aronacher@339: line_re = re.compile(r'(\r\n|\n|\r)')
aronacher@339: line_join_re = re.compile(r'\\' + line_re.pattern)
aronacher@339: uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
aronacher@339: 
aronacher@339: 
aronacher@339: class Token(tuple):
aronacher@339:     """Represents a token as returned by `tokenize`."""
aronacher@339:     __slots__ = ()
aronacher@339: 
aronacher@339:     def __new__(cls, type, value, lineno):
aronacher@339:         return tuple.__new__(cls, (type, value, lineno))
aronacher@339: 
aronacher@339:     type = property(itemgetter(0))
aronacher@339:     value = property(itemgetter(1))
aronacher@339:     lineno = property(itemgetter(2))
aronacher@339: 
aronacher@340: 
aronacher@339: def indicates_division(token):
aronacher@339:     """A helper function that helps the tokenizer to decide if the current
aronacher@339:     token may be followed by a division operator.
aronacher@339:     """
aronacher@339:     if token.type == 'operator':
aronacher@339:         return token.value in (')', ']', '}', '++', '--')
aronacher@339:     return token.type in ('name', 'number', 'string', 'regexp')
aronacher@339: 
aronacher@340: 
aronacher@339: def unquote_string(string):
aronacher@339:     """Unquote a string with JavaScript rules.  The string has to start with
aronacher@339:     string delimiters (``'`` or ``"``.)
aronacher@339: 
aronacher@339:     :return: a string
aronacher@339:     """
aronacher@339:     assert string and string[0] == string[-1] and string[0] in '"\'', \
aronacher@339:         'string provided is not properly delimited'
aronacher@339:     string = line_join_re.sub('\\1', string[1:-1])
aronacher@339:     result = []
aronacher@339:     add = result.append
aronacher@339:     pos = 0
aronacher@339: 
aronacher@339:     while 1:
aronacher@339:         # scan for the next escape
aronacher@339:         escape_pos = string.find('\\', pos)
aronacher@339:         if escape_pos < 0:
aronacher@339:             break
aronacher@339:         add(string[pos:escape_pos])
aronacher@339: 
aronacher@339:         # check which character is escaped
aronacher@339:         next_char = string[escape_pos + 1]
aronacher@339:         if next_char in escapes:
aronacher@339:             add(escapes[next_char])
aronacher@339: 
aronacher@339:         # unicode escapes.  trie to consume up to four characters of
aronacher@339:         # hexadecimal characters and try to interpret them as unicode
aronacher@339:         # character point.  If there is no such character point, put
aronacher@339:         # all the consumed characters into the string.
aronacher@339:         elif next_char in 'uU':
aronacher@339:             escaped = uni_escape_re.match(string, escape_pos + 2)
aronacher@339:             if escaped is not None:
aronacher@339:                 escaped_value = escaped.group()
aronacher@339:                 if len(escaped_value) == 4:
aronacher@339:                     try:
aronacher@339:                         add(unichr(int(escaped_value, 16)))
aronacher@339:                     except ValueError:
aronacher@339:                         pass
aronacher@339:                     else:
aronacher@339:                         pos = escape_pos + 6
aronacher@339:                         continue
aronacher@339:                 add(next_char + escaped_value)
aronacher@339:                 pos = escaped.end()
aronacher@339:                 continue
aronacher@339:             else:
aronacher@339:                 add(next_char)
aronacher@339: 
aronacher@339:         # bogus escape.  Just remove the backslash.
aronacher@339:         else:
aronacher@339:             add(next_char)
aronacher@339:         pos = escape_pos + 2
aronacher@339: 
aronacher@339:     if pos < len(string):
aronacher@339:         add(string[pos:])
aronacher@339: 
aronacher@339:     return u''.join(result)
aronacher@339: 
aronacher@340: 
aronacher@339: def tokenize(source):
aronacher@339:     """Tokenize a JavaScript source.
aronacher@339: 
aronacher@339:     :return: generator of `Token`\s
aronacher@339:     """
aronacher@339:     may_divide = False
aronacher@339:     pos = 0
aronacher@339:     lineno = 1
aronacher@339:     end = len(source)
aronacher@339: 
aronacher@339:     while pos < end:
aronacher@339:         # handle regular rules first
aronacher@339:         for token_type, rule in rules:
aronacher@339:             match = rule.match(source, pos)
aronacher@339:             if match is not None:
aronacher@339:                 break
aronacher@339:         # if we don't have a match we don't give up yet, but check for
aronacher@339:         # division operators or regular expression literals, based on
aronacher@339:         # the status of `may_divide` which is determined by the last
aronacher@339:         # processed non-whitespace token using `indicates_division`.
aronacher@339:         else:
aronacher@339:             if may_divide:
aronacher@339:                 match = division_re.match(source, pos)
aronacher@339:                 token_type = 'operator'
aronacher@339:             else:
aronacher@339:                 match = regex_re.match(source, pos)
aronacher@339:                 token_type = 'regexp'
aronacher@339:             if match is None:
aronacher@341:                 # woops. invalid syntax. jump one char ahead and try again.
aronacher@341:                 pos += 1
aronacher@341:                 continue
aronacher@339: 
aronacher@339:         token_value = match.group()
aronacher@339:         if token_type is not None:
aronacher@339:             token = Token(token_type, token_value, lineno)
aronacher@339:             may_divide = indicates_division(token)
aronacher@339:             yield token
aronacher@339:         lineno += len(line_re.findall(token_value))
aronacher@339:         pos = match.end()