Mercurial > babel > old > mirror
comparison babel/messages/jslexer.py @ 343:6dae6a9e1096
JavaScript lexer falls back silently now on syntax errors and tries to recover.
| author | aronacher |
|---|---|
| date | Sat, 14 Jun 2008 22:07:41 +0000 |
| parents | 603192024857 |
| children | abe62ab2a889 |
comparison
equal
deleted
inserted
replaced
| 342:603192024857 | 343:6dae6a9e1096 |
|---|---|
| 21 | 21 |
| 22 operators = [ | 22 operators = [ |
| 23 '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=', | 23 '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=', |
| 24 '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=', | 24 '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=', |
| 25 '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')', | 25 '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')', |
| 26 '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.' | 26 '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':' |
| 27 ] | 27 ] |
| 28 operators.sort(lambda a, b: cmp(-len(a), -len(b))) | 28 operators.sort(lambda a, b: cmp(-len(a), -len(b))) |
| 29 | 29 |
| 30 escapes = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'} | 30 escapes = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'} |
| 31 | 31 |
| 51 division_re = re.compile(r'/=?') | 51 division_re = re.compile(r'/=?') |
| 52 regex_re = re.compile(r'/.+?/[a-zA-Z]*(?s)') | 52 regex_re = re.compile(r'/.+?/[a-zA-Z]*(?s)') |
| 53 line_re = re.compile(r'(\r\n|\n|\r)') | 53 line_re = re.compile(r'(\r\n|\n|\r)') |
| 54 line_join_re = re.compile(r'\\' + line_re.pattern) | 54 line_join_re = re.compile(r'\\' + line_re.pattern) |
| 55 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}') | 55 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}') |
| 56 | |
| 57 | |
| 58 class TokenError(ValueError): | |
| 59 """Raised if the tokenizer stumbled upon invalid tokens.""" | |
| 60 | 56 |
| 61 | 57 |
| 62 class Token(tuple): | 58 class Token(tuple): |
| 63 """Represents a token as returned by `tokenize`.""" | 59 """Represents a token as returned by `tokenize`.""" |
| 64 __slots__ = () | 60 __slots__ = () |
| 164 token_type = 'operator' | 160 token_type = 'operator' |
| 165 else: | 161 else: |
| 166 match = regex_re.match(source, pos) | 162 match = regex_re.match(source, pos) |
| 167 token_type = 'regexp' | 163 token_type = 'regexp' |
| 168 if match is None: | 164 if match is None: |
| 169 raise TokenError('invalid syntax around line %d' % lineno) | 165 # woops. invalid syntax. jump one char ahead and try again. |
| 166 pos += 1 | |
| 167 continue | |
| 170 | 168 |
| 171 token_value = match.group() | 169 token_value = match.group() |
| 172 if token_type is not None: | 170 if token_type is not None: |
| 173 token = Token(token_type, token_value, lineno) | 171 token = Token(token_type, token_value, lineno) |
| 174 may_divide = indicates_division(token) | 172 may_divide = indicates_division(token) |