changeset 343:6dae6a9e1096

JavaScript lexer falls back silently now on syntax errors and tries to recover.
author aronacher
date Sat, 14 Jun 2008 22:07:41 +0000
parents 603192024857
children 2a7b818fa5a0
files babel/messages/jslexer.py
diffstat 1 files changed, 4 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -23,7 +23,7 @@
     '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=',
     '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=',
     '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')',
-    '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.'
+    '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':'
 ]
 operators.sort(lambda a, b: cmp(-len(a), -len(b)))
 
@@ -55,10 +55,6 @@
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
 
 
-class TokenError(ValueError):
-    """Raised if the tokenizer stumbled upon invalid tokens."""
-
-
 class Token(tuple):
     """Represents a token as returned by `tokenize`."""
     __slots__ = ()
@@ -166,7 +162,9 @@
                 match = regex_re.match(source, pos)
                 token_type = 'regexp'
             if match is None:
-                raise TokenError('invalid syntax around line %d' % lineno)
+                # woops. invalid syntax. jump one char ahead and try again.
+                pos += 1
+                continue
 
         token_value = match.group()
         if token_type is not None:
Copyright (C) 2012-2017 Edgewall Software