Mercurial > babel > old > mirror
diff babel/messages/jslexer.py @ 342:603192024857
added some newlines to extract and jslexer to stay consistent with the rest of the sourcecode.
author | aronacher |
---|---|
date | Sat, 14 Jun 2008 19:00:35 +0000 |
parents | 672b6b8e945d |
children | 6dae6a9e1096 |
line wrap: on
line diff
--- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -58,6 +58,7 @@ class TokenError(ValueError): """Raised if the tokenizer stumbled upon invalid tokens.""" + class Token(tuple): """Represents a token as returned by `tokenize`.""" __slots__ = () @@ -69,6 +70,7 @@ value = property(itemgetter(1)) lineno = property(itemgetter(2)) + def indicates_division(token): """A helper function that helps the tokenizer to decide if the current token may be followed by a division operator. @@ -77,6 +79,7 @@ return token.value in (')', ']', '}', '++', '--') return token.type in ('name', 'number', 'string', 'regexp') + def unquote_string(string): """Unquote a string with JavaScript rules. The string has to start with string delimiters (``'`` or ``"``.) @@ -134,6 +137,7 @@ return u''.join(result) + def tokenize(source): """Tokenize a JavaScript source.