Mercurial > genshi > genshi-test
changeset 934:7c9ec79caedc
Merge r1142 from py3k:
add support for python 3 to genshi.template expression evaluator:
* add support for python 3 AST:
* AST for raise has changed in Python 3.
* Python 3 adds AST nodes for individual arguments and Bytes.
* use genshi.compat functions for dealing with code objects.
* do not coerce byte strings to unicode in Python 3 ASTTransformer.
* replace doctests that reply on exception names with uglier but more compatible try:.. except:.. doctest
* handle filename preferences of Python 2 and 3 (2 prefers bytes, 3 prefers unicode).
* ifilter is gone from itertools in Python 3 so use repeat for tests instead.
| author | hodgestar |
|---|---|
| date | Fri, 18 Mar 2011 09:15:29 +0000 |
| parents | feba07fc925b |
| children | 705727288d7e |
| files | genshi/template/astutil.py genshi/template/eval.py genshi/template/tests/eval.py |
| diffstat | 3 files changed, 113 insertions(+), 67 deletions(-) [+] |
line wrap: on
line diff
--- a/genshi/template/astutil.py +++ b/genshi/template/astutil.py @@ -21,6 +21,7 @@ def parse(source, mode): return compile(source, '', mode, _ast.PyCF_ONLY_AST) +from genshi.compat import IS_PYTHON2 __docformat__ = 'restructuredtext en' @@ -129,6 +130,11 @@ first = False self._write('**' + node.kwarg) + if not IS_PYTHON2: + # In Python 3 arguments get a special node + def visit_arg(self, node): + self._write(node.arg) + # FunctionDef(identifier name, arguments args, # stmt* body, expr* decorator_list) def visit_FunctionDef(self, node): @@ -289,22 +295,36 @@ self._change_indent(-1) - # Raise(expr? type, expr? inst, expr? tback) - def visit_Raise(self, node): - self._new_line() - self._write('raise') - if not node.type: - return - self._write(' ') - self.visit(node.type) - if not node.inst: - return - self._write(', ') - self.visit(node.inst) - if not node.tback: - return - self._write(', ') - self.visit(node.tback) + if IS_PYTHON2: + # Raise(expr? type, expr? inst, expr? tback) + def visit_Raise(self, node): + self._new_line() + self._write('raise') + if not node.type: + return + self._write(' ') + self.visit(node.type) + if not node.inst: + return + self._write(', ') + self.visit(node.inst) + if not node.tback: + return + self._write(', ') + self.visit(node.tback) + else: + # Raise(expr? exc from expr? cause) + def visit_Raise(self, node): + self._new_line() + self._write('raise') + if not node.exc: + return + self._write(' ') + self.visit(node.exc) + if not node.cause: + return + self._write(' from ') + self.visit(node.cause) # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) def visit_TryExcept(self, node): @@ -626,6 +646,11 @@ def visit_Str(self, node): self._write(repr(node.s)) + if not IS_PYTHON2: + # Bytes(bytes s) + def visit_Bytes(self, node): + self._write(repr(node.s)) + # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, node): self.visit(node.value)
--- a/genshi/template/eval.py +++ b/genshi/template/eval.py @@ -24,6 +24,8 @@ from genshi.template.base import TemplateRuntimeError from genshi.util import flatten +from genshi.compat import get_code_params, build_code_chunk, IS_PYTHON2 + __all__ = ['Code', 'Expression', 'Suite', 'LenientLookup', 'StrictLookup', 'Undefined', 'UndefinedError'] __docformat__ = 'restructuredtext en' @@ -98,10 +100,7 @@ def __getstate__(self): state = {'source': self.source, 'ast': self.ast, 'lookup': self._globals.im_self} - c = self.code - state['code'] = (c.co_nlocals, c.co_stacksize, c.co_flags, c.co_code, - c.co_consts, c.co_names, c.co_varnames, c.co_filename, - c.co_name, c.co_firstlineno, c.co_lnotab, (), ()) + state['code'] = get_code_params(self.code) return state def __setstate__(self, state): @@ -236,15 +235,17 @@ of that variable, will raise an exception that includes the name used to reference that undefined variable. - >>> foo('bar') - Traceback (most recent call last): - ... - UndefinedError: "foo" not defined + >>> try: + ... foo('bar') + ... except UndefinedError, e: + ... print e.msg + "foo" not defined - >>> foo.bar - Traceback (most recent call last): - ... - UndefinedError: "foo" not defined + >>> try: + ... foo.bar + ... except UndefinedError, e: + ... print e.msg + "foo" not defined :see: `LenientLookup` """ @@ -388,19 +389,21 @@ raise an ``UndefinedError``: >>> expr = Expression('nothing', lookup='strict') - >>> expr.evaluate({}) - Traceback (most recent call last): - ... - UndefinedError: "nothing" not defined + >>> try: + ... expr.evaluate({}) + ... except UndefinedError, e: + ... print e.msg + "nothing" not defined The same happens when a non-existing attribute or item is accessed on an existing object: >>> expr = Expression('something.nil', lookup='strict') - >>> expr.evaluate({'something': dict()}) - Traceback (most recent call last): - ... - UndefinedError: {} has no member named "nil" + >>> try: + ... expr.evaluate({'something': dict()}) + ... except UndefinedError, e: + ... print e.msg + {} has no member named "nil" """ @classmethod @@ -421,17 +424,22 @@ rest = '\n'.join([' %s' % line for line in rest.splitlines()]) source = '\n'.join([first, rest]) if isinstance(source, unicode): - source = '\xef\xbb\xbf' + source.encode('utf-8') + source = (u'\ufeff' + source).encode('utf-8') return parse(source, mode) def _compile(node, source=None, mode='eval', filename=None, lineno=-1, xform=None): - if isinstance(filename, unicode): - # unicode file names not allowed for code objects - filename = filename.encode('utf-8', 'replace') - elif not filename: + if not filename: filename = '<string>' + if IS_PYTHON2: + # Python 2 requires non-unicode filenames + if isinstance(filename, unicode): + filename = filename.encode('utf-8', 'replace') + else: + # Python 3 requires unicode filenames + if not isinstance(filename, unicode): + filename = filename.decode('utf-8', 'replace') if lineno <= 0: lineno = 1 @@ -458,10 +466,7 @@ try: # We'd like to just set co_firstlineno, but it's readonly. So we need # to clone the code object while adjusting the line number - return CodeType(0, code.co_nlocals, code.co_stacksize, - code.co_flags | 0x0040, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, name, - lineno, code.co_lnotab, (), ()) + return build_code_chunk(code, filename, name, lineno) except RuntimeError: return code @@ -493,6 +498,8 @@ def _extract_names(self, node): names = set() def _process(node): + if not IS_PYTHON2 and isinstance(node, _ast.arg): + names.add(node.arg) if isinstance(node, _ast.Name): names.add(node.id) elif isinstance(node, _ast.alias): @@ -513,7 +520,7 @@ return names def visit_Str(self, node): - if isinstance(node.s, str): + if not isinstance(node.s, unicode): try: # If the string is ASCII, return a `str` object node.s.decode('ascii') except ValueError: # Otherwise return a `unicode` object
--- a/genshi/template/tests/eval.py +++ b/genshi/template/tests/eval.py @@ -14,7 +14,6 @@ import doctest import os import pickle -from StringIO import StringIO import sys from tempfile import mkstemp import unittest @@ -23,6 +22,7 @@ from genshi.template.base import Context from genshi.template.eval import Expression, Suite, Undefined, UndefinedError, \ UNDEFINED +from genshi.compat import BytesIO, IS_PYTHON2, wrapped_bytes class ExpressionTestCase(unittest.TestCase): @@ -39,7 +39,7 @@ def test_pickle(self): expr = Expression('1 < 2') - buf = StringIO() + buf = BytesIO() pickle.dump(expr, buf, 2) buf.seek(0) unpickled = pickle.load(buf) @@ -58,7 +58,8 @@ def test_str_literal(self): self.assertEqual('foo', Expression('"foo"').evaluate({})) self.assertEqual('foo', Expression('"""foo"""').evaluate({})) - self.assertEqual('foo', Expression("'foo'").evaluate({})) + self.assertEqual(u'foo'.encode('utf-8'), + Expression(wrapped_bytes("b'foo'")).evaluate({})) self.assertEqual('foo', Expression("'''foo'''").evaluate({})) self.assertEqual('foo', Expression("u'foo'").evaluate({})) self.assertEqual('foo', Expression("r'foo'").evaluate({})) @@ -68,14 +69,23 @@ self.assertEqual(u'þ', expr.evaluate({})) expr = Expression("u'\xfe'") self.assertEqual(u'þ', expr.evaluate({})) - expr = Expression("'\xc3\xbe'") - self.assertEqual(u'þ', expr.evaluate({})) + # On Python2 strings are converted to unicode if they contained + # non-ASCII characters. + # On Py3k, we have no need to do this as non-prefixed strings aren't + # raw. + expr = Expression(wrapped_bytes(r"b'\xc3\xbe'")) + if IS_PYTHON2: + self.assertEqual(u'þ', expr.evaluate({})) + else: + self.assertEqual(u'þ'.encode('utf-8'), expr.evaluate({})) def test_num_literal(self): self.assertEqual(42, Expression("42").evaluate({})) - self.assertEqual(42L, Expression("42L").evaluate({})) + if IS_PYTHON2: + self.assertEqual(42L, Expression("42L").evaluate({})) self.assertEqual(.42, Expression(".42").evaluate({})) - self.assertEqual(07, Expression("07").evaluate({})) + if IS_PYTHON2: + self.assertEqual(07, Expression("07").evaluate({})) self.assertEqual(0xF2, Expression("0xF2").evaluate({})) self.assertEqual(0XF2, Expression("0XF2").evaluate({})) @@ -246,12 +256,15 @@ def test_lambda(self): data = {'items': range(5)} expr = Expression("filter(lambda x: x > 2, items)") - self.assertEqual([3, 4], expr.evaluate(data)) + self.assertEqual([3, 4], list(expr.evaluate(data))) def test_lambda_tuple_arg(self): + # This syntax goes away in Python 3 + if not IS_PYTHON2: + return data = {'items': [(1, 2), (2, 1)]} expr = Expression("filter(lambda (x, y): x > y, items)") - self.assertEqual([(2, 1)], expr.evaluate(data)) + self.assertEqual([(2, 1)], list(expr.evaluate(data))) def test_list_comprehension(self): expr = Expression("[n for n in numbers if n < 2]") @@ -470,7 +483,7 @@ def test_pickle(self): suite = Suite('foo = 42') - buf = StringIO() + buf = BytesIO() pickle.dump(suite, buf, 2) buf.seek(0) unpickled = pickle.load(buf) @@ -645,26 +658,26 @@ assert 'plain' in data def test_import(self): - suite = Suite("from itertools import ifilter") + suite = Suite("from itertools import repeat") data = {} suite.execute(data) - assert 'ifilter' in data + assert 'repeat' in data def test_import_star(self): suite = Suite("from itertools import *") data = Context() suite.execute(data) - assert 'ifilter' in data + assert 'repeat' in data def test_import_in_def(self): suite = Suite("""def fun(): - from itertools import ifilter - return ifilter(None, range(3)) + from itertools import repeat + return repeat(1, 3) """) data = Context() suite.execute(data) - assert 'ifilter' not in data - self.assertEqual([1, 2], list(data['fun']())) + assert 'repeat' not in data + self.assertEqual([1, 1, 1], list(data['fun']())) def test_for(self): suite = Suite("""x = [] @@ -766,7 +779,7 @@ self.assertEqual("foo", d["k"]) def test_exec(self): - suite = Suite("x = 1; exec d['k']; assert x == 42, x") + suite = Suite("x = 1; exec(d['k']); assert x == 42, x") suite.execute({"d": {"k": "x = 42"}}) def test_return(self): @@ -828,7 +841,8 @@ def test_yield_expression(self): d = {} - suite = Suite("""results = [] + suite = Suite("""from genshi.compat import next +results = [] def counter(maximum): i = 0 while i < maximum: @@ -838,9 +852,9 @@ else: i += 1 it = counter(5) -results.append(it.next()) +results.append(next(it)) results.append(it.send(3)) -results.append(it.next()) +results.append(next(it)) """) suite.execute(d) self.assertEqual([0, 3, 4], d['results'])