Mercurial > genshi > mirror

comparison genshi/template/astutil.py @ 794:f9e23d472a6e trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged AST branch back into trunk. Most of this code was written by Marcin Kurczych for his Google Summer of Code 2008 project. The merge of this branch means that Genshi now uses the native `_ast` module on Python >= 2.5, and an emulation thereof on Python 2.4. This replaces the usage of the `compiler` package, which was deprecated in Python 2.6 and removed in Python 3.0. Another effect is that Genshi now runs on Google AppEngine (although performance is bad due to the lack of template caching).
author cmlenz
date Tue, 16 Dec 2008 23:02:36 +0000
parents
children 47f715774b12
comparison
equal deleted inserted replaced
793:be88c77839fc 794:f9e23d472a6e
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (C) 2008 Edgewall Software
4 # All rights reserved.
5 #
6 # This software is licensed as described in the file COPYING, which
7 # you should have received as part of this distribution. The terms
8 # are also available at http://genshi.edgewall.org/wiki/License.
9 #
10 # This software consists of voluntary contributions made by many
11 # individuals. For the exact contribution history, see the revision
12 # history and logs, available at http://genshi.edgewall.org/log/.
13
14 """Support classes for generating code from abstract syntax trees."""
15
16 try:
17 import _ast
18 except ImportError:
19 from genshi.template.ast24 import _ast, parse
20 else:
21 if not hasattr(_ast, 'AST'):
22 from genshi.template.astgae import restore
23 restore(_ast)
24 def parse(source, mode):
25 return compile(source, '', mode, _ast.PyCF_ONLY_AST)
26
27
28 __docformat__ = 'restructuredtext en'
29
30
31 class ASTCodeGenerator(object):
32 """General purpose base class for AST transformations.
33
34 Every visitor method can be overridden to return an AST node that has been
35 altered or replaced in some way.
36 """
37 def __init__(self, tree):
38 self.lines_info = []
39 self.line_info = None
40 self.code = ''
41 self.line = None
42 self.last = None
43 self.indent = 0
44 self.blame_stack = []
45 self.visit(tree)
46 if self.line.strip():
47 self.code += self.line + '\n'
48 self.lines_info.append(self.line_info)
49 self.line = None
50 self.line_info = None
51
52 def _change_indent(self, delta):
53 self.indent += delta
54
55 def _new_line(self):
56 if self.line is not None:
57 self.code += self.line + '\n'
58 self.lines_info.append(self.line_info)
59 self.line = ' '*4*self.indent
60 if len(self.blame_stack) == 0:
61 self.line_info = []
62 self.last = None
63 else:
64 self.line_info = [(0, self.blame_stack[-1],)]
65 self.last = self.blame_stack[-1]
66
67 def _write(self, s):
68 if len(s) == 0:
69 return
70 if len(self.blame_stack) == 0:
71 if self.last is not None:
72 self.last = None
73 self.line_info.append((len(self.line), self.last))
74 else:
75 if self.last != self.blame_stack[-1]:
76 self.last = self.blame_stack[-1]
77 self.line_info.append((len(self.line), self.last))
78 self.line += s
79
80 def visit(self, node):
81 if node is None:
82 return None
83 if type(node) is tuple:
84 return tuple([self.visit(n) for n in node])
85 try:
86 self.blame_stack.append((node.lineno, node.col_offset,))
87 info = True
88 except AttributeError:
89 info = False
90 visitor = getattr(self, 'visit_%s' % node.__class__.__name__, None)
91 if visitor is None:
92 raise Exception('Unhandled node type %r' % type(node))
93 ret = visitor(node)
94 if info:
95 self.blame_stack.pop()
96 return ret
97
98 def visit_Module(self, node):
99 for n in node.body:
100 self.visit(n)
101 visit_Interactive = visit_Module
102 visit_Suite = visit_Module
103
104 def visit_Expression(self, node):
105 self._new_line()
106 return self.visit(node.body)
107
108 # arguments = (expr* args, identifier? vararg,
109 # identifier? kwarg, expr* defaults)
110 def visit_arguments(self, node):
111 first = True
112 for i, arg in enumerate(node.args):
113 if not first:
114 self._write(', ')
115 else:
116 first = False
117 self.visit(arg)
118 if i < len(node.defaults):
119 self._write('=')
120 self.visit(node.defaults[i])
121 if getattr(node, 'vararg', None):
122 if not first:
123 self._write(', ')
124 else:
125 first = False
126 self._write('*' + node.vararg)
127 if getattr(node, 'kwarg', None):
128 if not first:
129 self._write(', ')
130 else:
131 first = False
132 self._write('**' + node.kwarg)
133
134 # FunctionDef(identifier name, arguments args,
135 # stmt* body, expr* decorators)
136 def visit_FunctionDef(self, node):
137 for decorator in getattr(node, 'decorators', ()):
138 self._new_line()
139 self._write('@')
140 self.visit(decorator)
141 self._new_line()
142 self._write('def ' + node.name + '(')
143 self.visit(node.args)
144 self._write('):')
145 self._change_indent(1)
146 for statement in node.body:
147 self.visit(statement)
148 self._change_indent(-1)
149
150 # ClassDef(identifier name, expr* bases, stmt* body)
151 def visit_ClassDef(self, node):
152 self._new_line()
153 self._write('class ' + node.name)
154 if node.bases:
155 self._write('(')
156 self.visit(node.bases[0])
157 for base in node.bases[1:]:
158 self._write(', ')
159 self.visit(base)
160 self._write(')')
161 self._write(':')
162 self._change_indent(1)
163 for statement in node.body:
164 self.visit(statement)
165 self._change_indent(-1)
166
167 # Return(expr? value)
168 def visit_Return(self, node):
169 self._new_line()
170 self._write('return')
171 if getattr(node, 'value', None):
172 self._write(' ')
173 self.visit(node.value)
174
175 # Delete(expr* targets)
176 def visit_Delete(self, node):
177 self._new_line()
178 self._write('del ')
179 self.visit(node.targets[0])
180 for target in node.targets[1:]:
181 self._write(', ')
182 self.visit(target)
183
184 # Assign(expr* targets, expr value)
185 def visit_Assign(self, node):
186 self._new_line()
187 for target in node.targets:
188 self.visit(target)
189 self._write(' = ')
190 self.visit(node.value)
191
192 # AugAssign(expr target, operator op, expr value)
193 def visit_AugAssign(self, node):
194 self._new_line()
195 self.visit(node.target)
196 self._write(' ' + self.binary_operators[node.op.__class__] + '= ')
197 self.visit(node.value)
198
199 # Print(expr? dest, expr* values, bool nl)
200 def visit_Print(self, node):
201 self._new_line()
202 self._write('print')
203 if getattr(node, 'dest', None):
204 self._write(' >> ')
205 self.visit(node.dest)
206 if getattr(node, 'values', None):
207 self._write(', ')
208 if getattr(node, 'values', None):
209 self.visit(node.values[0])
210 for value in node.values[1:]:
211 self._write(', ')
212 self.visit(value)
213 if not node.nl:
214 self._write(',')
215
216 # For(expr target, expr iter, stmt* body, stmt* orelse)
217 def visit_For(self, node):
218 self._new_line()
219 self._write('for ')
220 self.visit(node.target)
221 self._write(' in ')
222 self.visit(node.iter)
223 self._write(':')
224 self._change_indent(1)
225 for statement in node.body:
226 self.visit(statement)
227 self._change_indent(-1)
228 if getattr(node, 'orelse', None):
229 self._new_line()
230 self._write('else:')
231 self._change_indent(1)
232 for statement in node.orelse:
233 self.visit(statement)
234 self._change_indent(-1)
235
236 # While(expr test, stmt* body, stmt* orelse)
237 def visit_While(self, node):
238 self._new_line()
239 self._write('while ')
240 self.visit(node.test)
241 self._write(':')
242 self._change_indent(1)
243 for statement in node.body:
244 self.visit(statement)
245 self._change_indent(-1)
246 if getattr(node, 'orelse', None):
247 self._new_line()
248 self._write('else:')
249 self._change_indent(1)
250 for statement in node.orelse:
251 self.visit(statement)
252 self._change_indent(-1)
253
254 # If(expr test, stmt* body, stmt* orelse)
255 def visit_If(self, node):
256 self._new_line()
257 self._write('if ')
258 self.visit(node.test)
259 self._write(':')
260 self._change_indent(1)
261 for statement in node.body:
262 self.visit(statement)
263 self._change_indent(-1)
264 if getattr(node, 'orelse', None):
265 self._new_line()
266 self._write('else:')
267 self._change_indent(1)
268 for statement in node.orelse:
269 self.visit(statement)
270 self._change_indent(-1)
271
272 # With(expr context_expr, expr? optional_vars, stmt* body)
273 def visit_With(self, node):
274 self._new_line()
275 self._write('with ')
276 self.visit(node.context_expr)
277 if getattr(node, 'optional_vars', None):
278 self._write(' as ')
279 self.visit(node.optional_vars)
280 self._write(':')
281 self._change_indent(1)
282 for statement in node.body:
283 self.visit(statement)
284 self._change_indent(-1)
285
286
287 # Raise(expr? type, expr? inst, expr? tback)
288 def visit_Raise(self, node):
289 self._new_line()
290 self._write('raise')
291 if not node.type:
292 return
293 self._write(' ')
294 self.visit(node.type)
295 if not node.inst:
296 return
297 self._write(', ')
298 self.visit(node.inst)
299 if not node.tback:
300 return
301 self._write(', ')
302 self.visit(node.tback)
303
304 # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
305 def visit_TryExcept(self, node):
306 self._new_line()
307 self._write('try:')
308 self._change_indent(1)
309 for statement in node.body:
310 self.visit(statement)
311 self._change_indent(-1)
312 if getattr(node, 'handlers', None):
313 for handler in node.handlers:
314 self.visit(handler)
315 self._new_line()
316 if getattr(node, 'orelse', None):
317 self._write('else:')
318 self._change_indent(1)
319 for statement in node.orelse:
320 self.visit(statement)
321 self._change_indent(-1)
322
323 # excepthandler = (expr? type, expr? name, stmt* body)
324 def visit_ExceptHandler(self, node):
325 self._new_line()
326 self._write('except')
327 if getattr(node, 'type', None):
328 self._write(' ')
329 self.visit(node.type)
330 if getattr(node, 'name', None):
331 self._write(', ')
332 self.visit(node.name)
333 self._write(':')
334 self._change_indent(1)
335 for statement in node.body:
336 self.visit(statement)
337 self._change_indent(-1)
338 visit_excepthandler = visit_ExceptHandler
339
340 # TryFinally(stmt* body, stmt* finalbody)
341 def visit_TryFinally(self, node):
342 self._new_line()
343 self._write('try:')
344 self._change_indent(1)
345 for statement in node.body:
346 self.visit(statement)
347 self._change_indent(-1)
348
349 if getattr(node, 'finalbody', None):
350 self._new_line()
351 self._write('finally:')
352 self._change_indent(1)
353 for statement in node.finalbody:
354 self.visit(statement)
355 self._change_indent(-1)
356
357 # Assert(expr test, expr? msg)
358 def visit_Assert(self, node):
359 self._new_line()
360 self._write('assert ')
361 self.visit(node.test)
362 if getattr(node, 'msg', None):
363 self._write(', ')
364 self.visit(node.msg)
365
366 def visit_alias(self, node):
367 self._write(node.name)
368 if getattr(node, 'asname', None):
369 self._write(' as ')
370 self._write(node.asname)
371
372 # Import(alias* names)
373 def visit_Import(self, node):
374 self._new_line()
375 self._write('import ')
376 self.visit(node.names[0])
377 for name in node.names[1:]:
378 self._write(', ')
379 self.visit(name)
380
381 # ImportFrom(identifier module, alias* names, int? level)
382 def visit_ImportFrom(self, node):
383 self._new_line()
384 self._write('from ')
385 if node.level:
386 self._write('.' * node.level)
387 self._write(node.module)
388 self._write(' import ')
389 self.visit(node.names[0])
390 for name in node.names[1:]:
391 self._write(', ')
392 self.visit(name)
393
394 # Exec(expr body, expr? globals, expr? locals)
395 def visit_Exec(self, node):
396 self._new_line()
397 self._write('exec ')
398 self.visit(node.body)
399 if not node.globals:
400 return
401 self._write(', ')
402 self.visit(node.globals)
403 if not node.locals:
404 return
405 self._write(', ')
406 self.visit(node.locals)
407
408 # Global(identifier* names)
409 def visit_Global(self, node):
410 self._new_line()
411 self._write('global ')
412 self.visit(node.names[0])
413 for name in node.names[1:]:
414 self._write(', ')
415 self.visit(name)
416
417 # Expr(expr value)
418 def visit_Expr(self, node):
419 self._new_line()
420 self.visit(node.value)
421
422 # Pass
423 def visit_Pass(self, node):
424 self._new_line()
425 self._write('pass')
426
427 # Break
428 def visit_Break(self, node):
429 self._new_line()
430 self._write('break')
431
432 # Continue
433 def visit_Continue(self, node):
434 self._new_line()
435 self._write('continue')
436
437 ### EXPRESSIONS
438 def with_parens(f):
439 def _f(self, node):
440 self._write('(')
441 f(self, node)
442 self._write(')')
443 return _f
444
445 bool_operators = {_ast.And: 'and', _ast.Or: 'or'}
446
447 # BoolOp(boolop op, expr* values)
448 @with_parens
449 def visit_BoolOp(self, node):
450 joiner = ' ' + self.bool_operators[node.op.__class__] + ' '
451 self.visit(node.values[0])
452 for value in node.values[1:]:
453 self._write(joiner)
454 self.visit(value)
455
456 binary_operators = {
457 _ast.Add: '+',
458 _ast.Sub: '-',
459 _ast.Mult: '*',
460 _ast.Div: '/',
461 _ast.Mod: '%',
462 _ast.Pow: '**',
463 _ast.LShift: '<<',
464 _ast.RShift: '>>',
465 _ast.BitOr: '|',
466 _ast.BitXor: '^',
467 _ast.BitAnd: '&',
468 _ast.FloorDiv: '//'
469 }
470
471 # BinOp(expr left, operator op, expr right)
472 @with_parens
473 def visit_BinOp(self, node):
474 self.visit(node.left)
475 self._write(' ' + self.binary_operators[node.op.__class__] + ' ')
476 self.visit(node.right)
477
478 unary_operators = {
479 _ast.Invert: '~',
480 _ast.Not: 'not',
481 _ast.UAdd: '+',
482 _ast.USub: '-',
483 }
484
485 # UnaryOp(unaryop op, expr operand)
486 def visit_UnaryOp(self, node):
487 self._write(self.unary_operators[node.op.__class__] + ' ')
488 self.visit(node.operand)
489
490 # Lambda(arguments args, expr body)
491 @with_parens
492 def visit_Lambda(self, node):
493 self._write('lambda ')
494 self.visit(node.args)
495 self._write(': ')
496 self.visit(node.body)
497
498 # IfExp(expr test, expr body, expr orelse)
499 @with_parens
500 def visit_IfExp(self, node):
501 self.visit(node.body)
502 self._write(' if ')
503 self.visit(node.test)
504 self._write(' else ')
505 self.visit(node.orelse)
506
507 # Dict(expr* keys, expr* values)
508 def visit_Dict(self, node):
509 self._write('{')
510 for key, value in zip(node.keys, node.values):
511 self.visit(key)
512 self._write(': ')
513 self.visit(value)
514 self._write(', ')
515 self._write('}')
516
517 # ListComp(expr elt, comprehension* generators)
518 def visit_ListComp(self, node):
519 self._write('[')
520 self.visit(node.elt)
521 for generator in node.generators:
522 # comprehension = (expr target, expr iter, expr* ifs)
523 self._write(' for ')
524 self.visit(generator.target)
525 self._write(' in ')
526 self.visit(generator.iter)
527 for ifexpr in generator.ifs:
528 self._write(' if ')
529 self.visit(ifexpr)
530 self._write(']')
531
532 # GeneratorExp(expr elt, comprehension* generators)
533 def visit_GeneratorExp(self, node):
534 self._write('(')
535 self.visit(node.elt)
536 for generator in node.generators:
537 # comprehension = (expr target, expr iter, expr* ifs)
538 self._write(' for ')
539 self.visit(generator.target)
540 self._write(' in ')
541 self.visit(generator.iter)
542 for ifexpr in generator.ifs:
543 self._write(' if ')
544 self.visit(ifexpr)
545 self._write(')')
546
547 # Yield(expr? value)
548 def visit_Yield(self, node):
549 self._write('yield')
550 if getattr(node, 'value', None):
551 self._write(' ')
552 self.visit(node.value)
553
554 comparision_operators = {
555 _ast.Eq: '==',
556 _ast.NotEq: '!=',
557 _ast.Lt: '<',
558 _ast.LtE: '<=',
559 _ast.Gt: '>',
560 _ast.GtE: '>=',
561 _ast.Is: 'is',
562 _ast.IsNot: 'is not',
563 _ast.In: 'in',
564 _ast.NotIn: 'not in',
565 }
566
567 # Compare(expr left, cmpop* ops, expr* comparators)
568 @with_parens
569 def visit_Compare(self, node):
570 self.visit(node.left)
571 for op, comparator in zip(node.ops, node.comparators):
572 self._write(' ' + self.comparision_operators[op.__class__] + ' ')
573 self.visit(comparator)
574
575 # Call(expr func, expr* args, keyword* keywords,
576 # expr? starargs, expr? kwargs)
577 def visit_Call(self, node):
578 self.visit(node.func)
579 self._write('(')
580 first = True
581 for arg in node.args:
582 if not first:
583 self._write(', ')
584 first = False
585 self.visit(arg)
586
587 for keyword in node.keywords:
588 if not first:
589 self._write(', ')
590 first = False
591 # keyword = (identifier arg, expr value)
592 self._write(keyword.arg)
593 self._write('=')
594 self.visit(keyword.value)
595 if getattr(node, 'starargs', None):
596 if not first:
597 self._write(', ')
598 first = False
599 self._write('*')
600 self.visit(node.starargs)
601
602 if getattr(node, 'kwargs', None):
603 if not first:
604 self._write(', ')
605 first = False
606 self._write('**')
607 self.visit(node.kwargs)
608 self._write(')')
609
610 # Repr(expr value)
611 def visit_Repr(self, node):
612 self._write('`')
613 self.visit(node.value)
614 self._write('`')
615
616 # Num(object n)
617 def visit_Num(self, node):
618 self._write(repr(node.n))
619
620 # Str(string s)
621 def visit_Str(self, node):
622 self._write(repr(node.s))
623
624 # Attribute(expr value, identifier attr, expr_context ctx)
625 def visit_Attribute(self, node):
626 self.visit(node.value)
627 self._write('.')
628 self._write(node.attr)
629
630 # Subscript(expr value, slice slice, expr_context ctx)
631 def visit_Subscript(self, node):
632 self.visit(node.value)
633 self._write('[')
634 def _process_slice(node):
635 if isinstance(node, _ast.Ellipsis):
636 self._write('...')
637 elif isinstance(node, _ast.Slice):
638 if getattr(node, 'lower', 'None'):
639 self.visit(node.lower)
640 self._write(':')
641 if getattr(node, 'upper', None):
642 self.visit(node.upper)
643 if getattr(node, 'step', None):
644 self._write(':')
645 self.visit(node.step)
646 elif isinstance(node, _ast.Index):
647 self.visit(node.value)
648 elif isinstance(node, _ast.ExtSlice):
649 self.visit(node.dims[0])
650 for dim in node.dims[1:]:
651 self._write(', ')
652 self.visit(dim)
653 else:
654 raise NotImplemented, 'Slice type not implemented'
655 _process_slice(node.slice)
656 self._write(']')
657
658 # Name(identifier id, expr_context ctx)
659 def visit_Name(self, node):
660 self._write(node.id)
661
662 # List(expr* elts, expr_context ctx)
663 def visit_List(self, node):
664 self._write('[')
665 for elt in node.elts:
666 self.visit(elt)
667 self._write(', ')
668 self._write(']')
669
670 # Tuple(expr *elts, expr_context ctx)
671 def visit_Tuple(self, node):
672 self._write('(')
673 for elt in node.elts:
674 self.visit(elt)
675 self._write(', ')
676 self._write(')')
677
678
679 class ASTTransformer(object):
680 """General purpose base class for AST transformations.
681
682 Every visitor method can be overridden to return an AST node that has been
683 altered or replaced in some way.
684 """
685
686 def visit(self, node):
687 if node is None:
688 return None
689 if type(node) is tuple:
690 return tuple([self.visit(n) for n in node])
691 visitor = getattr(self, 'visit_%s' % node.__class__.__name__, None)
692 if visitor is None:
693 return node
694 return visitor(node)
695
696 def _clone(self, node):
697 clone = node.__class__()
698 for name in getattr(clone, '_attributes', ()):
699 try:
700 setattr(clone, 'name', getattr(node, name))
701 except AttributeError:
702 pass
703 for name in clone._fields:
704 try:
705 value = getattr(node, name)
706 except AttributeError:
707 pass
708 else:
709 if value is None:
710 pass
711 elif isinstance(value, list):
712 value = [self.visit(x) for x in value]
713 elif isinstance(value, tuple):
714 value = tuple(self.visit(x) for x in value)
715 else:
716 value = self.visit(value)
717 setattr(clone, name, value)
718 return clone
719
720 visit_Module = _clone
721 visit_Interactive = _clone
722 visit_Expression = _clone
723 visit_Suite = _clone
724
725 visit_FunctionDef = _clone
726 visit_ClassDef = _clone
727 visit_Return = _clone
728 visit_Delete = _clone
729 visit_Assign = _clone
730 visit_AugAssign = _clone
731 visit_Print = _clone
732 visit_For = _clone
733 visit_While = _clone
734 visit_If = _clone
735 visit_With = _clone
736 visit_Raise = _clone
737 visit_TryExcept = _clone
738 visit_TryFinally = _clone
739 visit_Assert = _clone
740
741 visit_Import = _clone
742 visit_ImportFrom = _clone
743 visit_Exec = _clone
744 visit_Global = _clone
745 visit_Expr = _clone
746 # Pass, Break, Continue don't need to be copied
747
748 visit_BoolOp = _clone
749 visit_BinOp = _clone
750 visit_UnaryOp = _clone
751 visit_Lambda = _clone
752 visit_IfExp = _clone
753 visit_Dict = _clone
754 visit_ListComp = _clone
755 visit_GeneratorExp = _clone
756 visit_Yield = _clone
757 visit_Compare = _clone
758 visit_Call = _clone
759 visit_Repr = _clone
760 # Num, Str don't need to be copied
761
762 visit_Attribute = _clone
763 visit_Subscript = _clone
764 visit_Name = _clone
765 visit_List = _clone
766 visit_Tuple = _clone
767
768 visit_comprehension = _clone
769 visit_excepthandler = _clone
770 visit_arguments = _clone
771 visit_keyword = _clone
772 visit_alias = _clone
773
774 visit_Slice = _clone
775 visit_ExtSlice = _clone
776 visit_Index = _clone
777
778 del _clone
Copyright (C) 2012-2017 Edgewall Software