# HG changeset patch # User zzzeek # Date 1162674102 0 # Node ID 387d5892244b412077c3ba2923a18b25c6a45b0b # Parent d60a60ba4224711694399cb5c1cda46d02b55706 - further dev on codegen. - had to add a few instance variables to genshi core to help in generating string-based python code; i Expression holds onto its AST node, DefDirective holds onto the original string-based signature. - generator code is a little messy right now, will clean up ! diff --git a/genshi/codegen/generator.py b/genshi/codegen/generator.py --- a/genshi/codegen/generator.py +++ b/genshi/codegen/generator.py @@ -15,64 +15,156 @@ from genshi import template from genshi.template import Template from genshi.codegen.printer import PythonPrinter, PYTHON_LINE, PYTHON_COMMENT, PYTHON_BLOCK +from compiler import ast, parse, visitor +import sets, re _directive_printers = {} - + +def ident_from_assign(assign): + # a little trick to get the variable name from the already + # compiled assignment expression + x = {} + assign(x, None) + return list(x)[0] + class DirectivePrinter(object): def __init__(self): _directive_printers[self.__directive__] = self - def start_directive(self, gencontext, directive): + def produce_directive(self, gencontext, directive, event, substream): pass - def end_directive(self, gencontext, directive): - pass - + def declared_identifiers(self, gencontext, directive, event): + return [] + def undeclared_identifiers(self, gencontext, directive, event): + return [] + class ForDirectivePrinter(DirectivePrinter): __directive__ = template.ForDirective - def start_directive(self, gencontext, directive): - x = {} - directive.assign(x, None) - varname = list(x)[0] + def produce_directive(self, gencontext, directive, event, substream): + varname = ident_from_assign(directive.assign) yield (PYTHON_LINE, "for %s in %s:" % (varname, directive.expr.source)) - def end_directive(self, gencontext, directive): + for evt in gencontext.gen_stream(substream): + yield evt yield (PYTHON_LINE, "") + def declared_identifiers(self, gencontext, directive, event): + return [ident_from_assign(directive.assign)] + def undeclared_identifiers(self, gencontext, directive, event): + s = SearchIdents(directive.expr.source) + return list(s.identifiers) ForDirectivePrinter() +class IfDirectivePrinter(DirectivePrinter): + __directive__ = template.IfDirective + def produce_directive(self, gencontext, directive, event, substream): + yield (PYTHON_LINE, "if %s:" % (directive.expr.source)) + for evt in gencontext.gen_stream(substream): + yield evt + yield (PYTHON_LINE, "") +IfDirectivePrinter() + +class ReplaceDirectivePrinter(DirectivePrinter): + __directive__ = template.ReplaceDirective + def produce_directive(self, gencontext, directive, event, substream): + for expr in gencontext.produce_expr_event(event, directive.expr): + yield expr +ReplaceDirectivePrinter() + +class DefDirectivePrinter(DirectivePrinter): + __directive__ = template.DefDirective + def produce_directive(self, gencontext, directive, event, substream): + sig = directive.signature + if not re.search(r'\(.*\)$', sig): + gencontext.defs_without_params.add(sig) + sig += '()' + yield (PYTHON_LINE, "def %s:" % (sig)) + for evt in gencontext.gen_stream(substream): + yield evt + yield (PYTHON_LINE, "") + def declared_identifiers(self, gencontext, directive, event): + return [directive.name] + directive.args + directive.defaults.keys() + def undeclared_identifiers(self, gencontext, directive, event): + result = sets.Set() + for expr in directive.defaults.values(): + s = SearchIdents(expr.node) + result = result.union(s.identifiers) + return iter(result) + +DefDirectivePrinter() + + class Generator(object): """given a Template, generates Python modules (as strings or code objects) optimized to a particular Serializer.""" def __init__(self, template): self.template = template - def generate(self, serializer): + def generate_stream(self, serializer): return PythonPrinter( PythonGenerator( self.template.stream, serializer ).generate() ).generate() + def generate_module(self, serializer): + import imp + module = imp.new_module("_some_ident") + pycode = u''.join(self.generate_stream(serializer)) + code = compile(pycode, '', 'exec') + exec code in module.__dict__, module.__dict__ + return module + +class SearchIdents(visitor.ASTVisitor): + """an ASTVisitor that can locate identifier names in a string-based code block. + + This is not used in this example module, but will be used to locate and pre-declare + all identifiers that are referenced in expressions at the start of each generated callable.""" + def __init__(self, expr): + self.identifiers = sets.Set() + if isinstance(expr, basestring): + expr = parse(expr, "eval") + visitor.walk(expr, self) + def visitName(self, node, *args, **kwargs): + self.identifiers.add(node.name) class PythonGenerator(object): def __init__(self, stream, serializer): self.stream = stream self.serializer = serializer + self.defs_without_params = sets.Set() def generate(self): for evt in self.start(): yield evt - for evt in self.gen_stream(self.stream): + stream = list(self.stream) + for expr in self.find_identifiers(stream, sets.ImmutableSet()): + yield expr + for evt in self.gen_stream(stream): yield evt for evt in self.end(): yield evt + def find_identifiers(self, stream, stack): + """locate undeclared python identifiers in the given stream. stack is an empty set.""" + for evt in stream: + if evt[0] is template.EXPR: + s = SearchIdents(evt[1].source) + for ident in s.identifiers.difference(stack): + yield (PYTHON_LINE, "%s = context.get('%s', None)" % (ident, ident)) + elif evt[0] is template.SUB: + directives, substream = evt[1] + decl_ident = [] + for d in directives: + decl_ident += self.get_declared_identifiers(d, evt[1][1]) + for ident in self.get_undeclared_identifiers(d, evt[1][1]): + yield (PYTHON_LINE, "%s = context.get('%s', None)" % (ident, ident)) + stack = stack.union(sets.Set(decl_ident)) + for evt in self.find_identifiers(evt[1][1], stack): + yield evt + def gen_stream(self, stream): for event in self.serializer(stream): (kind, data, pos, literal) = event + #print "INCOMING:", event if kind is template.SUB: directives, substream = event[1] for d in directives: - for evt in self.produce_directive_start(d): - yield evt - for evt in self.gen_stream(substream): - yield evt - for d in directives: - for evt in self.produce_directive_end(d): + for evt in self.produce_directive(d, event, substream): yield evt elif kind is template.START: for evt in self.produce_start_event(event): @@ -80,19 +172,26 @@ elif kind is template.END: for evt in self.produce_end_event(event): yield evt + elif kind is template.EXPR: + for evt in self.produce_expr_event(event): + yield evt + elif kind is template.TEXT: + for evt in self.produce_text_event(event): + yield evt def produce_preamble(self): for line in [ "from genshi.core import START, END, START_NS, END_NS, TEXT, COMMENT, DOCTYPE, QName, Stream", - "from genshi.template import Context", - "from genshi.path import Path" + "from genshi.template import Context, Template", + "from genshi.path import Path", + "EXPR = Template.EXPR" ]: yield (PYTHON_LINE, line) - - def produce_directive_start(self, directive): - for evt in _directive_printers[directive.__class__].start_directive(self, directive): - yield evt - def produce_directive_end(self, directive): - for evt in _directive_printers[directive.__class__].end_directive(self, directive): + def get_declared_identifiers(self, directive, event): + return _directive_printers[directive.__class__].declared_identifiers(self, directive, event) + def get_undeclared_identifiers(self, directive, event): + return _directive_printers[directive.__class__].undeclared_identifiers(self, directive, event) + def produce_directive(self, directive, event, substream): + for evt in _directive_printers[directive.__class__].produce_directive(self, directive, event, substream): yield evt def start(self): for evt in self.produce_preamble(): @@ -101,7 +200,7 @@ def end(self): yield (PYTHON_LINE, "") def produce_start_event(self, event): - yield (PYTHON_LINE, "yield (START, (Qname(%s), %s), %s, %s)" % ( + yield (PYTHON_LINE, "yield (START, (QName(%s), %s), %s, %s)" % ( repr(event[1][0]), repr(event[1][1]), repr(event[2]), @@ -113,6 +212,18 @@ repr(event[2]), repr(event[3])) ) + def produce_expr_event(self, event, expr=None): + if expr is None: + expr = event[1] + if expr.source in self.defs_without_params: + yield (PYTHON_LINE, "_expr = %s()" % (expr.source)) + else: + yield (PYTHON_LINE, "_expr = %s" % (expr.source)) + yield (PYTHON_LINE, "yield (EXPR, _expr, %s, unicode(_expr))" % (repr(event[2]))) + def produce_text_event(self, event): + yield (PYTHON_LINE, "yield (TEXT, (%s), %s, %s)" % ( + repr(unicode(event[1])), + repr(event[2]), + repr(unicode(event[3])) + )) - - diff --git a/genshi/codegen/interp.py b/genshi/codegen/interp.py new file mode 100644 --- /dev/null +++ b/genshi/codegen/interp.py @@ -0,0 +1,78 @@ +from genshi.template import MarkupTemplate, Template, Context +from genshi.path import Path +from genshi.output import HTMLSerializer +from compiler import ast, parse, visitor +from genshi.core import START, END, START_NS, END_NS, TEXT, COMMENT, DOCTYPE, QName, Stream +EXPR = Template.EXPR +import sets +from itertools import chain + + +# we re-implement our own _match function, based on MarkupTemplate._match. +def _match(stream, ctxt, match_templates=None): + """match method from MarkupTemplate, modified to handle inlined stream of events. + + comments from the original function in template.py are removed, to highlight new commentary + unique to this method.""" + if match_templates is None: + match_templates = ctxt._match_templates + + tail = [] + def _strip(stream): + depth = 1 + while 1: + event = stream.next() + if event[0] is START: + depth += 1 + elif event[0] is END: + depth -= 1 + if depth > 0: + yield event + else: + tail[:] = [event] + break + + for event in stream: + if not match_templates or (event[0] is not START and + event[0] is not END): + yield event + continue + # no need for a sub-list of directives (nor _apply_directives function) since inlined code + # expands all nesting explicitly (TODO: is this really true ?) + for idx, (test, path, template, namespaces) in \ + enumerate(match_templates): + + if test(event, namespaces, ctxt) is True: + for test in [mt[0] for mt in match_templates[idx + 1:]]: + test(event, namespaces, ctxt, updateonly=True) + + content = chain([event], _match(_strip(stream), ctxt), + tail) + # TODO: not sure if extra list of filters is needed + #for filter_ in self.filters[3:]: + # content = filter_(content, ctxt) + content = list(content) + + for test in [mt[0] for mt in match_templates]: + test(tail[0][0:3], namespaces, ctxt, updateonly=True) + + def select(path): + return render_inline(InlinedPath(path).select(Stream(content), namespaces, ctxt)) + + # ctxt.push()/pop() is usually not going to be necessary for inlined execution + # since python scopes will handle most pushing/popping + ctxt.push(dict(select=select)) + + # similarly, no need for _eval (eval is inlined) as well as _flatten (inlined code already "flattened") + for event in _match(template, ctxt, match_templates[:idx] + match_templates[idx + 1:]): + yield event + + ctxt.pop() + break + else: + yield event + +def run_inlined(module, data): + context = Context(**data) + for item in _match(module.go(context), context): + yield item[3] diff --git a/genshi/codegen/tests/test_generator.py b/genshi/codegen/tests/test_generator.py --- a/genshi/codegen/tests/test_generator.py +++ b/genshi/codegen/tests/test_generator.py @@ -11,9 +11,9 @@ # individuals. For the exact contribution history, see the revision # history and logs, available at http://genshi.edgewall.org/log/. -from genshi.template import MarkupTemplate, Template +from genshi.template import MarkupTemplate, Template, Context from genshi.output import HTMLSerializer -from genshi.codegen import generator +from genshi.codegen import generator, interp from genshi.codegen.serialize import HTMLSerializeFilter text = """
+ ${lala + 'hi'}
i am a greeting, ${item} + + now heres replace + Hey Ho
+ +

+ ${greeting}, ${name}! +

+ ${echo('Hi', name='you')} +

+ Hello, world! +

+ ${helloworld} yo """ +def items(): + return ["one", "two", "three"] + +data = {'lala':'hi', 'items':items, 'foo':['f1', 'f2', 'f3']} + t = MarkupTemplate(text) g = generator.Generator(t) -print u''.join(g.generate(HTMLSerializeFilter())) +pycode = u''.join(g.generate_stream(HTMLSerializeFilter())) +print pycode + +g = generator.Generator(t) +module = g.generate_module(HTMLSerializeFilter()) +print u''.join(interp.run_inlined(module, data)) + diff --git a/genshi/eval.py b/genshi/eval.py --- a/genshi/eval.py +++ b/genshi/eval.py @@ -62,7 +62,7 @@ >>> Expression('len(items)').evaluate(data) 3 """ - __slots__ = ['source', 'code'] + __slots__ = ['source', 'code', 'node'] def __init__(self, source, filename=None, lineno=-1): """Create the expression, either from a string, or from an AST node. @@ -75,11 +75,13 @@ """ if isinstance(source, basestring): self.source = source - self.code = _compile(_parse(source), self.source, filename=filename, + self.node = _parse(source) + self.code = _compile(self.node, self.source, filename=filename, lineno=lineno) else: assert isinstance(source, ast.Node) self.source = '?' + self.node = source self.code = _compile(ast.Expression(source), filename=filename, lineno=lineno) diff --git a/genshi/template.py b/genshi/template.py --- a/genshi/template.py +++ b/genshi/template.py @@ -352,7 +352,7 @@

""" - __slots__ = ['name', 'args', 'defaults'] + __slots__ = ['name', 'args', 'defaults', 'signature'] ATTRIBUTE = 'function' @@ -360,6 +360,7 @@ offset=-1): Directive.__init__(self, None, namespaces, filename, lineno, offset) ast = _parse(args).node + self.signature = args self.args = [] self.defaults = {} if isinstance(ast, compiler.ast.CallFunc):