changeset 327:c72243905470 experimental-compiler

- further dev on codegen. - had to add a few instance variables to genshi core to help in generating string-based python code; i Expression holds onto its AST node, DefDirective holds onto the original string-based signature. - generator code is a little messy right now, will clean up !
author zzzeek
date Sat, 04 Nov 2006 21:01:42 +0000
parents 59b860d15675
children f013a84050ac
files genshi/codegen/generator.py genshi/codegen/interp.py genshi/codegen/tests/test_generator.py genshi/eval.py genshi/template.py
diffstat 5 files changed, 251 insertions(+), 35 deletions(-) [+]
line wrap: on
line diff
--- a/genshi/codegen/generator.py
+++ b/genshi/codegen/generator.py
@@ -15,64 +15,156 @@
 from genshi import template
 from genshi.template import Template
 from genshi.codegen.printer import PythonPrinter, PYTHON_LINE, PYTHON_COMMENT, PYTHON_BLOCK
+from compiler import ast, parse, visitor
+import sets, re
 
 _directive_printers = {}
-        
+
+def ident_from_assign(assign):
+    # a little trick to get the variable name from the already 
+    # compiled assignment expression
+    x = {}
+    assign(x, None)
+    return list(x)[0]
+    
 class DirectivePrinter(object):
     def __init__(self):
         _directive_printers[self.__directive__] = self
-    def start_directive(self, gencontext, directive):
+    def produce_directive(self, gencontext, directive, event, substream):
         pass
-    def end_directive(self, gencontext, directive):
-        pass
-        
+    def declared_identifiers(self, gencontext, directive, event):
+        return []
+    def undeclared_identifiers(self, gencontext, directive, event):
+        return []
+                
 class ForDirectivePrinter(DirectivePrinter):
     __directive__ = template.ForDirective
-    def start_directive(self, gencontext, directive):
-        x = {}
-        directive.assign(x, None)
-        varname = list(x)[0]
+    def produce_directive(self, gencontext, directive, event, substream):
+        varname = ident_from_assign(directive.assign)
         yield (PYTHON_LINE, "for %s in %s:" % (varname, directive.expr.source))
-    def end_directive(self, gencontext, directive):
+        for evt in gencontext.gen_stream(substream):
+            yield evt
         yield (PYTHON_LINE, "")
+    def declared_identifiers(self, gencontext, directive, event):
+        return [ident_from_assign(directive.assign)]
+    def undeclared_identifiers(self, gencontext, directive, event):
+        s = SearchIdents(directive.expr.source)
+        return list(s.identifiers)
 ForDirectivePrinter()
 
+class IfDirectivePrinter(DirectivePrinter):
+    __directive__ = template.IfDirective
+    def produce_directive(self, gencontext, directive, event, substream):
+        yield (PYTHON_LINE, "if %s:" % (directive.expr.source))
+        for evt in gencontext.gen_stream(substream):
+            yield evt
+        yield (PYTHON_LINE, "")
+IfDirectivePrinter()
+
+class ReplaceDirectivePrinter(DirectivePrinter):
+    __directive__ = template.ReplaceDirective
+    def produce_directive(self, gencontext, directive, event, substream):
+        for expr in gencontext.produce_expr_event(event, directive.expr):
+            yield expr
+ReplaceDirectivePrinter()
+
+class DefDirectivePrinter(DirectivePrinter):
+    __directive__ = template.DefDirective
+    def produce_directive(self, gencontext, directive, event, substream):
+        sig = directive.signature
+        if not re.search(r'\(.*\)$', sig):
+            gencontext.defs_without_params.add(sig)
+            sig += '()'
+        yield (PYTHON_LINE, "def %s:" % (sig))
+        for evt in gencontext.gen_stream(substream):
+            yield evt
+        yield (PYTHON_LINE, "")
+    def declared_identifiers(self, gencontext, directive, event):
+        return [directive.name] + directive.args + directive.defaults.keys()
+    def undeclared_identifiers(self, gencontext, directive, event):
+        result = sets.Set()
+        for expr in directive.defaults.values():
+            s = SearchIdents(expr.node)
+            result = result.union(s.identifiers)
+        return iter(result)
+            
+DefDirectivePrinter()
+    
+
 class Generator(object):
     """given a Template, generates Python modules (as strings or code objects)
     optimized to a particular Serializer."""
     def __init__(self, template):
         self.template = template
-    def generate(self, serializer):
+    def generate_stream(self, serializer):
         return PythonPrinter(
             PythonGenerator(
                 self.template.stream, serializer
             ).generate()
         ).generate()
+    def generate_module(self, serializer):
+        import imp
+        module = imp.new_module("_some_ident")
+        pycode = u''.join(self.generate_stream(serializer))
+        code = compile(pycode, '<String>', 'exec')
+        exec code in module.__dict__, module.__dict__
+        return module
+        
+class SearchIdents(visitor.ASTVisitor):
+    """an ASTVisitor that can locate identifier names in a string-based code block.
+
+    This is not used in this example module, but will be used to locate and pre-declare 
+    all identifiers that are referenced in expressions at the start of each generated callable."""
+    def __init__(self, expr):
+        self.identifiers = sets.Set()
+        if isinstance(expr, basestring):
+            expr = parse(expr, "eval")
+        visitor.walk(expr, self)
+    def visitName(self, node, *args, **kwargs):
+        self.identifiers.add(node.name)
 
 class PythonGenerator(object):
     def __init__(self, stream, serializer):
         self.stream = stream
         self.serializer = serializer
+        self.defs_without_params = sets.Set()
     def generate(self):
         for evt in self.start():
             yield evt
-        for evt in self.gen_stream(self.stream):
+        stream = list(self.stream)
+        for expr in self.find_identifiers(stream, sets.ImmutableSet()):
+            yield expr
+        for evt in self.gen_stream(stream):
             yield evt
         for  evt in self.end():
             yield evt
 
+    def find_identifiers(self, stream, stack):
+        """locate undeclared python identifiers in the given stream.  stack is an empty set."""
+        for evt in stream:
+            if evt[0] is template.EXPR:
+                s = SearchIdents(evt[1].source)
+                for ident in s.identifiers.difference(stack):
+                    yield (PYTHON_LINE, "%s = context.get('%s', None)" % (ident, ident))
+            elif evt[0] is template.SUB:
+                directives, substream = evt[1]
+                decl_ident = []
+                for d in directives:
+                    decl_ident += self.get_declared_identifiers(d, evt[1][1])
+                    for ident in self.get_undeclared_identifiers(d, evt[1][1]):
+                        yield (PYTHON_LINE, "%s = context.get('%s', None)" % (ident, ident))
+                stack = stack.union(sets.Set(decl_ident))
+                for evt in self.find_identifiers(evt[1][1], stack):
+                    yield evt
+                
     def gen_stream(self, stream):
         for event in self.serializer(stream):
             (kind, data, pos, literal) = event
+            #print "INCOMING:", event
             if kind is template.SUB:
                 directives, substream = event[1]
                 for d in directives:
-                    for evt in self.produce_directive_start(d):
-                        yield evt
-                for evt in self.gen_stream(substream):
-                    yield evt
-                for d in directives:
-                    for evt in self.produce_directive_end(d):
+                    for evt in self.produce_directive(d, event, substream):
                         yield evt
             elif kind is template.START:
                 for evt in self.produce_start_event(event):
@@ -80,19 +172,26 @@
             elif kind is template.END:
                 for evt in self.produce_end_event(event):
                     yield evt
+            elif kind is template.EXPR:
+                for evt in self.produce_expr_event(event):
+                    yield evt
+            elif kind is template.TEXT:
+                for evt in self.produce_text_event(event):
+                    yield evt
     def produce_preamble(self):
         for line in [
             "from genshi.core import START, END, START_NS, END_NS, TEXT, COMMENT, DOCTYPE, QName, Stream",
-            "from genshi.template import Context",
-            "from genshi.path import Path"
+            "from genshi.template import Context, Template",
+            "from genshi.path import Path",
+            "EXPR = Template.EXPR"
         ]:
             yield (PYTHON_LINE, line)
-
-    def produce_directive_start(self, directive):
-        for evt in _directive_printers[directive.__class__].start_directive(self, directive):
-            yield evt
-    def produce_directive_end(self, directive):
-        for evt in _directive_printers[directive.__class__].end_directive(self, directive):
+    def get_declared_identifiers(self, directive, event):
+        return _directive_printers[directive.__class__].declared_identifiers(self, directive, event)
+    def get_undeclared_identifiers(self, directive, event):
+        return _directive_printers[directive.__class__].undeclared_identifiers(self, directive, event)
+    def produce_directive(self, directive, event, substream):
+        for evt in _directive_printers[directive.__class__].produce_directive(self, directive, event, substream):
             yield evt
     def start(self):
         for evt in self.produce_preamble():
@@ -101,7 +200,7 @@
     def end(self):
         yield (PYTHON_LINE, "")
     def produce_start_event(self, event):
-        yield (PYTHON_LINE, "yield (START, (Qname(%s), %s), %s, %s)" % (
+        yield (PYTHON_LINE, "yield (START, (QName(%s), %s), %s, %s)" % (
             repr(event[1][0]), 
             repr(event[1][1]), 
             repr(event[2]), 
@@ -113,6 +212,18 @@
             repr(event[2]), 
             repr(event[3]))
         )
+    def produce_expr_event(self, event, expr=None):
+        if expr is None:
+            expr = event[1]
+        if expr.source in self.defs_without_params:
+            yield (PYTHON_LINE, "_expr = %s()" % (expr.source))
+        else:
+            yield (PYTHON_LINE, "_expr = %s" % (expr.source))
+        yield (PYTHON_LINE, "yield (EXPR, _expr, %s, unicode(_expr))" % (repr(event[2])))
+    def produce_text_event(self, event):
+        yield (PYTHON_LINE, "yield (TEXT, (%s), %s, %s)" % (
+            repr(unicode(event[1])),
+            repr(event[2]),
+            repr(unicode(event[3]))
+        ))
 
-
-
new file mode 100644
--- /dev/null
+++ b/genshi/codegen/interp.py
@@ -0,0 +1,78 @@
+from genshi.template import MarkupTemplate, Template, Context
+from genshi.path import Path
+from genshi.output import HTMLSerializer
+from compiler import ast, parse, visitor
+from genshi.core import START, END, START_NS, END_NS, TEXT, COMMENT, DOCTYPE, QName, Stream
+EXPR = Template.EXPR
+import sets
+from itertools import chain
+
+
+# we re-implement our own _match function, based on MarkupTemplate._match.
+def _match(stream, ctxt, match_templates=None):
+    """match method from MarkupTemplate, modified to handle inlined stream of events.
+    
+    comments from the original function in template.py are removed, to highlight new commentary
+    unique to this method."""
+    if match_templates is None:
+        match_templates = ctxt._match_templates
+
+    tail = []
+    def _strip(stream):
+        depth = 1
+        while 1:
+            event = stream.next()
+            if event[0] is START:
+                depth += 1
+            elif event[0] is END:
+                depth -= 1
+            if depth > 0:
+                yield event
+            else:
+                tail[:] = [event]
+                break
+
+    for event in stream:
+        if not match_templates or (event[0] is not START and
+                                   event[0] is not END):
+            yield event
+            continue
+        # no need for a sub-list of directives (nor _apply_directives function) since inlined code 
+        # expands all nesting explicitly (TODO: is this really true ?)
+        for idx, (test, path, template, namespaces) in \
+                enumerate(match_templates):
+
+            if test(event, namespaces, ctxt) is True:
+                for test in [mt[0] for mt in match_templates[idx + 1:]]:
+                    test(event, namespaces, ctxt, updateonly=True)
+                    
+                content = chain([event], _match(_strip(stream), ctxt),
+                                tail)
+                # TODO: not sure if extra list of filters is needed
+                #for filter_ in self.filters[3:]:
+                #    content = filter_(content, ctxt)
+                content = list(content)
+
+                for test in [mt[0] for mt in match_templates]:
+                    test(tail[0][0:3], namespaces, ctxt, updateonly=True)
+
+                def select(path):
+                    return render_inline(InlinedPath(path).select(Stream(content), namespaces, ctxt))
+                
+                # ctxt.push()/pop() is usually not going to be necessary for inlined execution
+                # since python scopes will handle most pushing/popping
+                ctxt.push(dict(select=select))
+
+                # similarly, no need for _eval (eval is inlined) as well as _flatten (inlined code already "flattened")
+                for event in _match(template, ctxt, match_templates[:idx] + match_templates[idx + 1:]):
+                    yield event
+
+                ctxt.pop()
+                break
+        else:
+            yield event
+
+def run_inlined(module, data):
+    context = Context(**data)
+    for item in _match(module.go(context), context):
+        yield item[3]
--- a/genshi/codegen/tests/test_generator.py
+++ b/genshi/codegen/tests/test_generator.py
@@ -11,9 +11,9 @@
 # individuals. For the exact contribution history, see the revision
 # history and logs, available at http://genshi.edgewall.org/log/.
 
-from genshi.template import MarkupTemplate, Template
+from genshi.template import MarkupTemplate, Template, Context
 from genshi.output import HTMLSerializer
-from genshi.codegen import generator
+from genshi.codegen import generator, interp
 from genshi.codegen.serialize import HTMLSerializeFilter
 
 text = """<!DOCTYPE html
@@ -25,16 +25,40 @@
       lang="en">
  <body>
     <div py:for="item in items()">
+        ${lala + 'hi'}
         <div py:for="x in foo">
         i am a greeting, ${item}
+        
+        now heres replace
+        <span py:replace="item">Hey Ho</span>
         </div>
     </div>
+
+    <p py:def="echo(greeting, name='world', value=somecontextvalue)" class="message">
+        ${greeting}, ${name}!
+    </p>
+    ${echo('Hi', name='you')}
     
+    <p py:def="helloworld" class="message">
+        Hello, world!
+      </p>
+    ${helloworld}
      yo
      <hi></hi>
  </body>
 </html>"""
 
+def items():
+    return ["one", "two", "three"]
+
+data = {'lala':'hi', 'items':items, 'foo':['f1', 'f2', 'f3']}
+    
 t = MarkupTemplate(text)
 g = generator.Generator(t)
-print u''.join(g.generate(HTMLSerializeFilter()))
+pycode =  u''.join(g.generate_stream(HTMLSerializeFilter()))
+print pycode
+
+g = generator.Generator(t)
+module = g.generate_module(HTMLSerializeFilter())
+print u''.join(interp.run_inlined(module, data))
+
--- a/genshi/eval.py
+++ b/genshi/eval.py
@@ -62,7 +62,7 @@
     >>> Expression('len(items)').evaluate(data)
     3
     """
-    __slots__ = ['source', 'code']
+    __slots__ = ['source', 'code', 'node']
 
     def __init__(self, source, filename=None, lineno=-1):
         """Create the expression, either from a string, or from an AST node.
@@ -75,11 +75,13 @@
         """
         if isinstance(source, basestring):
             self.source = source
-            self.code = _compile(_parse(source), self.source, filename=filename,
+            self.node = _parse(source)
+            self.code = _compile(self.node, self.source, filename=filename,
                                  lineno=lineno)
         else:
             assert isinstance(source, ast.Node)
             self.source = '?'
+            self.node = source
             self.code = _compile(ast.Expression(source), filename=filename,
                                  lineno=lineno)
 
--- a/genshi/template.py
+++ b/genshi/template.py
@@ -352,7 +352,7 @@
       </p>
     </div>
     """
-    __slots__ = ['name', 'args', 'defaults']
+    __slots__ = ['name', 'args', 'defaults', 'signature']
 
     ATTRIBUTE = 'function'
 
@@ -360,6 +360,7 @@
                  offset=-1):
         Directive.__init__(self, None, namespaces, filename, lineno, offset)
         ast = _parse(args).node
+        self.signature = args
         self.args = []
         self.defaults = {}
         if isinstance(ast, compiler.ast.CallFunc):
Copyright (C) 2012-2017 Edgewall Software