# HG changeset patch # User cmlenz # Date 1207903331 0 # Node ID acf7c5ee36e72945ed5d9771da916d3df0a52f2e # Parent 837786a584d54e22e7fe23f725de019adc8302b9 newctxt branch: Merged revisions [678:835] via svnmerge from [source:trunk]. diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -17,11 +17,90 @@ only if the template loader is not set to do automatic reloading. Included templates are basically inlined into the including template, which can speed up rendering of that template a bit. + * Added new syntax for text templates, which is more powerful and flexible + with respect to white-space and line breaks. It also supports Python code + blocks. The old syntax is still available and the default for now, but in a + future release the new syntax will become the default, and some time after + that the old syntax will be removed. + * Added support for passing optimization hints to `` directives, + which can speed up match templates in many cases, for example when a match + template should only be applied once to a stream, or when it should not be + applied recursively. + * Text templates now default to rendering as plain text; it is no longer + necessary to explicitly specify the "text" method to the `render()` or + `serialize()` method of the generated markup stream. + * XInclude elements in markup templates now support the `parse` attribute; + when set to "xml" (the default), the include is processed as before, but + when set to "text", the included template is parsed as a text template using + the new syntax (ticket #101). + * Python code blocks inside match templates are now executed (ticket #155). + * The template engine plugin no longer adds the `default_doctype` when the + `fragment` parameter is `True`. + * The `striptags` function now also removes HTML/XML-style comments (ticket + #150). + * The `py:replace` directive can now also be used as an element, with an + attribute named `value` (ticket #144). + * The `TextSerializer` class no longer strips all markup in text by default, + so that it is still possible to use the Genshi `escape` function even with + text templates. The old behavior is available via the `strip_markup` option + of the serializer (ticket #146). + * Assigning to a variable named `data` in a Python code block no longer + breaks context lookup. + * The `Stream.render` now accepts an optional `out` parameter that can be + used to pass in a writable file-like object to use for assembling the + output, instead of building a big string and returning it. + * The XHTML serializer now strips `xml:space` attributes as they are only + allowed on very few tags. + * Match templates are now applied in a more controlled fashion: in the order + they are declared in the template source, all match templates up to (and + including) the matching template itself are applied to the matched content, + whereas the match templates declared after the matching template are only + applied to the generated content (ticket #186). + * The `TemplateLoader` class now provides an `_instantiate()` method that can + be overridden by subclasses to implement advanced template instantiation + logic (ticket #204). + * The search path of the `TemplateLoader` class can now contain ''load + functions'' in addition to path strings. A load function is passed the + name of the requested template file, and should return a file-like object + and some metadata. New load functions are supplied for loading from egg + package data, and loading from different loaders depending on the path + prefix of the requested filename (ticket #182). + * Match templates can now be processed without keeping the complete matched + content in memory, which could cause excessive memory use on long pages. + The buffering can be disabled using the new `buffer` optimization hint on + the `` directive. + * Improve error reporting when accessing an attribute in a Python expression + raises an `AttributeError` (ticket #191). + * The `Markup` class now supports mappings for right hand of the `%` (modulo) + operator in the same way the Python string classes do, except that the + substituted values are escape. Also, the special constructor which took + positional arguments that would be substituted was removed. Thus the + `Markup` class now supports the same arguments as that of its `unicode` + base class (ticket #211). + * The `Template` class and its subclasses, as well as the interpolation API, + now take an `filepath` parameter instead of `basedir` (ticket #207). + + +Version 0.4.4 +http://svn.edgewall.org/repos/genshi/tags/0.4.4/ +(Aug 14, 2007, from branches/stable/0.4.x) + + * Fixed augmented assignment to local variables in Python code blocks. + * Fixed handling of nested function and class definitions in Python code + blocks. + * Includes were not raising `TemplateNotFound` exceptions even when no + fallback has been specified. That has been corrected. + * The template loader now raises a `TemplateNotFound` error when a previously + cached template is removed or renamed, where it previously was passing up + an `OSError`. + * The Genshi I18n filter can be configured to only extract messages found in + `gettext` function calls, ignoring any text nodes and attribute values + (ticket #138). Version 0.4.3 http://svn.edgewall.org/repos/genshi/tags/0.4.3/ -(?, from branches/stable/0.4.x) +(Jul 17 2007, from branches/stable/0.4.x) * The I18n filter no longer extracts or translates literal strings in attribute values that also contain expressions. @@ -34,11 +113,16 @@ ignored tags (ticket #132). * The HTML sanitizer now strips any CSS comments in style attributes, which could previously be used to hide malicious property values. + * The HTML sanitizer now also removes any HTML comments encountered, as those + may be used to hide malicious payloads targetting a certain "innovative" + browser that goes and interprets the content of specially prepared comments. + * Attribute access in template expressions no longer silently ignores + exceptions other than `AttributeError` raised in the attribute accessor. Version 0.4.2 http://svn.edgewall.org/repos/genshi/tags/0.4.2/ -(Jun 20, from branches/stable/0.4.x) +(Jun 20 2007, from branches/stable/0.4.x) * The `doctype` parameter of the markup serializers now also accepts the "name" of the doctype as string, in addition to the `(name, pubid, sysid)` tuple. diff --git a/INSTALL.txt b/INSTALL.txt deleted file mode 100644 --- a/INSTALL.txt +++ /dev/null @@ -1,37 +0,0 @@ -Installing Genshi -================= - -Prerequisites -------------- - - * Python 2.3 or later (2.4 or later is recommended) - * Optional: setuptools 0.6a2 or later - - -Installation ------------- - -Once you've downloaded and unpacked a Genshi source release, enter the -directory where the archive was unpacked, and run: - - $ python setup.py install - -Note that you may need administrator/root privileges for this step, as -this command will by default attempt to install Genshi to the Python -site-packages directory on your system. - -For advanced options, please refer to the easy_install and/or the distutils -documentation: - - http://peak.telecommunity.com/DevCenter/EasyInstall - http://docs.python.org/inst/inst.html - - -Support -------- - -If you encounter any problems with Genshi, please don't hesitate to ask -questions on the Genshi mailing list or IRC channel: - - http://genshi.edgewall.org/wiki/MailingList - http://genshi.edgewall.org/wiki/IrcChannel diff --git a/README.txt b/README.txt --- a/README.txt +++ b/README.txt @@ -1,11 +1,12 @@ About Genshi ============ -Genshi is a Python library that provides an integrated set of components -for parsing, generating, and processing HTML, XML or other textual -content for output generation on the web. The major feature is a -template language, which is heavily inspired by Kid. +Genshi is a Python library that provides an integrated set of +components for parsing, generating, and processing HTML, XML or other +textual content for output generation on the web. The major feature is +a template language, which is heavily inspired by Kid. -For more information please visit the Genshi web site: +For more information please see the documentation in the `doc` +directory, and visit the Genshi web site: diff --git a/UPGRADE.txt b/UPGRADE.txt deleted file mode 100644 --- a/UPGRADE.txt +++ /dev/null @@ -1,51 +0,0 @@ -Upgrading Genshi -================ - -Upgrading from Genshi 0.3.x to 0.4.x ------------------------------------- - -The modules ``genshi.filters`` and ``genshi.template`` have been -refactored into packages containing multiple modules. While code using -the regular APIs should continue to work without problems, you should -make sure to remove any leftover traces of the ``template.py`` file on -the installation path. This is not necessary when Genshi was installed -as a Python egg. - -Results of evaluating template expressions are no longer implicitly -called if they are callable. If you have been using that feature, you -will need to add the parenthesis to actually call the function. - -Instances of `genshi.core.Attrs` are now immutable. Filters -manipulating the attributes in a stream may need to be updated. Also, -the `Attrs` class no longer automatically wraps all attribute names -in `QName` objects, so users of the `Attrs` class need to do this -themselves. See the documentation of the `Attrs` class for more -information. - - -Upgrading from Markup ---------------------- - -Prior to version 0.3, the name of the Genshi project was "Markup". The -name change means that you will have to adjust your import statements -and the namespace URI of XML templates, among other things: - - * The package name was changed from "markup" to "genshi". Please - adjust any import statements referring to the old package name. - * The namespace URI for directives in Genshi XML templates has changed - from http://markup.edgewall.org/ to http://genshi.edgewall.org/. - Please update the xmlns:py declaration in your template files - accordingly. - -Furthermore, due to the inclusion of a text-based template language, -the class: - - `markup.template.Template` - -has been renamed to: - - `genshi.template.MarkupTemplate` - -If you've been using the Template class directly, you'll need to -update your code (a simple find/replace should do--the API itself -did not change). diff --git a/doc/conf/docutils.ini b/doc/conf/docutils.ini deleted file mode 100644 --- a/doc/conf/docutils.ini +++ /dev/null @@ -1,9 +0,0 @@ -[general] -input_encoding = utf-8 -strip_comments = yes -toc_backlinks = none - -[html4css1 writer] -embed_stylesheet = no -stylesheet = style/edgewall.css -xml_declaration = no diff --git a/doc/conf/epydoc.ini b/doc/conf/epydoc.ini deleted file mode 100644 --- a/doc/conf/epydoc.ini +++ /dev/null @@ -1,24 +0,0 @@ -[epydoc] - -name: Documentation Index -url: ../index.html -modules: genshi -verbosity: 1 - -# Extraction -docformat: restructuredtext -parse: yes -introspect: yes -exclude: .*\.tests.* -inheritance: listed -private: no -imports: no -include-log: no - -# HTML output -output: html -target: doc/api/ -css: doc/style/epydoc.css -top: genshi -frames: no -sourcecode: no diff --git a/doc/filters.txt b/doc/filters.txt --- a/doc/filters.txt +++ b/doc/filters.txt @@ -19,7 +19,7 @@ ================ The filter ``genshi.filters.html.HTMLFormFiller`` can automatically populate an -HTML form from values provided as a simple dictionary. When using thi filter, +HTML form from values provided as a simple dictionary. When using this filter, you can basically omit any ``value``, ``selected``, or ``checked`` attributes from form controls in your templates, and let the filter do all that work for you. diff --git a/doc/i18n.txt b/doc/i18n.txt --- a/doc/i18n.txt +++ b/doc/i18n.txt @@ -145,20 +145,31 @@ .. _`text templates`: text-templates.html ``encoding`` ------------------- +------------ The encoding of the template file. This is only used for text templates. The default is to assume “utf-8”. ``include_attrs`` ------------------- +----------------- Comma-separated list of attribute names that should be considered to have localizable values. Only used for markup templates. -``include_tags`` ------------------- +``ignore_tags`` +--------------- Comma-separated list of tag names that should be ignored. Only used for markup templates. +``extract_text`` +---------------- +Whether text outside explicit ``gettext`` function calls should be extracted. +By default, any text nodes not inside ignored tags, and values of attribute in +the ``include_attrs`` list are extracted. If this option is disabled, only +strings in ``gettext`` function calls are extracted. + +.. note:: If you disable this option, it's not necessary to add the translation + filter as described above. You only need to make sure that the + template has access to the ``gettext`` functions it uses. + Translation =========== diff --git a/doc/index.txt b/doc/index.txt --- a/doc/index.txt +++ b/doc/index.txt @@ -1,8 +1,8 @@ .. -*- mode: rst; encoding: utf-8 -*- -====== -Genshi -====== +======= +Preface +======= .. image:: logo.png :width: 225 @@ -11,15 +11,24 @@ :alt: Genshi - Generate output for the web :class: logo ---------------------------------------------------------- -Toolkit for stream-based generation of output for the web ---------------------------------------------------------- +-------------------------------------------- +Toolkit for generation of output for the web +-------------------------------------------- Genshi is a Python library that provides an integrated set of components for parsing, generating, and processing HTML, XML or other textual content for output generation on the web. The major feature is a template language, which is heavily inspired by Kid. +Installation +------------ + +* `Installing Genshi `_ +* `Upgrading from Previous Versions `_ + +Usage +----- + * `Markup Streams `_ * `Templating Basics `_ * `XML Template Language `_ @@ -28,4 +37,8 @@ * `Using XPath `_ * `Internationalization and Localization `_ * `Using the Templating Plugin `_ + +API Documentation +----------------- + * `Generated API Documentation `_ diff --git a/doc/install.txt b/doc/install.txt new file mode 100644 --- /dev/null +++ b/doc/install.txt @@ -0,0 +1,88 @@ +Installing Genshi +================= + + +.. contents:: Contents + :depth: 2 +.. sectnum:: + + +Prerequisites +------------- + +* Python_ 2.3 or later (2.4 or later is strongly recommended) +* Optional: Setuptools_ 0.6a2 or later + +.. _python: http://www.python.org/ +.. _setuptools: http://cheeseshop.python.org/pypi/setuptools + +Setuptools is only required for the `template engine plugin`_, which can be +used to integrate Genshi with Python web application frameworks such as Pylons +or TurboGears. Genshi also provides a Setuptools-based plugin that integrates +its `internationalization support`_ with the Babel_ library, but that support +can also be used without Setuptools being available (although in a slightly +less convenient fashion). + +.. _`template engine plugin`: plugin.html +.. _`internationalization support`: i18n.html +.. _babel: http://babel.edgewall.org/ + + +Installing via ``easy_install`` +------------------------------- + +If you have a recent version of Setuptools_ installed, you can directly install +Genshi using the easy_install command-line tool:: + + $ easy_install Genshi + +This downloads and installs the latest version of the Genshi package. + +If you have an older Genshi release installed and would like to upgrade, add +the ``-U`` option to the above command. + + +Installing from a Binary Installer +---------------------------------- + +Binary packages for Windows and Mac OS X are provided for Genshi. To install +from such a package, simply download and open it. + + +Installing from a Source Tarball +-------------------------------- + +Once you've downloaded and unpacked a Genshi source release, enter the +directory where the archive was unpacked, and run:: + + $ python setup.py install + +Note that you may need administrator/root privileges for this step, as this +command will by default attempt to install Genshi to the Python +``site-packages`` directory on your system. + +Genshi comes with an optional extension module written in C that is used to +improve performance in some areas. This extension is automatically compiled +when you run the ``setup.py`` script as shown above. In the case that the +extension can not be compiled, possibly due to a missing or incompatible C +compiler, the compilation is skipped. If you'd prefer Genshi to not use this +native extension module, you can explicitly bypass the compilation using the +``--without-speedups`` option:: + + $ python setup.py --without-speedups install + +For other build and installation options, please consult the easy_install_ +and/or the Python distutils_ documentation. + +.. _easy_install: http://peak.telecommunity.com/DevCenter/EasyInstall +.. _distutils: http://docs.python.org/inst/inst.html + + +Support +------- + +If you encounter any problems with Genshi, please don't hesitate to ask +questions on the Genshi `mailing list`_ or `IRC channel`_. + +.. _`mailing list`: http://genshi.edgewall.org/wiki/MailingList +.. _`irc channel`: http://genshi.edgewall.org/wiki/IrcChannel diff --git a/doc/plugin.txt b/doc/plugin.txt --- a/doc/plugin.txt +++ b/doc/plugin.txt @@ -222,6 +222,14 @@ The default value is **25**. You may want to choose a higher value if your web site uses a larger number of templates, and you have enough memory to spare. +``genshi.new_text_syntax`` +-------------------------- +Whether the new syntax for text templates should be used. Specify "yes" to +enable the new syntax, or "no" to use the old syntax. + +In the version of Genshi, the default is to use the old syntax for +backwards-compatibility, but that will change in a future release. + ``genshi.search_path`` ---------------------- A colon-separated list of file-system path names that the template loader should diff --git a/doc/style/bkgnd_pattern.png b/doc/style/bkgnd_pattern.png deleted file mode 100644 index 90e92682135d3f7213332f870f973bd06d6d57ee..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch literal 0 Hc$@>> from genshi.template import MarkupTemplate >>> tmpl = MarkupTemplate('

Hello, $name!

') >>> stream = tmpl.generate(name='world') - >>> print stream.render() + >>> print stream.render('xhtml')

Hello, world!

+.. note:: See the Serialization_ section of the `Markup Streams`_ page for + information on configuring template output options. + Using a text template is similar: .. code-block:: pycon @@ -126,13 +129,15 @@ >>> from genshi.template import TextTemplate >>> tmpl = TextTemplate('Hello, $name!') >>> stream = tmpl.generate(name='world') - >>> print stream.render() + >>> print stream Hello, world! -.. note:: See the Serialization_ section of the `Markup Streams`_ page for - information on configuring template output options. +.. note:: If you want to use text templates, you should consider using the + ``NewTextTemplate`` class instead of simply ``TextTemplate``. See + the `Text Template Language`_ page. .. _serialization: streams.html#serialization +.. _`Text Template Language`: text-templates.html .. _`Markup Streams`: streams.html Using a template loader provides the advantage that “compiled” templates are @@ -203,8 +208,8 @@ Code Blocks =========== -XML templates also support full Python code blocks using the ```` -processing instruction: +Templates also support full Python code blocks, using the ```` +processing instruction in XML templates: .. code-block:: genshi @@ -212,21 +217,38 @@ + return tag.b('Hello, %s!' % name) ?> ${greeting('world')} This will produce the following output: -.. code-block:: genshi +.. code-block:: xml
Hello, world!
+In text templates (although only those using the new syntax introduced in +Genshi 0.5), code blocks use the special ``{% python %}`` directive: + +.. code-block:: genshitext + + {% python + from genshi.builder import tag + def greeting(name): + return 'Hello, %s!' % name + %} + ${greeting('world')} + +This will produce the following output:: + + Hello, world! + + Code blocks can import modules, define classes and functions, and basically do anything you can do in normal Python code. What code blocks can *not* do is to -produce content that is included directly in the generated page. +produce content that is emitted directly tp the generated output. .. note:: Using the ``print`` statement will print to the standard output stream, just as it does for other Python code in your application. @@ -248,7 +270,11 @@ into a sandboxable template engine; there are sufficient ways to do harm even using plain expressions. -.. note:: Code blocks are not currently supported in text templates. +.. warning:: Unfortunately, code blocks are severely limited when running + under Python 2.3: For example, it is not possible to access + variables defined in outer scopes. If you plan to use code blocks + extensively, it is strongly recommended that you run Python 2.4 + or later. .. _`error handling`: @@ -256,13 +282,56 @@ Error Handling ============== -By default, Genshi allows you to access variables that are not defined, without -raising a ``NameError`` exception as regular Python code would: +By default, Genshi raises an ``UndefinedError`` if a template expression +attempts to access a variable that is not defined: .. code-block:: pycon >>> from genshi.template import MarkupTemplate >>> tmpl = MarkupTemplate('

${doh}

') + >>> tmpl.generate().render('xhtml') + Traceback (most recent call last): + ... + UndefinedError: "doh" not defined + +You can change this behavior by setting the variable lookup mode to "lenient". +In that case, accessing undefined variables returns an `Undefined` object, +meaning that the expression does not fail immediately. See below for details. + +If you need to check whether a variable exists in the template context, use the +defined_ or the value_of_ function described below. To check for existence of +attributes on an object, or keys in a dictionary, use the ``hasattr()``, +``getattr()`` or ``get()`` functions, or the ``in`` operator, just as you would +in regular Python code: + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('

${defined("doh")}

') + >>> print tmpl.generate().render('xhtml') +

False

+ +.. note:: Lenient error handling was the default in Genshi prior to version 0.5. + Strict mode was introduced in version 0.4, and became the default in + 0.5. The reason for this change was that the lenient error handling + was masking actual errors in templates, thereby also making it harder + to debug some problems. + + +.. _`lenient`: + +Lenient Mode +------------ + +If you instruct Genshi to use the lenient variable lookup mode, it allows you +to access variables that are not defined, without raising an ``UndefinedError``. + +This mode can be chosen by passing the ``lookup='lenient'`` keyword argument to +the template initializer, or by passing the ``variable_lookup='lenient'`` +keyword argument to the ``TemplateLoader`` initializer: + +.. code-block:: pycon + + >>> from genshi.template import MarkupTemplate + >>> tmpl = MarkupTemplate('

${doh}

', lookup='lenient') >>> print tmpl.generate().render('xhtml')

@@ -272,7 +341,7 @@ .. code-block:: pycon >>> from genshi.template import MarkupTemplate - >>> tmpl = MarkupTemplate('

${doh.oops}

') + >>> tmpl = MarkupTemplate('

${doh.oops}

', lookup='lenient') >>> print tmpl.generate().render('xhtml') Traceback (most recent call last): ... @@ -284,42 +353,14 @@ .. code-block:: pycon >>> from genshi.template import MarkupTemplate - >>> tmpl = MarkupTemplate('

${type(doh) is not Undefined}

') + >>> tmpl = MarkupTemplate('

${type(doh) is not Undefined}

', + ... lookup='lenient') >>> print tmpl.generate().render('xhtml')

False

Alternatively, the built-in functions defined_ or value_of_ can be used in this case. -Strict Mode ------------ - -In addition to the default "lenient" error handling, Genshi lets you use a less -forgiving mode if you prefer errors blowing up loudly instead of being ignored -silently. - -This mode can be chosen by passing the ``lookup='strict'`` keyword argument to -the template initializer, or by passing the ``variable_lookup='strict'`` keyword -argument to the ``TemplateLoader`` initializer: - -.. code-block:: pycon - - >>> from genshi.template import MarkupTemplate - >>> tmpl = MarkupTemplate('

${doh}

', lookup='strict') - >>> print tmpl.generate().render('xhtml') - Traceback (most recent call last): - ... - UndefinedError: "doh" not defined - -When using strict mode, any reference to an undefined variable, as well as -trying to access an non-existing item or attribute of an object, will cause an -``UndefinedError`` to be raised immediately. - -.. note:: While this mode is currently not the default, it may be promoted to - the default in future versions of Genshi. In general, the default - lenient error handling mode can be considered dangerous as it silently - ignores typos. - Custom Modes ------------ diff --git a/doc/text-templates.txt b/doc/text-templates.txt --- a/doc/text-templates.txt +++ b/doc/text-templates.txt @@ -6,10 +6,7 @@ In addition to the XML-based template language, Genshi provides a simple text-based template language, intended for basic plain text generation needs. -The language is similar to Cheetah_ or Velocity_. - -.. _cheetah: http://cheetahtemplate.org/ -.. _velocity: http://jakarta.apache.org/velocity/ +The language is similar to the Django_ template language. This document describes the template language and will be most useful as reference to those developing Genshi text templates. Templates are text files of @@ -20,6 +17,13 @@ See `Genshi Templating Basics `_ for general information on embedding Python code in templates. +.. note:: Actually, Genshi currently has two different syntaxes for text + templates languages: One implemented by the class ``OldTextTemplate`` + and another implemented by ``NewTextTemplate``. This documentation + concentrates on the latter, which is planned to completely replace the + older syntax. The older syntax is briefly described under legacy_. + +.. _django: http://www.djangoproject.com/ .. contents:: Contents :depth: 3 @@ -32,38 +36,35 @@ Template Directives ------------------- -Directives are lines starting with a ``#`` character followed immediately by -the directive name. They can affect how the template is rendered in a number of -ways: Genshi provides directives for conditionals and looping, among others. +Directives are template commands enclosed by ``{% ... %}`` characters. They can +affect how the template is rendered in a number of ways: Genshi provides +directives for conditionals and looping, among others. -Directives must be on separate lines, and the ``#`` character must be be the -first non-whitespace character on that line. Each directive must be “closed” -using a ``#end`` marker. You can add after the ``#end`` marker, for example to -document which directive is being closed, or even the expression associated with -that directive. Any text after ``#end`` (but on the same line) is ignored, -and effectively treated as a comment. +Each directive must be terminated using an ``{% end %}`` marker. You can add +a string inside the ``{% end %}`` marker, for example to document which +directive is being closed, or even the expression associated with that +directive. Any text after ``end`` inside the delimiters is ignored, and +effectively treated as a comment. -If you want to include a literal ``#`` in the output, you need to escape it -by prepending a backslash character (``\``). Note that this is **not** required -if the ``#`` isn't immediately followed by a letter, or it isn't the first -non-whitespace character on the line. +If you want to include a literal delimiter in the output, you need to escape it +by prepending a backslash character (``\``). Conditional Sections ==================== -.. _`#if`: +.. _`if`: -``#if`` ---------- +``{% if %}`` +------------ The content is only rendered if the expression evaluates to a truth value: .. code-block:: genshitext - #if foo + {% if foo %} ${bar} - #end + {% end %} Given the data ``foo=True`` and ``bar='Hello'`` in the template context, this would produce:: @@ -71,58 +72,46 @@ Hello -.. _`#choose`: -.. _`#when`: -.. _`#otherwise`: - -``#choose`` -------------- +.. _`choose`: +.. _`when`: +.. _`otherwise`: -The ``#choose`` directive, in combination with the directives ``#when`` and -``#otherwise`` provides advanced contional processing for rendering one of -several alternatives. The first matching ``#when`` branch is rendered, or, if -no ``#when`` branch matches, the ``#otherwise`` branch is be rendered. +``{% choose %}`` +---------------- -If the ``#choose`` directive has no argument the nested ``#when`` directives -will be tested for truth: +The ``choose`` directive, in combination with the directives ``when`` and +``otherwise``, provides advanced contional processing for rendering one of +several alternatives. The first matching ``when`` branch is rendered, or, if +no ``when`` branch matches, the ``otherwise`` branch is be rendered. + +If the ``choose`` directive has no argument the nested ``when`` directives will +be tested for truth: .. code-block:: genshitext The answer is: - #choose - #when 0 == 1 - 0 - #end - #when 1 == 1 - 1 - #end - #otherwise - 2 - #end - #end + {% choose %} + {% when 0 == 1 %}0{% end %} + {% when 1 == 1 %}1{% end %} + {% otherwise %}2{% end %} + {% end %} This would produce the following output:: The answer is: - 1 + 1 -If the ``#choose`` does have an argument, the nested ``#when`` directives will -be tested for equality to the parent ``#choose`` value: +If the ``choose`` does have an argument, the nested ``when`` directives will +be tested for equality to the parent ``choose`` value: .. code-block:: genshitext The answer is: - #choose 1 - #when 0 - 0 - #end - #when 1 - 1 - #end - #otherwise - 2 - #end - #end + {% choose 1 %}\ + {% when 0 %}0{% end %}\ + {% when 1 %}1{% end %}\ + {% otherwise %}2{% end %}\ + {% end %} This would produce the following output:: @@ -133,19 +122,19 @@ Looping ======= -.. _`#for`: +.. _`for`: -``#for`` ----------- +``{% for %}`` +------------- The content is repeated for every item in an iterable: .. code-block:: genshitext Your items: - #for item in items + {% for item in items %}\ * ${item} - #end + {% end %} Given ``items=[1, 2, 3]`` in the context data, this would produce:: @@ -158,21 +147,21 @@ Snippet Reuse ============= -.. _`#def`: +.. _`def`: .. _`macros`: -``#def`` ----------- +``{% def %}`` +------------- -The ``#def`` directive can be used to create macros, i.e. snippets of template +The ``def`` directive can be used to create macros, i.e. snippets of template text that have a name and optionally some parameters, and that can be inserted in other places: .. code-block:: genshitext - #def greeting(name) + {% def greeting(name) %} Hello, ${name}! - #end + {% end %} ${greeting('world')} ${greeting('everyone else')} @@ -181,15 +170,15 @@ Hello, world! Hello, everyone else! -If a macro doesn't require parameters, it can be defined as well as called -without the parenthesis. For example: +If a macro doesn't require parameters, it can be defined without the +parenthesis. For example: .. code-block:: genshitext - #def greeting + {% def greeting %} Hello, world! - #end - ${greeting} + {% end %} + ${greeting()} The above would be rendered to:: @@ -197,17 +186,17 @@ .. _includes: -.. _`#include`: +.. _`include`: -``#include`` ------------- +``{% include %}`` +----------------- To reuse common parts of template text across template files, you can include -other files using the ``#include`` directive: +other files using the ``include`` directive: .. code-block:: genshitext - #include "base.txt" + {% include base.txt %} Any content included this way is inserted into the generated output. The included template sees the context data as it exists at the point of the @@ -220,13 +209,13 @@ the included file at "``myapp/base.txt``". You can also use Unix-style relative paths, for example "``../base.txt``" to look in the parent directory. -Just like other directives, the argument to the ``#include`` directive accepts +Just like other directives, the argument to the ``include`` directive accepts any Python expression, so the path to the included template can be determined dynamically: .. code-block:: genshitext - #include '%s.txt' % filename + {% include ${'%s.txt' % filename} %} Note that a ``TemplateNotFound`` exception is raised if an included file can't be found. @@ -237,12 +226,12 @@ Variable Binding ================ -.. _`#with`: +.. _`with`: -``#with`` ------------ +``{% with %}`` +-------------- -The ``#with`` directive lets you assign expressions to variables, which can +The ``{% with %}`` directive lets you assign expressions to variables, which can be used to make expressions inside the directive less verbose and more efficient. For example, if you need use the expression ``author.posts`` more than once, and that actually results in a database query, assigning the results @@ -253,9 +242,9 @@ .. code-block:: genshitext Magic numbers! - #with y=7; z=x+10 + {% with y=7; z=x+10 %} $x $y $z - #end + {% end %} Given ``x=42`` in the context data, this would produce:: @@ -263,17 +252,109 @@ 42 7 52 Note that if a variable of the same name already existed outside of the scope -of the ``#with`` directive, it will **not** be overwritten. Instead, it will -have the same value it had prior to the ``#with`` assignment. Effectively, +of the ``with`` directive, it will **not** be overwritten. Instead, it will +have the same value it had prior to the ``with`` assignment. Effectively, this means that variables are immutable in Genshi. +.. _whitespace: + +--------------------------- +White-space and Line Breaks +--------------------------- + +Note that space or line breaks around directives is never automatically removed. +Consider the following example: + +.. code-block:: genshitext + + {% for item in items %} + {% if item.visible %} + ${item} + {% end %} + {% end %} + +This will result in two empty lines above and beneath every item, plus the +spaces used for indentation. If you want to supress a line break, simply end +the line with a backslash: + +.. code-block:: genshitext + + {% for item in items %}\ + {% if item.visible %}\ + ${item} + {% end %}\ + {% end %}\ + +Now there would be no empty lines between the items in the output. But you still +get the spaces used for indentation, and because the line breaks are removed, +they actually continue and add up between lines. There are numerous ways to +control white-space in the output while keeping the template readable, such as +moving the indentation into the delimiters, or moving the end delimiter on the +next line, and so on. + + .. _comments: -------- Comments -------- -Lines where the first non-whitespace characters are ``##`` are removed from -the output, and can thus be used for comments. This can be escaped using a +Parts in templates can be commented out using the delimiters ``{# ... #}``. +Any content in comments are removed from the output. + +.. code-block:: genshitext + + {# This won't end up in the output #} + This will. + +Just like directive delimiters, these can be escaped by prefixing with a backslash. + +.. code-block:: genshitext + + \{# This *will* end up in the output, including delimiters #} + This too. + + +.. _legacy: + +--------------------------- +Legacy Text Template Syntax +--------------------------- + +The syntax for text templates was redesigned in version 0.5 of Genshi to make +the language more flexible and powerful. The older syntax is based on line +starting with dollar signs, similar to e.g. Cheetah_ or Velocity_. + +.. _cheetah: http://cheetahtemplate.org/ +.. _velocity: http://jakarta.apache.org/velocity/ + +A simple template using the old syntax looked like this: + +.. code-block:: genshitext + + Dear $name, + + We have the following items for you: + #for item in items + * $item + #end + + All the best, + Foobar + +Beyond the requirement of putting directives on separate lines prefixed with +dollar signs, the language itself is very similar to the new one. Except that +comments are lines that start with two ``#`` characters, and a line-break at the +end of a directive is removed automatically. + +.. note:: If you're using this old syntax, it is strongly recommended to + migrate to the new syntax. Simply replace any references to + ``TextTemplate`` by ``NewTextTemplate`` (and also change the + text templates, of course). On the other hand, if you want to stick + with the old syntax for a while longer, replace references to + ``TextTemplate`` by ``OldTextTemplate``; while ``TextTemplate`` is + still an alias for the old language at this point, that will change + in a future release. But also note that the old syntax may be + dropped entirely in a future release. diff --git a/doc/upgrade.txt b/doc/upgrade.txt new file mode 100644 --- /dev/null +++ b/doc/upgrade.txt @@ -0,0 +1,133 @@ +================ +Upgrading Genshi +================ + + +.. contents:: Contents + :depth: 2 +.. sectnum:: + + +------------------------------------ +Upgrading from Genshi 0.4.x to 0.5.x +------------------------------------ + +Error Handling +-------------- + +The default error handling mode has been changed to "strict". This +means that accessing variables not defined in the template data will +now generate an immediate exception, as will accessing object +attributes or dictionary keys that don't exist. If your templates rely +on the old lenient behavior, you can configure Genshi to use that +instead. See the documentation for details on how to do that. But be +warned that lenient error handling may be removed completely in a +future release. + +Match Template Processing +------------------------- + +There has also been a subtle change to how ``py:match`` templates are +processed: in previous versions, all match templates would be applied +to the content generated by the matching template, and only the +matching template itself was applied recursively to the original +content. This behavior resulted in problems with many kinds of +recursive matching, and hence was changed for 0.5: now, all match +templates declared before the matching template are applied to the +original content, and match templates declared after the matching +template are applied to the generated content. This change should not +have any effect on most applications, but you may want to check your +use of match templates to make sure. + +Text Templates +-------------- + +Genshi 0.5 introduces a new, alternative syntax for text templates, +which is more flexible and powerful compared to the old syntax. For +backwards compatibility, this new syntax is not used by default, +though it will be in a future version. It is recommended that you +migrate to using this new syntax. To do so, simply rename any +references in your code to ``TextTemplate`` to ``NewTextTemplate``. To +explicitly use the old syntax, use ``OldTextTemplate`` instead, so +that you can be sure you'll be using the same language when the +default in Genshi is changed (at least until the old implementation is +completely removed). + +``Markup`` Constructor +---------------------- + +The ``Markup`` class now longer has a specialized constructor. The old +(undocumented) constructor provided a shorthand for doing positional +substitutions. If you have code like this: + +.. code-block:: python + + Markup('%s', name) + +You can simply replace it by the more explicit: + +.. code-block:: python + + Markup('%s') % name + +``Template`` Constructor +------------------------ + +The constructor of the ``Template`` class and its subclasses has changed +slightly: instead of the optional ``basedir`` parameter, it now expects +an (also optional) ``filepath`` parameter, which specifies the absolute +path to the template. You probably aren't using those constructors +directly, anyway, but using the ``TemplateLoader`` API instead. + + +------------------------------------ +Upgrading from Genshi 0.3.x to 0.4.x +------------------------------------ + +The modules ``genshi.filters`` and ``genshi.template`` have been +refactored into packages containing multiple modules. While code using +the regular APIs should continue to work without problems, you should +make sure to remove any leftover traces of the files ``filters.py`` +and ``template.py`` in the ``genshi`` package on the installation +path (including the corresponding ``.pyc`` files). This is not +necessary when Genshi was installed as a Python egg. + +Results of evaluating template expressions are no longer implicitly +called if they are callable. If you have been using that feature, you +will need to add the parenthesis to actually call the function. + +Instances of ``genshi.core.Attrs`` are now immutable. Filters +manipulating the attributes in a stream may need to be updated. Also, +the ``Attrs`` class no longer automatically wraps all attribute names +in ``QName`` objects, so users of the ``Attrs`` class need to do this +themselves. See the documentation of the ``Attrs`` class for more +information. + + +--------------------- +Upgrading from Markup +--------------------- + +Prior to version 0.3, the name of the Genshi project was "Markup". The +name change means that you will have to adjust your import statements +and the namespace URI of XML templates, among other things: + +* The package name was changed from "markup" to "genshi". Please + adjust any import statements referring to the old package name. +* The namespace URI for directives in Genshi XML templates has changed + from ``http://markup.edgewall.org/`` to + ``http://genshi.edgewall.org/``. Please update the ``xmlns:py`` + declaration in your template files accordingly. + +Furthermore, due to the inclusion of a text-based template language, +the class:: + + markup.template.Template + +has been renamed to:: + + genshi.template.MarkupTemplate + +If you've been using the Template class directly, you'll need to +update your code (a simple find/replace should do—the API itself +did not change). diff --git a/doc/xml-templates.txt b/doc/xml-templates.txt --- a/doc/xml-templates.txt +++ b/doc/xml-templates.txt @@ -112,7 +112,7 @@ Given the data ``foo=True`` and ``bar='Hello'`` in the template context, this would produce: -.. code-block:: html +.. code-block:: xml
Hello @@ -153,7 +153,7 @@ This would produce the following output: -.. code-block:: html +.. code-block:: xml
1 @@ -172,7 +172,7 @@ This would produce the following output: -.. code-block:: html +.. code-block:: xml
1 @@ -197,7 +197,7 @@ Given ``items=[1, 2, 3]`` in the context data, this would produce: -.. code-block:: html +.. code-block:: xml
  • 1
  • 2
  • 3
    • @@ -239,7 +239,7 @@ The above would be rendered to: -.. code-block:: html +.. code-block:: xml

    @@ -264,7 +264,7 @@ The above would be rendered to: -.. code-block:: html +.. code-block:: xml

    @@ -307,7 +307,7 @@ This would result in the following output: -.. code-block:: html +.. code-block:: xml

    @@ -322,6 +322,12 @@ .. _`Using XPath`: streams.html#using-xpath +Match templates are applied both to the original markup as well to the +generated markup. The order in which they are applied depends on the order +they are declared in the template source: a match template defined after +another match template is applied to the output generated by the first match +template. The match templates basically form a pipeline. + This directive can also be used as an element: .. code-block:: genshi @@ -333,6 +339,54 @@
    +When used this way, the ``py:match`` directive can also be annotated with a +couple of optimization hints. For example, the following informs the matching +engine that the match should only be applied once: + +.. code-block:: genshi + + + + + ${select("*|text()")} + + + + +The following optimization hints are recognized: + ++---------------+-----------+-----------------------------------------------+ +| Attribute | Default | Description | ++===============+===========+===============================================+ +| ``buffer`` | ``true`` | Whether the matched content should be | +| | | buffered in memory. Buffering can improve | +| | | performance a bit at the cost of needing more | +| | | memory during rendering. Buffering is | +| | | ''required'' for match templates that contain | +| | | more than one invocation of the ``select()`` | +| | | function. If there is only one call, and the | +| | | matched content can potentially be very long, | +| | | consider disabling buffering to avoid | +| | | excessive memory use. | ++---------------+-----------+-----------------------------------------------+ +| ``once`` | ``false`` | Whether the engine should stop looking for | +| | | more matching elements after the first match. | +| | | Use this on match templates that match | +| | | elements that can only occur once in the | +| | | stream, such as the ```` or ```` | +| | | elements in an HTML template, or elements | +| | | with a specific ID. | ++---------------+-----------+-----------------------------------------------+ +| ``recursive`` | ``true`` | Whether the match template should be applied | +| | | to its own output. Note that ``once`` implies | +| | | non-recursive behavior, so this attribute | +| | | only needs to be set for match templates that | +| | | don't also have ``once`` set. | ++---------------+-----------+-----------------------------------------------+ + +.. note:: The ``py:match`` optimization hints were added in the 0.5 release. In + earlier versions, the attributes have no effect. + Variable Binding ================ @@ -358,7 +412,7 @@ Given ``x=42`` in the context data, this would produce: -.. code-block:: html +.. code-block:: xml
    42 7 52 @@ -397,7 +451,7 @@ Given ``foo={'class': 'collapse'}`` in the template context, this would produce: -.. code-block:: html +.. code-block:: xml
    • Bar
      • @@ -406,7 +460,7 @@ Attributes with the value ``None`` are omitted, so given ``foo={'class': None}`` in the context for the same template this would produce: -.. code-block:: html +.. code-block:: xml
      • Bar
        • @@ -431,7 +485,7 @@ Given ``bar='Bye'`` in the context data, this would produce: -.. code-block:: html +.. code-block:: xml
        • Bye
          • @@ -456,13 +510,20 @@ Given ``bar='Bye'`` in the context data, this would produce: -.. code-block:: html +.. code-block:: xml
          Bye
          -This directive can only be used as an attribute. +This directive can also be used as an element (since version 0.5): + +.. code-block:: genshi + +
          + Placeholder +
          + .. _`py:strip`: @@ -482,7 +543,7 @@ This would be rendered as: -.. code-block:: html +.. code-block:: xml
          foo @@ -564,6 +625,10 @@ .. _`xinclude specification`: http://www.w3.org/TR/xinclude/ + +Dynamic Includes +================ + Incudes in Genshi are fully dynamic: Just like normal attributes, the `href` attribute accepts expressions, and directives_ can be used on the ```` element just as on any other element, meaning you can do @@ -575,6 +640,23 @@ py:for="name in ('foo', 'bar', 'baz')" /> +Including Text Templates +======================== + +The ``parse`` attribute of the ```` element can be used to specify +whether the included template is an XML template or a text template (using the +new syntax added in Genshi 0.5): + +.. code-block:: genshi + + + +This example would load the ``myscript.js`` file as a ``NewTextTemplate``. See +`text templates`_ for details on the syntax of text templates. + +.. _`text templates`: text-templates.html + + .. _comments: -------- diff --git a/examples/bench/basic.py b/examples/bench/basic.py --- a/examples/bench/basic.py +++ b/examples/bench/basic.py @@ -9,7 +9,8 @@ import sys import timeit -__all__ = ['clearsilver', 'mako', 'django', 'kid', 'genshi', 'simpletal'] +__all__ = ['clearsilver', 'mako', 'django', 'kid', 'genshi', 'genshi_text', + 'simpletal'] def genshi(dirname, verbose=False): from genshi.template import TemplateLoader @@ -24,6 +25,20 @@ print render() return render +def genshi_text(dirname, verbose=False): + from genshi.core import escape + from genshi.template import TemplateLoader, NewTextTemplate + loader = TemplateLoader([dirname], auto_reload=False) + template = loader.load('template.txt', cls=NewTextTemplate) + def render(): + data = dict(escape=escape, title='Just a test', user='joe', + items=['Number %d' % num for num in range(1, 15)]) + return template.generate(**data).render('text') + + if verbose: + print render() + return render + def mako(dirname, verbose=False): from mako.lookup import TemplateLookup lookup = TemplateLookup(directories=[dirname], filesystem_checks=False) @@ -107,7 +122,7 @@ try: import kid except ImportError: - print>>sys.stderr, "SimpleTAL not installed, skipping" + print>>sys.stderr, "Kid not installed, skipping" return lambda: None kid.path = kid.TemplatePath([dirname]) template = kid.load_template('template.kid').Template diff --git a/examples/bench/bigtable.py b/examples/bench/bigtable.py --- a/examples/bench/bigtable.py +++ b/examples/bench/bigtable.py @@ -10,7 +10,7 @@ import timeit from StringIO import StringIO from genshi.builder import tag -from genshi.template import MarkupTemplate +from genshi.template import MarkupTemplate, NewTextTemplate try: from elementtree import ElementTree as et @@ -60,6 +60,14 @@ $table
          """) +genshi_text_tmpl = NewTextTemplate(""" + +{% for row in table %} +{% for c in row.values() %}{% end %} +{% end %} +
          $c
          +""") + if DjangoTemplate: django_tmpl = DjangoTemplate(""" @@ -95,6 +103,11 @@ stream = genshi_tmpl.generate(table=table) stream.render('html', strip_whitespace=False) +def test_genshi_text(): + """Genshi text template""" + stream = genshi_text_tmpl.generate(table=table) + stream.render('text') + def test_genshi_builder(): """Genshi template + tag builder""" stream = tag.TABLE([ @@ -183,9 +196,9 @@ def run(which=None, number=10): - tests = ['test_builder', 'test_genshi', 'test_genshi_builder', - 'test_mako', 'test_kid', 'test_kid_et', 'test_et', 'test_cet', - 'test_clearsilver', 'test_django'] + tests = ['test_builder', 'test_genshi', 'test_genshi_text', + 'test_genshi_builder', 'test_mako', 'test_kid', 'test_kid_et', + 'test_et', 'test_cet', 'test_clearsilver', 'test_django'] if which: tests = filter(lambda n: n[5:] in which, tests) diff --git a/examples/bench/genshi/base.html b/examples/bench/genshi/base.html --- a/examples/bench/genshi/base.html +++ b/examples/bench/genshi/base.html @@ -6,12 +6,12 @@ Hello, ${name}!

          - + ${select('*')}
          + + + + + + + + +
          + + ${errors.username} +
          + +
          ${errors.content}           +

          You can use HTML tags here for formatting.

          +
          + + +
          + diff --git a/examples/tutorial/geddit/templates/comment.html b/examples/tutorial/geddit/templates/comment.html new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/comment.html @@ -0,0 +1,18 @@ + + + + + Comment on “${link.title}” + + +

          Comment on “${link.title}”

          +

          + In reply to ${comment.username} + at ${comment.time.strftime('%x %X')}: +

          ${comment.content}
          +

          + + + diff --git a/examples/tutorial/geddit/templates/index.html b/examples/tutorial/geddit/templates/index.html new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/index.html @@ -0,0 +1,28 @@ + + + + + News + + + +

          News

          + +
            +
          1. + ${link.title} + posted by ${link.username} at ${link.time.strftime('%x %X')} +
            + + ${len(link.comments)} comments + +
            +
            2. +
          + +

          Submit new link

          + + diff --git a/examples/tutorial/geddit/templates/index.xml b/examples/tutorial/geddit/templates/index.xml new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/index.xml @@ -0,0 +1,23 @@ + + + + Geddit News + + + + ${links[0].time.isoformat()} + + + ${link.url} + + + ${url('/info/%s/' % link.id)} + + ${link.username} + + ${link.time.isoformat()} + ${link.title} + + + diff --git a/examples/tutorial/geddit/templates/info.html b/examples/tutorial/geddit/templates/info.html new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/info.html @@ -0,0 +1,62 @@ + + + + + ${link.title} + + + + +

          ${link.title}

          + ${link.url}
          + posted by ${link.username} at ${link.time.strftime('%x %X')}
          + +
            + +
          + +

          Add comment

          + + diff --git a/examples/tutorial/geddit/templates/info.xml b/examples/tutorial/geddit/templates/info.xml new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/info.xml @@ -0,0 +1,28 @@ + + + + Geddit: ${link.title} + + + + + ${time.isoformat()} + + + + + Comment ${len(link.comments) - idx} on “${link.title}” + + ${url('/info/%s/' % link.id)}#comment${idx} + + ${comment.username} + + ${comment.time.isoformat()} +
          + ${HTML(comment.content)} +
          +           + +           diff --git a/examples/tutorial/geddit/templates/layout.html b/examples/tutorial/geddit/templates/layout.html new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/layout.html @@ -0,0 +1,31 @@ + + + + + + + Geddit<py:if test="title">: ${title}</py:if> + + + + ${select('*[local-name()!="title"]')} + + + + +
          + +
          + ${select('*|text()')} +
          + +
          +           + + diff --git a/examples/tutorial/geddit/templates/submit.html b/examples/tutorial/geddit/templates/submit.html new file mode 100644 --- /dev/null +++ b/examples/tutorial/geddit/templates/submit.html @@ -0,0 +1,41 @@ + + + + + Submit new link + + +

          Submit new link

          + +
          + + + + + + + + + + + + +
          + + ${errors.username} +
          + + ${errors.url} +
          + + ${errors.title} +
          + + +
          +
          + + + diff --git a/genshi/__init__.py b/genshi/__init__.py --- a/genshi/__init__.py +++ b/genshi/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2007 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which diff --git a/genshi/_speedups.c b/genshi/_speedups.c --- a/genshi/_speedups.c +++ b/genshi/_speedups.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2006 Edgewall Software + * Copyright (C) 2006-2008 Edgewall Software * All rights reserved. * * This software is licensed as described in the file COPYING, which @@ -87,6 +87,7 @@ out = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, len); if (out == NULL) { + Py_DECREF((PyObject *) in); return NULL; } @@ -130,6 +131,8 @@ inp++; } + Py_DECREF((PyObject *) in); + args = PyTuple_New(1); if (args == NULL) { Py_DECREF((PyObject *) out); @@ -141,46 +144,6 @@ return ret; } -static PyObject * -Markup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyObject *self, *text, *tmp, *args2; - int nargs, i; - - nargs = PyTuple_GET_SIZE(args); - if (nargs < 2) { - return PyUnicode_Type.tp_new(type, args, NULL); - } - - text = PyTuple_GET_ITEM(args, 0); - args2 = PyTuple_New(nargs - 1); - if (args2 == NULL) { - return NULL; - } - for (i = 1; i < nargs; i++) { - tmp = escape(PyTuple_GET_ITEM(args, i), 1); - if (tmp == NULL) { - Py_DECREF(args2); - return NULL; - } - PyTuple_SET_ITEM(args2, i - 1, tmp); - } - tmp = PyUnicode_Format(text, args2); - Py_DECREF(args2); - if (tmp == NULL) { - return NULL; - } - args = PyTuple_New(1); - if (args == NULL) { - Py_DECREF(tmp); - return NULL; - } - PyTuple_SET_ITEM(args, 0, tmp); - self = PyUnicode_Type.tp_new(type, args, NULL); - Py_DECREF(args); - return self; -} - PyDoc_STRVAR(escape__doc__, "Create a Markup instance from a string and escape special characters\n\ it may contain (<, >, & and \").\n\ @@ -242,7 +205,7 @@ Markup_join(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"seq", "escape_quotes", 0}; - PyObject *seq = NULL, *seq2, *tmp; + PyObject *seq = NULL, *seq2, *tmp, *tmp2; char quotes = 1; int n, i; @@ -266,12 +229,13 @@ Py_DECREF(seq2); return NULL; } - tmp = escape(tmp, quotes); - if (tmp == NULL) { + tmp2 = escape(tmp, quotes); + if (tmp2 == NULL) { Py_DECREF(seq2); return NULL; } - PyTuple_SET_ITEM(seq2, i, tmp); + PyTuple_SET_ITEM(seq2, i, tmp2); + Py_DECREF(tmp); } tmp = PyUnicode_Join(self, seq2); Py_DECREF(seq2); @@ -303,11 +267,9 @@ return NULL; tmp2 = PyUnicode_Concat(tmp, other); } - if (tmp2 == NULL) { - Py_DECREF(tmp); + Py_DECREF(tmp); + if (tmp2 == NULL) return NULL; - } - Py_DECREF(tmp); args = PyTuple_New(1); if (args == NULL) { Py_DECREF(tmp2); @@ -323,9 +285,38 @@ Markup_mod(PyObject *self, PyObject *args) { PyObject *tmp, *tmp2, *ret, *args2; - int i, nargs; + int i, nargs = 0; + PyObject *kwds = NULL; - if (PyTuple_Check(args)) { + if (PyDict_Check(args)) { + kwds = args; + } + if (kwds && PyDict_Size(kwds)) { + PyObject *kwcopy, *key, *value; + Py_ssize_t pos = 0; + + kwcopy = PyDict_Copy( kwds ); + if (kwcopy == NULL) { + return NULL; + } + while (PyDict_Next(kwcopy, &pos, &key, &value)) { + tmp = escape(value, 1); + if (tmp == NULL) { + Py_DECREF(kwcopy); + return NULL; + } + if (PyDict_SetItem(kwcopy, key, tmp) < 0) { + Py_DECREF(tmp); + Py_DECREF(kwcopy); + return NULL; + } + } + tmp = PyUnicode_Format(self, kwcopy); + Py_DECREF(kwcopy); + if (tmp == NULL) { + return NULL; + } + } else if (PyTuple_Check(args)) { nargs = PyTuple_GET_SIZE(args); args2 = PyTuple_New(nargs); if (args2 == NULL) { @@ -380,6 +371,7 @@ if (unicode == NULL) return NULL; result = PyNumber_Multiply(unicode, self); } + Py_DECREF(unicode); if (result == NULL) return NULL; args = PyTuple_New(1); @@ -402,14 +394,19 @@ format = PyString_FromString(""); if (format == NULL) return NULL; result = PyObject_Unicode(self); - if (result == NULL) return NULL; + if (result == NULL) { + Py_DECREF(format); + return NULL; + } args = PyTuple_New(1); if (args == NULL) { + Py_DECREF(format); Py_DECREF(result); return NULL; } PyTuple_SET_ITEM(args, 0, result); result = PyString_Format(format, args); + Py_DECREF(format); Py_DECREF(args); return result; } @@ -581,7 +578,7 @@ 0, /*tp_init*/ 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ - Markup_new, /*tp_new*/ + 0, /*tp_new*/ 0, /*tp_free Low-level free-memory routine */ 0, /*tp_is_gc For PyObject_IS_GC */ 0, /*tp_bases*/ diff --git a/genshi/core.py b/genshi/core.py --- a/genshi/core.py +++ b/genshi/core.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -13,6 +13,7 @@ """Core classes for markup processing.""" +from itertools import chain import operator from genshi.util import plaintext, stripentities, striptags @@ -51,7 +52,7 @@ returns the complete generated text at once. Both accept various parameters that impact the way the stream is serialized. """ - __slots__ = ['events'] + __slots__ = ['events', 'serializer'] START = StreamEventKind('START') #: a start tag END = StreamEventKind('END') #: an end tag @@ -65,12 +66,17 @@ PI = StreamEventKind('PI') #: processing instruction COMMENT = StreamEventKind('COMMENT') #: comment - def __init__(self, events): + def __init__(self, events, serializer=None): """Initialize the stream with a sequence of markup events. :param events: a sequence or iterable providing the events + :param serializer: the default serialization method to use for this + stream + + :note: Changed in 0.5: added the `serializer` argument """ self.events = events #: The underlying iterable producing the events + self.serializer = serializer #: The default serializion method def __iter__(self): return iter(self.events) @@ -119,7 +125,7 @@ :return: the filtered stream :rtype: `Stream` """ - return Stream(_ensure(function(self))) + return Stream(_ensure(function(self)), serializer=self.serializer) def filter(self, *filters): """Apply filters to the stream. @@ -143,7 +149,7 @@ """ return reduce(operator.or_, (self,) + filters) - def render(self, method='xml', encoding='utf-8', **kwargs): + def render(self, method=None, encoding='utf-8', out=None, **kwargs): """Return a string representation of the stream. Any additional keyword arguments are passed to the serializer, and thus @@ -151,21 +157,52 @@ :param method: determines how the stream is serialized; can be either "xml", "xhtml", "html", "text", or a custom serializer - class + class; if `None`, the default serialization method of + the stream is used :param encoding: how the output string should be encoded; if set to `None`, this method returns a `unicode` object - :return: a `str` or `unicode` object + :param out: a file-like object that the output should be written to + instead of being returned as one big string; note that if + this is a file or socket (or similar), the `encoding` must + not be `None` (that is, the output must be encoded) + :return: a `str` or `unicode` object (depending on the `encoding` + parameter), or `None` if the `out` parameter is provided :rtype: `basestring` + :see: XMLSerializer, XHTMLSerializer, HTMLSerializer, TextSerializer + :note: Changed in 0.5: added the `out` parameter """ from genshi.output import encode + if method is None: + method = self.serializer or 'xml' generator = self.serialize(method=method, **kwargs) - return encode(generator, method=method, encoding=encoding) + return encode(generator, method=method, encoding=encoding, out=out) def select(self, path, namespaces=None, variables=None): """Return a new stream that contains the events matching the given XPath expression. + >>> from genshi import HTML + >>> stream = HTML('foobar') + >>> print stream.select('elem') + foobar + >>> print stream.select('elem/text()') + foobar + + Note that the outermost element of the stream becomes the *context + node* for the XPath test. That means that the expression "doc" would + not match anything in the example above, because it only tests against + child elements of the outermost element: + + >>> print stream.select('doc') + + + You can use the "." expression to match the context node itself + (although that usually makes little sense): + + >>> print stream.select('.') + foobar + :param path: a string containing the XPath expression :param namespaces: mapping of namespace prefixes used in the path :param variables: mapping of variable names to values @@ -190,13 +227,16 @@ :param method: determines how the stream is serialized; can be either "xml", "xhtml", "html", "text", or a custom serializer - class + class; if `None`, the default serialization method of + the stream is used :return: an iterator over the serialization results (`Markup` or `unicode` objects, depending on the serialization method) :rtype: ``iterator`` :see: XMLSerializer, XHTMLSerializer, HTMLSerializer, TextSerializer """ from genshi.output import get_serializer + if method is None: + method = self.serializer or 'xml' return get_serializer(method, **kwargs)(_ensure(self)) def __str__(self): @@ -220,12 +260,24 @@ def _ensure(stream): """Ensure that every item on the stream is actually a markup event.""" - for event in stream: - if type(event) is not tuple: + stream = iter(stream) + event = stream.next() + + # Check whether the iterable is a real markup event stream by examining the + # first item it yields; if it's not we'll need to do some conversion + if type(event) is not tuple or len(event) != 3: + for event in chain([event], stream): if hasattr(event, 'totuple'): event = event.totuple() else: event = TEXT, unicode(event), (None, -1, -1) + yield event + return + + # This looks like a markup event stream, so we'll just pass it through + # unchanged + yield event + for event in stream: yield event @@ -295,6 +347,12 @@ return True def __getslice__(self, i, j): + """Return a slice of the attributes list. + + >>> attrs = Attrs([('href', '#'), ('title', 'Foo')]) + >>> attrs[1:] + Attrs([('title', 'Foo')]) + """ return Attrs(tuple.__getslice__(self, i, j)) def __or__(self, attrs): @@ -361,11 +419,6 @@ """ __slots__ = [] - def __new__(cls, text='', *args): - if args: - text %= tuple(map(escape, args)) - return unicode.__new__(cls, text) - def __add__(self, other): return Markup(unicode(self) + unicode(escape(other))) @@ -373,9 +426,13 @@ return Markup(unicode(escape(other)) + unicode(self)) def __mod__(self, args): - if not isinstance(args, (list, tuple)): - args = [args] - return Markup(unicode.__mod__(self, tuple(map(escape, args)))) + if isinstance(args, dict): + args = dict(zip(args.keys(), map(escape, args.values()))) + elif isinstance(args, (list, tuple)): + args = tuple(map(escape, args)) + else: + args = escape(args) + return Markup(unicode.__mod__(self, args)) def __mul__(self, num): return Markup(unicode(self) * num) @@ -548,7 +605,7 @@ def __new__(cls, uri): if type(uri) is cls: return uri - return object.__new__(cls, uri) + return object.__new__(cls) def __getnewargs__(self): return (self.uri,) @@ -595,9 +652,9 @@ """A qualified element or attribute name. The unicode value of instances of this class contains the qualified name of - the element or attribute, in the form ``{namespace}localname``. The namespace - URI can be obtained through the additional `namespace` attribute, while the - local name can be accessed through the `localname` attribute. + the element or attribute, in the form ``{namespace-uri}local-name``. The + namespace URI can be obtained through the additional `namespace` attribute, + while the local name can be accessed through the `localname` attribute. >>> qname = QName('foo') >>> qname @@ -617,10 +674,16 @@ __slots__ = ['namespace', 'localname'] def __new__(cls, qname): + """Create the `QName` instance. + + :param qname: the qualified name as a string of the form + ``{namespace-uri}local-name``, where the leading curly + brace is optional + """ if type(qname) is cls: return qname - parts = qname.split(u'}', 1) + parts = qname.lstrip(u'{').split(u'}', 1) if len(parts) > 1: self = unicode.__new__(cls, u'{%s' % qname) self.namespace, self.localname = map(unicode, parts) diff --git a/genshi/filters/html.py b/genshi/filters/html.py --- a/genshi/filters/html.py +++ b/genshi/filters/html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -14,13 +14,14 @@ """Implementation of a number of stream filters.""" try: - frozenset + set except NameError: from sets import ImmutableSet as frozenset + from sets import Set as set import re from genshi.core import Attrs, QName, stripentities -from genshi.core import END, START, TEXT +from genshi.core import END, START, TEXT, COMMENT __all__ = ['HTMLFormFiller', 'HTMLSanitizer'] __docformat__ = 'restructuredtext en' @@ -69,7 +70,9 @@ """ in_form = in_select = in_option = in_textarea = False select_value = option_value = textarea_value = None - option_start = option_text = None + option_start = None + option_text = [] + no_option_value = False for kind, data, pos in stream: @@ -94,13 +97,13 @@ checked = False if isinstance(value, (list, tuple)): if declval: - checked = declval in [str(v) for v + checked = declval in [unicode(v) for v in value] else: checked = bool(filter(None, value)) else: if declval: - checked = declval == str(value) + checked = declval == unicode(value) elif type == 'checkbox': checked = bool(value) if checked: @@ -130,15 +133,18 @@ elif in_select and tagname == 'option': option_start = kind, data, pos option_value = attrs.get('value') + if option_value is None: + no_option_value = True + option_value = '' in_option = True continue yield kind, (tag, attrs), pos elif in_form and kind is TEXT: if in_select and in_option: - if option_value is None: - option_value = data - option_text = kind, data, pos + if no_option_value: + option_value += data + option_text.append((kind, data, pos)) continue elif in_textarea: continue @@ -153,10 +159,10 @@ select_value = None elif in_select and tagname == 'option': if isinstance(select_value, (tuple, list)): - selected = option_value in [str(v) for v + selected = option_value in [unicode(v) for v in select_value] else: - selected = option_value == str(select_value) + selected = option_value == unicode(select_value) okind, (tag, attrs), opos = option_start if selected: attrs |= [(QName('selected'), 'selected')] @@ -164,9 +170,12 @@ attrs -= 'selected' yield okind, (tag, attrs), opos if option_text: - yield option_text + for event in option_text: + yield event in_option = False - option_start = option_text = option_value = None + no_option_value = False + option_start = option_value = None + option_text = [] elif tagname == 'textarea': if textarea_value: yield TEXT, unicode(textarea_value), pos @@ -206,6 +215,9 @@ well as a lot of other things. However, the style tag is still excluded by default because it is very hard for such sanitizing to be completely safe, especially considering how much error recovery current web browsers perform. + + :warn: Note that this special processing of CSS is currently only applied to + style attributes, **not** style elements. """ SAFE_TAGS = frozenset(['a', 'abbr', 'acronym', 'address', 'area', 'b', @@ -247,9 +259,13 @@ :param uri_attrs: a set of names of attributes that contain URIs """ self.safe_tags = safe_tags + "The set of tag names that are considered safe." self.safe_attrs = safe_attrs + "The set of attribute names that are considered safe." self.uri_attrs = uri_attrs + "The set of names of attributes that may contain URIs." self.safe_schemes = safe_schemes + "The set of URI schemes that are considered safe." def __call__(self, stream): """Apply the filter to the given stream. @@ -258,12 +274,6 @@ """ waiting_for = None - def _get_scheme(href): - if ':' not in href: - return None - chars = [char for char in href.split(':', 1)[0] if char.isalnum()] - return ''.join(chars).lower() - for kind, data, pos in stream: if kind is START: if waiting_for: @@ -280,24 +290,11 @@ continue elif attr in self.uri_attrs: # Don't allow URI schemes such as "javascript:" - if _get_scheme(value) not in self.safe_schemes: + if not self.is_safe_uri(value): continue elif attr == 'style': # Remove dangerous CSS declarations from inline styles - decls = [] - value = self._strip_css_comments( - self._replace_unicode_escapes(value) - ) - for decl in filter(None, value.split(';')): - is_evil = False - if 'expression' in decl: - is_evil = True - for m in re.finditer(r'url\s*\(([^)]+)', decl): - if _get_scheme(m.group(1)) not in self.safe_schemes: - is_evil = True - break - if not is_evil: - decls.append(decl.strip()) + decls = self.sanitize_css(value) if not decls: continue value = '; '.join(decls) @@ -313,10 +310,79 @@ else: yield kind, data, pos - else: + elif kind is not COMMENT: if not waiting_for: yield kind, data, pos + def is_safe_uri(self, uri): + """Determine whether the given URI is to be considered safe for + inclusion in the output. + + The default implementation checks whether the scheme of the URI is in + the set of allowed URIs (`safe_schemes`). + + >>> sanitizer = HTMLSanitizer() + >>> sanitizer.is_safe_uri('http://example.org/') + True + >>> sanitizer.is_safe_uri('javascript:alert(document.cookie)') + False + + :param uri: the URI to check + :return: `True` if the URI can be considered safe, `False` otherwise + :rtype: `bool` + :since: version 0.4.3 + """ + if ':' not in uri: + return True # This is a relative URI + chars = [char for char in uri.split(':', 1)[0] if char.isalnum()] + return ''.join(chars).lower() in self.safe_schemes + + def sanitize_css(self, text): + """Remove potentially dangerous property declarations from CSS code. + + In particular, properties using the CSS ``url()`` function with a scheme + that is not considered safe are removed: + + >>> sanitizer = HTMLSanitizer() + >>> sanitizer.sanitize_css(u''' + ... background: url(javascript:alert("foo")); + ... color: #000; + ... ''') + [u'color: #000'] + + Also, the proprietary Internet Explorer function ``expression()`` is + always stripped: + + >>> sanitizer.sanitize_css(u''' + ... background: #fff; + ... color: #000; + ... width: e/**/xpression(alert("foo")); + ... ''') + [u'background: #fff', u'color: #000'] + + :param text: the CSS text; this is expected to be `unicode` and to not + contain any character or numeric references + :return: a list of declarations that are considered safe + :rtype: `list` + :since: version 0.4.3 + """ + decls = [] + text = self._strip_css_comments(self._replace_unicode_escapes(text)) + for decl in filter(None, text.split(';')): + decl = decl.strip() + if not decl: + continue + is_evil = False + if 'expression' in decl: + is_evil = True + for match in re.finditer(r'url\s*\(([^)]+)', decl): + if not self.is_safe_uri(match.group(1)): + is_evil = True + break + if not is_evil: + decls.append(decl.strip()) + return decls + _NORMALIZE_NEWLINES = re.compile(r'\r\n').sub _UNICODE_ESCAPE = re.compile(r'\\([0-9a-fA-F]{1,6})\s?').sub diff --git a/genshi/filters/i18n.py b/genshi/filters/i18n.py --- a/genshi/filters/i18n.py +++ b/genshi/filters/i18n.py @@ -11,7 +11,10 @@ # individuals. For the exact contribution history, see the revision # history and logs, available at http://genshi.edgewall.org/log/. -"""Utilities for internationalization and localization of templates.""" +"""Utilities for internationalization and localization of templates. + +:since: version 0.4 +""" from compiler import ast try: @@ -93,17 +96,21 @@ 'summary', 'title']) def __init__(self, translate=gettext, ignore_tags=IGNORE_TAGS, - include_attrs=INCLUDE_ATTRS): + include_attrs=INCLUDE_ATTRS, extract_text=True): """Initialize the translator. :param translate: the translation function, for example ``gettext`` or ``ugettext``. :param ignore_tags: a set of tag names that should not be localized :param include_attrs: a set of attribute names should be localized + :param extract_text: whether the content of text nodes should be + extracted, or only text in explicit ``gettext`` + function calls """ self.translate = translate self.ignore_tags = ignore_tags self.include_attrs = include_attrs + self.extract_text = extract_text def __call__(self, stream, ctxt=None, search_text=True, msgbuf=None): """Translate any localizable strings in the given stream. @@ -124,6 +131,8 @@ ignore_tags = self.ignore_tags include_attrs = self.include_attrs translate = self.translate + if not self.extract_text: + search_text = False skip = 0 i18n_msg = I18N_NAMESPACE['msg'] ns_prefixes = [] @@ -153,7 +162,7 @@ changed = False for name, value in attrs: newval = value - if isinstance(value, basestring): + if search_text and isinstance(value, basestring): if name in include_attrs: newval = self.translate(value) else: @@ -165,7 +174,7 @@ changed = True new_attrs.append((name, value)) if changed: - attrs = new_attrs + attrs = Attrs(new_attrs) if msgbuf: msgbuf.append(kind, data, pos) @@ -255,6 +264,8 @@ (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. """ + if not self.extract_text: + search_text = False skip = 0 i18n_msg = I18N_NAMESPACE['msg'] xml_lang = XML_NAMESPACE['lang'] @@ -276,7 +287,7 @@ continue for name, value in attrs: - if isinstance(value, basestring): + if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: @@ -321,7 +332,10 @@ class MessageBuffer(object): - """Helper class for managing localizable mixed content.""" + """Helper class for managing localizable mixed content. + + :since: version 0.5 + """ def __init__(self, lineno=-1): self.lineno = lineno @@ -385,17 +399,21 @@ :param code: the `Code` object :type code: `genshi.template.eval.Code` :param gettext_functions: a sequence of function names + :since: version 0.5 """ def _walk(node): if isinstance(node, ast.CallFunc) and isinstance(node.node, ast.Name) \ and node.node.name in gettext_functions: strings = [] - for arg in node.args: + def _add(arg): if isinstance(arg, ast.Const) \ and isinstance(arg.value, basestring): - strings.append(unicode(arg.value)) - elif not isinstance(arg, ast.Keyword): + strings.append(unicode(arg.value, 'utf-8')) + elif arg and not isinstance(arg, ast.Keyword): strings.append(None) + [_add(arg) for arg in node.args] + _add(node.star_args) + _add(node.dstar_args) if len(strings) == 1: strings = strings[0] else: @@ -421,6 +439,8 @@ >>> parse_msg("[1:] Bilder pro Seite anzeigen.") [(1, ''), (0, ' Bilder pro Seite anzeigen.')] + + :since: version 0.5 """ parts = [] stack = [0] @@ -464,10 +484,15 @@ template_class = getattr(__import__(module, {}, {}, [clsname]), clsname) encoding = options.get('encoding', None) + extract_text = options.get('extract_text', True) + if isinstance(extract_text, basestring): + extract_text = extract_text.lower() in ('1', 'on', 'yes', 'true') + ignore_tags = options.get('ignore_tags', Translator.IGNORE_TAGS) if isinstance(ignore_tags, basestring): ignore_tags = ignore_tags.split() ignore_tags = [QName(tag) for tag in ignore_tags] + include_attrs = options.get('include_attrs', Translator.INCLUDE_ATTRS) if isinstance(include_attrs, basestring): include_attrs = include_attrs.split() @@ -475,7 +500,7 @@ tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None), encoding=encoding) - translator = Translator(None, ignore_tags, include_attrs) + translator = Translator(None, ignore_tags, include_attrs, extract_text) for lineno, func, message in translator.extract(tmpl.stream, gettext_functions=keywords): yield lineno, func, message, [] diff --git a/genshi/filters/tests/__init__.py b/genshi/filters/tests/__init__.py --- a/genshi/filters/tests/__init__.py +++ b/genshi/filters/tests/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -19,7 +19,8 @@ suite = unittest.TestSuite() suite.addTest(html.suite()) suite.addTest(i18n.suite()) - suite.addTest(transform.suite()) + if hasattr(doctest, 'NORMALIZE_WHITESPACE'): + suite.addTest(transform.suite()) return suite if __name__ == '__main__': diff --git a/genshi/filters/tests/html.py b/genshi/filters/tests/html.py --- a/genshi/filters/tests/html.py +++ b/genshi/filters/tests/html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -12,11 +12,15 @@ # history and logs, available at http://genshi.edgewall.org/log/. import doctest +try: + set +except NameError: + from sets import Set as set import unittest from genshi.input import HTML, ParseError from genshi.filters.html import HTMLFormFiller, HTMLSanitizer - +from genshi.template import MarkupTemplate class HTMLFormFillerTestCase(unittest.TestCase): @@ -270,6 +274,42 @@

          """, unicode(html)) + def test_fill_option_segmented_text(self): + html = MarkupTemplate("""
          + +
          """).generate(x=1) | HTMLFormFiller(data={'foo': '1'}) + self.assertEquals("""
          + +
          """, unicode(html)) + + def test_fill_option_segmented_text_no_value(self): + html = MarkupTemplate("""
          + +
          """).generate(x=1) | HTMLFormFiller(data={'foo': 'foo 1 bar'}) + self.assertEquals("""
          + +
          """, unicode(html)) + + def test_fill_option_unicode_value(self): + html = HTML(u"""
          + +
          """) | HTMLFormFiller(data={'foo': u'ö'}) + self.assertEquals(u"""
          + +
          """, unicode(html)) + class HTMLSanitizerTestCase(unittest.TestCase): @@ -318,6 +358,10 @@ html = HTML('
          ') self.assertEquals(u'
          ', unicode(html | HTMLSanitizer())) + def test_sanitize_remove_comments(self): + html = HTML('''
          ''') + self.assertEquals(u'
          ', unicode(html | HTMLSanitizer())) + def test_sanitize_remove_style_scripts(self): sanitizer = HTMLSanitizer(safe_attrs=HTMLSanitizer.SAFE_ATTRS | set(['style'])) # Inline style with url() using javascript: scheme diff --git a/genshi/filters/tests/i18n.py b/genshi/filters/tests/i18n.py --- a/genshi/filters/tests/i18n.py +++ b/genshi/filters/tests/i18n.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -15,12 +15,37 @@ from StringIO import StringIO import unittest +from genshi.core import Attrs from genshi.template import MarkupTemplate from genshi.filters.i18n import Translator, extract +from genshi.input import HTML class TranslatorTestCase(unittest.TestCase): + def test_translate_included_attribute_text(self): + """ + Verify that translated attributes end up in a proper `Attrs` instance. + """ + html = HTML(""" + + """) + translator = Translator(lambda s: u"Voh") + stream = list(html.filter(translator)) + kind, data, pos = stream[2] + assert isinstance(data[1], Attrs) + + def test_extract_without_text(self): + tmpl = MarkupTemplate(""" +

          Foo

          + ${ngettext("Singular", "Plural", num)} + """) + translator = Translator(extract_text=False) + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((3, 'ngettext', (u'Singular', u'Plural', None)), + messages[0]) + def test_extract_plural_form(self): tmpl = MarkupTemplate(""" ${ngettext("Singular", "Plural", num)} @@ -31,6 +56,24 @@ self.assertEqual((2, 'ngettext', (u'Singular', u'Plural', None)), messages[0]) + def test_extract_funky_plural_form(self): + tmpl = MarkupTemplate(""" + ${ngettext(len(items), *widget.display_names)} + """) + translator = Translator() + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((2, 'ngettext', (None, None)), messages[0]) + + def test_extract_gettext_with_unicode_string(self): + tmpl = MarkupTemplate(""" + ${gettext("Grüße")} + """) + translator = Translator() + messages = list(translator.extract(tmpl.stream)) + self.assertEqual(1, len(messages)) + self.assertEqual((2, 'gettext', u'Gr\xfc\xdfe'), messages[0]) + def test_extract_included_attribute_text(self): tmpl = MarkupTemplate(""" @@ -268,6 +311,18 @@ []), ], results) + def test_extraction_without_text(self): + buf = StringIO(""" +

          Foo

          + ${ngettext("Singular", "Plural", num)} + """) + results = list(extract(buf, ['_', 'ngettext'], [], { + 'extract_text': 'no' + })) + self.assertEqual([ + (3, 'ngettext', (u'Singular', u'Plural', None), []), + ], results) + def test_text_template_extraction(self): buf = StringIO("""${_("Dear %(name)s") % {'name': name}}, @@ -332,7 +387,7 @@ def suite(): suite = unittest.TestSuite() - suite.addTests(doctest.DocTestSuite(Translator.__module__)) + suite.addTest(doctest.DocTestSuite(Translator.__module__)) suite.addTest(unittest.makeSuite(TranslatorTestCase, 'test')) suite.addTest(unittest.makeSuite(ExtractTestCase, 'test')) return suite diff --git a/genshi/filters/tests/transform.py b/genshi/filters/tests/transform.py --- a/genshi/filters/tests/transform.py +++ b/genshi/filters/tests/transform.py @@ -19,9 +19,12 @@ def suite(): from genshi.input import HTML + from genshi.core import Markup + from genshi.builder import tag suite = doctest.DocTestSuite(genshi.filters.transform, optionflags=doctest.NORMALIZE_WHITESPACE, - extraglobs={'HTML': HTML}) + extraglobs={'HTML': HTML, 'tag': tag, + 'Markup': Markup}) return suite if __name__ == '__main__': diff --git a/genshi/filters/transform.py b/genshi/filters/transform.py --- a/genshi/filters/transform.py +++ b/genshi/filters/transform.py @@ -43,13 +43,15 @@ The ``Transformer`` support a large number of useful transformations out of the box, but custom transformations can be added easily. + +:since: version 0.5 """ import re import sys from genshi.builder import Element -from genshi.core import Stream, Attrs, QName, TEXT, START, END, _ensure +from genshi.core import Stream, Attrs, QName, TEXT, START, END, _ensure, Markup from genshi.path import Path __all__ = ['Transformer', 'StreamBuffer', 'InjectorTransformation', 'ENTER', @@ -141,14 +143,12 @@ __slots__ = ['transforms'] - def __init__(self, path=None): + def __init__(self, path='.'): """Construct a new transformation filter. :param path: an XPath expression (as string) or a `Path` instance """ - self.transforms = [] - if path: - self.transforms.append(SelectTransformation(path)) + self.transforms = [SelectTransformation(path)] def __call__(self, stream): """Apply the transform filter to the marked stream. @@ -160,7 +160,8 @@ transforms = self._mark(stream) for link in self.transforms: transforms = link(transforms) - return Stream(self._unmark(transforms)) + return Stream(self._unmark(transforms), + serializer=getattr(stream, 'serializer', None)) def apply(self, function): """Apply a transformation to the stream. @@ -548,6 +549,10 @@ ... 'some bold text') >>> print html | Transformer('body').substitute('(?i)some', 'SOME') SOME text, some more text and SOME bold text + >>> tags = tag.html(tag.body('Some text, some more text and ', + ... Markup('some bold text'))) + >>> print tags.generate() | Transformer('body').substitute('(?i)some', 'SOME') + SOME text, some more text and SOME bold text :param pattern: A regular expression object or string. :param replace: Replacement pattern. @@ -556,6 +561,16 @@ """ return self.apply(SubstituteTransformation(pattern, replace, count)) + def rename(self, name): + """Rename matching elements. + + >>> html = HTML('Some text, some more text and ' + ... 'some bold text') + >>> print html | Transformer('body/b').rename('strong') + Some text, some more text and some bold text + """ + return self.apply(RenameTransformation(name)) + def trace(self, prefix='', fileobj=None): """Print events as they pass through the transform. @@ -732,7 +747,10 @@ yield None, prefix yield mark, event while True: - mark, event = stream.next() + try: + mark, event = stream.next() + except StopIteration: + yield None, element[-1] if not mark: break yield mark, event @@ -791,8 +809,8 @@ if mark: queue.append(event) else: - for event in flush(queue): - yield event + for queue_event in flush(queue): + yield queue_event yield None, event for event in flush(queue): yield event @@ -851,7 +869,33 @@ """ for mark, (kind, data, pos) in stream: if kind is TEXT: - data = self.pattern.sub(self.replace, data, self.count) + new_data = self.pattern.sub(self.replace, data, self.count) + if isinstance(data, Markup): + data = Markup(new_data) + else: + data = new_data + yield mark, (kind, data, pos) + + +class RenameTransformation(object): + """Rename matching elements.""" + def __init__(self, name): + """Create the transform. + + :param name: New element name. + """ + self.name = QName(name) + + def __call__(self, stream): + """Apply the transform filter to the marked stream. + + :param stream: The marked event stream to filter + """ + for mark, (kind, data, pos) in stream: + if mark is ENTER: + data = self.name, data[1] + elif mark is EXIT: + data = self.name yield mark, (kind, data, pos) @@ -912,9 +956,15 @@ :param stream: The marked event stream to filter """ for mark, event in stream: - if mark in (ENTER, OUTSIDE): + if mark is not None: for subevent in self._inject(): yield subevent + yield mark, event + while True: + mark, event = stream.next() + if not mark: + break + yield mark, event yield mark, event @@ -930,7 +980,10 @@ yield mark, event if mark: while True: - mark, event = stream.next() + try: + mark, event = stream.next() + except StopIteration: + break if not mark: break yield mark, event diff --git a/genshi/output.py b/genshi/output.py --- a/genshi/output.py +++ b/genshi/output.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -30,7 +30,7 @@ 'XHTMLSerializer', 'HTMLSerializer', 'TextSerializer'] __docformat__ = 'restructuredtext en' -def encode(iterator, method='xml', encoding='utf-8'): +def encode(iterator, method='xml', encoding='utf-8', out=None): """Encode serializer output into a string. :param iterator: the iterator returned from serializing a stream (basically @@ -39,16 +39,27 @@ representable in the specified encoding are treated :param encoding: how the output string should be encoded; if set to `None`, this method returns a `unicode` object - :return: a string or unicode object (depending on the `encoding` parameter) + :param out: a file-like object that the output should be written to + instead of being returned as one big string; note that if + this is a file or socket (or similar), the `encoding` must + not be `None` (that is, the output must be encoded) + :return: a `str` or `unicode` object (depending on the `encoding` + parameter), or `None` if the `out` parameter is provided + :since: version 0.4.1 + :note: Changed in 0.5: added the `out` parameter """ - output = u''.join(list(iterator)) if encoding is not None: errors = 'replace' if method != 'text' and not isinstance(method, TextSerializer): errors = 'xmlcharrefreplace' - return output.encode(encoding, errors) - return output + _encode = lambda string: string.encode(encoding, errors) + else: + _encode = lambda string: string + if out is None: + return _encode(u''.join(list(iterator))) + for chunk in iterator: + out.write(_encode(chunk)) def get_serializer(method='xml', **kwargs): """Return a serializer object for the given method. @@ -103,6 +114,20 @@ ) XHTML = XHTML_STRICT + SVG_FULL = ( + 'svg', '-//W3C//DTD SVG 1.1//EN', + 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd' + ) + SVG_BASIC = ( + 'svg', '-//W3C//DTD SVG Basic 1.1//EN', + 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11-basic.dtd' + ) + SVG_TINY = ( + 'svg', '-//W3C//DTD SVG Tiny 1.1//EN', + 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11-tiny.dtd' + ) + SVG = SVG_FULL + def get(cls, name): """Return the ``(name, pubid, sysid)`` tuple of the ``DOCTYPE`` declaration for the specified name. @@ -115,6 +140,9 @@ * "xhtml" or "xhtml-strict" for the XHTML 1.0 strict DTD * "xhtml-transitional" for the XHTML 1.0 transitional DTD * "xhtml-frameset" for the XHTML 1.0 frameset DTD + * "svg" or "svg-full" for the SVG 1.1 DTD + * "svg-basic" for the SVG Basic 1.1 DTD + * "svg-tiny" for the SVG Tiny 1.1 DTD :param name: the name of the ``DOCTYPE`` :return: the ``(name, pubid, sysid)`` tuple for the requested @@ -129,6 +157,9 @@ 'xhtml': cls.XHTML, 'xhtml-strict': cls.XHTML_STRICT, 'xhtml-transitional': cls.XHTML_TRANSITIONAL, 'xhtml-frameset': cls.XHTML_FRAMESET, + 'svg': cls.SVG, 'svg-full': cls.SVG_FULL, + 'svg-basic': cls.SVG_BASIC, + 'svg-tiny': cls.SVG_TINY }.get(name.lower()) get = classmethod(get) @@ -156,21 +187,17 @@ stripped from the output :note: Changed in 0.4.2: The `doctype` parameter can now be a string. """ - self.preamble = [] - if doctype: - if isinstance(doctype, basestring): - doctype = DocType.get(doctype) - self.preamble.append((DOCTYPE, doctype, (None, -1, -1))) self.filters = [EmptyTagFilter()] if strip_whitespace: self.filters.append(WhitespaceFilter(self._PRESERVE_SPACE)) self.filters.append(NamespaceFlattener(prefixes=namespace_prefixes)) + if doctype: + self.filters.append(DocTypeInserter(doctype)) def __call__(self, stream): have_decl = have_doctype = False in_cdata = False - stream = chain(self.preamble, stream) for filter_ in self.filters: stream = filter_(stream) for kind, data, pos in stream: @@ -217,7 +244,7 @@ if sysid: buf.append(' "%s"') buf.append('>\n') - yield Markup(u''.join(buf), *filter(None, data)) + yield Markup(u''.join(buf)) % filter(None, data) have_doctype = True elif kind is START_CDATA: @@ -261,6 +288,8 @@ namespace_prefixes = namespace_prefixes or {} namespace_prefixes['http://www.w3.org/1999/xhtml'] = '' self.filters.append(NamespaceFlattener(prefixes=namespace_prefixes)) + if doctype: + self.filters.append(DocTypeInserter(doctype)) def __call__(self, stream): boolean_attrs = self._BOOLEAN_ATTRS @@ -268,7 +297,6 @@ have_doctype = False in_cdata = False - stream = chain(self.preamble, stream) for filter_ in self.filters: stream = filter_(stream) for kind, data, pos in stream: @@ -281,6 +309,8 @@ value = attr elif attr == u'xml:lang' and u'lang' not in attrib: buf += [' lang="', escape(value), '"'] + elif attr == u'xml:space': + continue buf += [' ', attr, '="', escape(value), '"'] if kind is EMPTY: if tag in empty_elems: @@ -313,7 +343,7 @@ if sysid: buf.append(' "%s"') buf.append('>\n') - yield Markup(u''.join(buf), *filter(None, data)) + yield Markup(u''.join(buf)) % filter(None, data) have_doctype = True elif kind is START_CDATA: @@ -359,6 +389,8 @@ self.filters.append(NamespaceFlattener(prefixes={ 'http://www.w3.org/1999/xhtml': '' })) + if doctype: + self.filters.append(DocTypeInserter(doctype)) def __call__(self, stream): boolean_attrs = self._BOOLEAN_ATTRS @@ -367,7 +399,6 @@ have_doctype = False noescape = False - stream = chain(self.preamble, stream) for filter_ in self.filters: stream = filter_(stream) for kind, data, pos in stream: @@ -415,7 +446,7 @@ if sysid: buf.append(' "%s"') buf.append('>\n') - yield Markup(u''.join(buf), *filter(None, data)) + yield Markup(u''.join(buf)) % filter(None, data) have_doctype = True elif kind is PI: @@ -436,20 +467,29 @@ If text events contain literal markup (instances of the `Markup` class), - tags or entities are stripped from the output: + that markup is by default passed through unchanged: - >>> elem = tag.div(Markup('Hello!
          ')) - >>> print elem - - >>> print ''.join(TextSerializer()(elem.generate())) - Hello! + >>> elem = tag.div(Markup('Hello & Bye!
          ')) + >>> print elem.generate().render(TextSerializer) + Hello & Bye!
          + + You can use the `strip_markup` to change this behavior, so that tags and + entities are stripped from the output (or in the case of entities, + replaced with the equivalent character): + + >>> print elem.generate().render(TextSerializer, strip_markup=True) + Hello & Bye! """ + def __init__(self, strip_markup=False): + self.strip_markup = strip_markup + def __call__(self, stream): + strip_markup = self.strip_markup for event in stream: if event[0] is TEXT: data = event[1] - if type(data) is Markup: + if strip_markup and type(data) is Markup: data = data.striptags().stripentities() yield unicode(data) @@ -667,3 +707,33 @@ if kind: yield kind, data, pos + + +class DocTypeInserter(object): + """A filter that inserts the DOCTYPE declaration in the correct location, + after the XML declaration. + """ + def __init__(self, doctype): + """Initialize the filter. + + :param doctype: DOCTYPE as a string or DocType object. + """ + if isinstance(doctype, basestring): + doctype = DocType.get(doctype) + self.doctype_event = (DOCTYPE, doctype, (None, -1, -1)) + + def __call__(self, stream): + doctype_inserted = False + for kind, data, pos in stream: + if not doctype_inserted: + doctype_inserted = True + if kind is XML_DECL: + yield (kind, data, pos) + yield self.doctype_event + continue + yield self.doctype_event + + yield (kind, data, pos) + + if not doctype_inserted: + yield self.doctype_event diff --git a/genshi/path.py b/genshi/path.py --- a/genshi/path.py +++ b/genshi/path.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -39,10 +39,12 @@ """ from math import ceil, floor +import operator import re from genshi.core import Stream, Attrs, Namespace, QName -from genshi.core import START, END, TEXT, COMMENT, PI +from genshi.core import START, END, TEXT, START_NS, END_NS, COMMENT, PI, \ + START_CDATA, END_CDATA __all__ = ['Path', 'PathSyntaxError'] __docformat__ = 'restructuredtext en' @@ -146,7 +148,8 @@ updateonly=True) elif result: yield result - return Stream(_generate()) + return Stream(_generate(), + serializer=getattr(stream, 'serializer', None)) def test(self, ignore_context=False): """Returns a function that can be used to track whether the path matches @@ -193,6 +196,9 @@ continue elif kind is START: cursors.append(cursors and cursors[-1] or 0) + elif kind is START_NS or kind is END_NS \ + or kind is START_CDATA or kind is END_CDATA: + continue if updateonly or retval or not cursors: continue @@ -270,6 +276,8 @@ if not matched or last_step or not ( axis is SELF or axis is DESCENDANT_OR_SELF): break + if ctxtnode and axis is DESCENDANT_OR_SELF: + ctxtnode = False if (retval or not matched) and kind is START and \ not (axis is DESCENDANT or axis is DESCENDANT_OR_SELF): @@ -771,7 +779,7 @@ string2 = as_string(self.string2(kind, data, pos, namespaces, variables)) return re.search(string2, string1, self.flags) def _map_flags(self, flags): - return reduce(lambda a, b: a | b, + return reduce(operator.or_, [self.flag_map[flag] for flag in flags], re.U) def __repr__(self): return 'contains(%r, %r)' % (self.string1, self.string2) diff --git a/genshi/template/__init__.py b/genshi/template/__init__.py --- a/genshi/template/__init__.py +++ b/genshi/template/__init__.py @@ -18,6 +18,6 @@ BadDirectiveError from genshi.template.loader import TemplateLoader, TemplateNotFound from genshi.template.markup import MarkupTemplate -from genshi.template.text import TextTemplate +from genshi.template.text import TextTemplate, OldTextTemplate, NewTextTemplate __docformat__ = 'restructuredtext en' diff --git a/genshi/template/base.py b/genshi/template/base.py --- a/genshi/template/base.py +++ b/genshi/template/base.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -15,6 +15,7 @@ import os from StringIO import StringIO +import sys from genshi.core import Attrs, Stream, StreamEventKind, START, TEXT, _ensure from genshi.input import ParseError @@ -27,7 +28,7 @@ class TemplateError(Exception): """Base exception class for errors related to template processing.""" - def __init__(self, message, filename='', lineno=-1, offset=-1): + def __init__(self, message, filename=None, lineno=-1, offset=-1): """Create the exception. :param message: the error message @@ -36,6 +37,8 @@ occurred :param offset: the column number at which the error occurred """ + if filename is None: + filename = '' self.msg = message #: the error message string if filename != '' or lineno >= 0: message = '%s (%s, line %d)' % (self.msg, filename, lineno) @@ -50,7 +53,7 @@ error, or the template is not well-formed. """ - def __init__(self, message, filename='', lineno=-1, offset=-1): + def __init__(self, message, filename=None, lineno=-1, offset=-1): """Create the exception :param message: the error message @@ -72,7 +75,7 @@ with a local name that doesn't match any registered directive. """ - def __init__(self, name, filename='', lineno=-1): + def __init__(self, name, filename=None, lineno=-1): """Create the exception :param name: the name of the directive @@ -169,19 +172,54 @@ self.pop() self.push(data) - -def _apply_directives(stream, ctxt, directives): +def _apply_directives(stream, directives, ctxt, **vars): """Apply the given directives to the stream. :param stream: the stream the directives should be applied to + :param directives: the list of directives to apply :param ctxt: the `Context` - :param directives: the list of directives to apply + :param vars: additional variables that should be available when Python + code is executed :return: the stream with the given directives applied """ if directives: - stream = directives[0](iter(stream), ctxt, directives[1:]) + stream = directives[0](iter(stream), directives[1:], ctxt, **vars) return stream +def _eval_expr(expr, ctxt, **vars): + """Evaluate the given `Expression` object. + + :param expr: the expression to evaluate + :param ctxt: the `Context` + :param vars: additional variables that should be available to the + expression + :return: the result of the evaluation + """ + if vars: + ctxt.push(vars) + retval = expr.evaluate(ctxt.data) + if vars: + ctxt.pop() + return retval + +def _exec_suite(suite, ctxt, **vars): + """Execute the given `Suite` object. + + :param suite: the code suite to execute + :param ctxt: the `Context` + :param vars: additional variables that should be available to the + code + """ + frame = {} + if vars: + ctxt.push(vars) + ctxt.push(frame) + suite.execute(ctxt.data) + if vars: + ctxt.pop() + ctxt.pop() + ctxt.data.update(frame) + class TemplateMeta(type): """Meta class for templates.""" @@ -202,6 +240,9 @@ """ __metaclass__ = TemplateMeta + EXEC = StreamEventKind('EXEC') + """Stream event kind representing a Python code suite to execute.""" + EXPR = StreamEventKind('EXPR') """Stream event kind representing a Python expression.""" @@ -213,38 +254,35 @@ directives should be applied. """ - def __init__(self, source, basedir=None, filename=None, loader=None, - encoding=None, lookup='lenient', allow_exec=True): + serializer = None + _number_conv = unicode # function used to convert numbers to event data + + def __init__(self, source, filepath=None, filename=None, loader=None, + encoding=None, lookup='strict', allow_exec=True): """Initialize a template from either a string, a file-like object, or an already parsed markup stream. :param source: a string, file-like object, or markup stream to read the template from - :param basedir: the base directory containing the template file; when - loaded from a `TemplateLoader`, this will be the - directory on the template search path in which the - template was found - :param filename: the name of the template file, relative to the given - base directory + :param filepath: the absolute path to the template file + :param filename: the path to the template file relative to the search + path :param loader: the `TemplateLoader` to use for loading included templates :param encoding: the encoding of the `source` - :param lookup: the variable lookup mechanism; either "lenient" (the - default), "strict", or a custom lookup class + :param lookup: the variable lookup mechanism; either "strict" (the + default), "lenient", or a custom lookup class :param allow_exec: whether Python code blocks in templates should be allowed :note: Changed in 0.5: Added the `allow_exec` argument """ - self.basedir = basedir + self.filepath = filepath or filename self.filename = filename - if basedir and filename: - self.filepath = os.path.join(basedir, filename) - else: - self.filepath = filename self.loader = loader self.lookup = lookup self.allow_exec = allow_exec + self._init_filters() if isinstance(source, basestring): source = StringIO(source) @@ -254,13 +292,24 @@ self.stream = list(self._prepare(self._parse(source, encoding))) except ParseError, e: raise TemplateSyntaxError(e.msg, self.filepath, e.lineno, e.offset) - self.filters = [self._flatten, self._eval] - if loader: - self.filters.append(self._include) + + def __getstate__(self): + state = self.__dict__.copy() + state['filters'] = [] + return state + + def __setstate__(self, state): + self.__dict__ = state + self._init_filters() def __repr__(self): return '<%s "%s">' % (self.__class__.__name__, self.filename) + def _init_filters(self): + self.filters = [self._flatten, self._eval, self._exec] + if self.loader: + self.filters.append(self._include) + def _parse(self, source, encoding): """Parse the template. @@ -299,7 +348,7 @@ yield event else: if kind is INCLUDE: - href, fallback = data + href, cls, fallback = data if isinstance(href, basestring) and \ not getattr(self.loader, 'auto_reload', True): # If the path to the included template is static, and @@ -307,7 +356,7 @@ # the template is inlined into the stream try: tmpl = self.loader.load(href, relative_to=pos[0], - cls=self.__class__) + cls=cls or self.__class__) for event in tmpl.stream: yield event except TemplateNotFound: @@ -316,9 +365,9 @@ for event in self._prepare(fallback): yield event continue - else: + elif fallback: # Otherwise the include is performed at run time - data = href, list(self._prepare(fallback)) + data = href, cls, list(self._prepare(fallback)) yield kind, data, pos @@ -335,25 +384,29 @@ :return: a markup event stream representing the result of applying the template to the context data. """ + vars = {} if args: assert len(args) == 1 ctxt = args[0] if ctxt is None: ctxt = Context(**kwargs) + else: + vars = kwargs assert isinstance(ctxt, Context) else: ctxt = Context(**kwargs) stream = self.stream for filter_ in self.filters: - stream = filter_(iter(stream), ctxt) - return Stream(stream) + stream = filter_(iter(stream), ctxt, **vars) + return Stream(stream, self.serializer) - def _eval(self, stream, ctxt): + def _eval(self, stream, ctxt, **vars): """Internal stream filter that evaluates any expressions in `START` and `TEXT` events. """ filters = (self._flatten, self._eval) + number_conv = self._number_conv for kind, data, pos in stream: @@ -368,7 +421,8 @@ else: values = [] for subkind, subdata, subpos in self._eval(substream, - ctxt): + ctxt, + **vars): if subkind is TEXT: values.append(subdata) value = [x for x in values if x is not None] @@ -378,17 +432,19 @@ yield kind, (tag, Attrs(new_attrs)), pos elif kind is EXPR: - result = data.evaluate(ctxt.data) + result = _eval_expr(data, ctxt, **vars) if result is not None: - # First check for a string, otherwise the iterable test below - # succeeds, and the string will be chopped up into individual - # characters + # First check for a string, otherwise the iterable test + # below succeeds, and the string will be chopped up into + # individual characters if isinstance(result, basestring): yield TEXT, result, pos + elif isinstance(result, (int, float, long)): + yield TEXT, number_conv(result), pos elif hasattr(result, '__iter__'): substream = _ensure(result) for filter_ in filters: - substream = filter_(substream, ctxt) + substream = filter_(substream, ctxt, **vars) for event in substream: yield event else: @@ -397,20 +453,29 @@ else: yield kind, data, pos - def _flatten(self, stream, ctxt): + def _exec(self, stream, ctxt, **vars): + """Internal stream filter that executes Python code blocks.""" + for event in stream: + if event[0] is EXEC: + _exec_suite(event[1], ctxt, **vars) + else: + yield event + + def _flatten(self, stream, ctxt, **vars): """Internal stream filter that expands `SUB` events in the stream.""" for event in stream: if event[0] is SUB: # This event is a list of directives and a list of nested # events to which those directives should be applied directives, substream = event[1] - substream = _apply_directives(substream, ctxt, directives) - for event in self._flatten(substream, ctxt): + substream = _apply_directives(substream, directives, ctxt, + **vars) + for event in self._flatten(substream, ctxt, **vars): yield event else: yield event - def _include(self, stream, ctxt): + def _include(self, stream, ctxt, **vars): """Internal stream filter that performs inclusion of external template files. """ @@ -418,29 +483,31 @@ for event in stream: if event[0] is INCLUDE: - href, fallback = event[1] + href, cls, fallback = event[1] if not isinstance(href, basestring): parts = [] - for subkind, subdata, subpos in self._eval(href, ctxt): + for subkind, subdata, subpos in self._eval(href, ctxt, + **vars): if subkind is TEXT: parts.append(subdata) href = u''.join([x for x in parts if x is not None]) try: tmpl = self.loader.load(href, relative_to=event[2][0], - cls=self.__class__) - for event in tmpl.generate(ctxt): + cls=cls or self.__class__) + for event in tmpl.generate(ctxt, **vars): yield event except TemplateNotFound: if fallback is None: raise for filter_ in self.filters: - fallback = filter_(iter(fallback), ctxt) + fallback = filter_(iter(fallback), ctxt, **vars) for event in fallback: yield event else: yield event +EXEC = Template.EXEC EXPR = Template.EXPR INCLUDE = Template.INCLUDE SUB = Template.SUB diff --git a/genshi/template/directives.py b/genshi/template/directives.py --- a/genshi/template/directives.py +++ b/genshi/template/directives.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -14,12 +14,17 @@ """Implementation of the various template directives.""" import compiler +try: + frozenset +except NameError: + from sets import ImmutableSet as frozenset from genshi.core import QName, Stream from genshi.path import Path from genshi.template.base import TemplateRuntimeError, TemplateSyntaxError, \ - EXPR, _apply_directives -from genshi.template.eval import Expression, _parse + EXPR, _apply_directives, _eval_expr, \ + _exec_suite +from genshi.template.eval import Expression, ExpressionASTTransformer, _parse __all__ = ['AttrsDirective', 'ChooseDirective', 'ContentDirective', 'DefDirective', 'ForDirective', 'IfDirective', 'MatchDirective', @@ -83,13 +88,15 @@ return cls(value, template, namespaces, *pos[1:]), stream attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): """Apply the directive to the given stream. :param stream: the event stream - :param ctxt: the context data :param directives: a list of the remaining directives that should process the stream + :param ctxt: the context data + :param vars: additional variables that should be made available when + Python code is executed """ raise NotImplementedError @@ -162,10 +169,10 @@ """ __slots__ = [] - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): def _generate(): kind, (tag, attrib), pos = stream.next() - attrs = self.expr.evaluate(ctxt.data) + attrs = _eval_expr(self.expr, ctxt, **vars) if attrs: if isinstance(attrs, Stream): try: @@ -181,7 +188,7 @@ for event in stream: yield event - return _apply_directives(_generate(), ctxt, directives) + return _apply_directives(_generate(), directives, ctxt, **vars) class ContentDirective(Directive): @@ -202,6 +209,10 @@ __slots__ = [] def attach(cls, template, stream, value, namespaces, pos): + if type(value) is dict: + raise TemplateSyntaxError('The content directive can not be used ' + 'as an element', template.filepath, + *pos[1:]) expr = cls._parse_expr(value, template, *pos[1:]) return None, [stream[0], (EXPR, expr, pos), stream[-1]] attach = classmethod(attach) @@ -282,7 +293,7 @@ namespaces, pos) attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): stream = list(stream) def function(*args, **kwargs): @@ -295,14 +306,14 @@ if name in kwargs: val = kwargs.pop(name) else: - val = self.defaults.get(name).evaluate(ctxt.data) + val = _eval_expr(self.defaults.get(name), ctxt, **vars) scope[name] = val if not self.star_args is None: scope[self.star_args] = args if not self.dstar_args is None: scope[self.dstar_args] = kwargs ctxt.push(scope) - for event in _apply_directives(stream, ctxt, directives): + for event in _apply_directives(stream, directives, ctxt, **vars): yield event ctxt.pop() try: @@ -343,10 +354,10 @@ template.filepath, lineno, offset) assign, value = value.split(' in ', 1) ast = _parse(assign, 'exec') + value = 'iter(%s)' % value.strip() self.assign = _assignment(ast.node.nodes[0].expr) self.filename = template.filepath - Directive.__init__(self, value.strip(), template, namespaces, lineno, - offset) + Directive.__init__(self, value, template, namespaces, lineno, offset) def attach(cls, template, stream, value, namespaces, pos): if type(value) is dict: @@ -355,24 +366,20 @@ namespaces, pos) attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): - iterable = self.expr.evaluate(ctxt.data) + def __call__(self, stream, directives, ctxt, **vars): + iterable = _eval_expr(self.expr, ctxt, **vars) if iterable is None: return assign = self.assign scope = {} stream = list(stream) - try: - iterator = iter(iterable) - for item in iterator: - assign(scope, item) - ctxt.push(scope) - for event in _apply_directives(stream, ctxt, directives): - yield event - ctxt.pop() - except TypeError, e: - raise TemplateRuntimeError(str(e), self.filename, *stream[0][2][1:]) + for item in iterable: + assign(scope, item) + ctxt.push(scope) + for event in _apply_directives(stream, directives, ctxt, **vars): + yield event + ctxt.pop() def __repr__(self): return '<%s>' % self.__class__.__name__ @@ -400,9 +407,10 @@ namespaces, pos) attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): - if self.expr.evaluate(ctxt.data): - return _apply_directives(stream, ctxt, directives) + def __call__(self, stream, directives, ctxt, **vars): + value = _eval_expr(self.expr, ctxt, **vars) + if value: + return _apply_directives(stream, directives, ctxt, **vars) return [] @@ -423,24 +431,33 @@
          """ - __slots__ = ['path', 'namespaces'] + __slots__ = ['path', 'namespaces', 'hints'] - def __init__(self, value, template, namespaces=None, lineno=-1, offset=-1): + def __init__(self, value, template, hints=None, namespaces=None, + lineno=-1, offset=-1): Directive.__init__(self, None, template, namespaces, lineno, offset) self.path = Path(value, template.filepath, lineno) self.namespaces = namespaces or {} + self.hints = hints or () def attach(cls, template, stream, value, namespaces, pos): + hints = [] if type(value) is dict: + if value.get('buffer', '').lower() == 'false': + hints.append('not_buffered') + if value.get('once', '').lower() == 'true': + hints.append('match_once') + if value.get('recursive', '').lower() == 'false': + hints.append('not_recursive') value = value.get('path') - return super(MatchDirective, cls).attach(template, stream, value, - namespaces, pos) + return cls(value, template, frozenset(hints), namespaces, *pos[1:]), \ + stream attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): ctxt._match_templates.append((self.path.test(ignore_context=True), - self.path, list(stream), self.namespaces, - directives)) + self.path, list(stream), self.hints, + self.namespaces, directives)) return [] def __repr__(self): @@ -476,6 +493,8 @@ __slots__ = [] def attach(cls, template, stream, value, namespaces, pos): + if type(value) is dict: + value = value.get('value') if not value: raise TemplateSyntaxError('missing value for "replace" directive', template.filepath, *pos[1:]) @@ -517,9 +536,9 @@ """ __slots__ = [] - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): def _generate(): - if self.expr.evaluate(ctxt.data): + if _eval_expr(self.expr, ctxt, **vars): stream.next() # skip start tag previous = stream.next() for event in stream: @@ -528,7 +547,7 @@ else: for event in stream: yield event - return _apply_directives(_generate(), ctxt, directives) + return _apply_directives(_generate(), directives, ctxt, **vars) def attach(cls, template, stream, value, namespaces, pos): if not value: @@ -586,12 +605,12 @@ namespaces, pos) attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): info = [False, bool(self.expr), None] if self.expr: info[2] = self.expr.evaluate(ctxt.data) ctxt._choice_stack.append(info) - for event in _apply_directives(stream, ctxt, directives): + for event in _apply_directives(stream, directives, ctxt, **vars): yield event ctxt._choice_stack.pop() @@ -615,7 +634,7 @@ namespaces, pos) attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): info = ctxt._choice_stack and ctxt._choice_stack[-1] if not info: raise TemplateRuntimeError('"when" directives can only be used ' @@ -630,16 +649,16 @@ if info[1]: value = info[2] if self.expr: - matched = value == self.expr.evaluate(ctxt.data) + matched = value == _eval_expr(self.expr, ctxt, **vars) else: matched = bool(value) else: - matched = bool(self.expr.evaluate(ctxt.data)) + matched = bool(_eval_expr(self.expr, ctxt, **vars)) info[0] = matched if not matched: return [] - return _apply_directives(stream, ctxt, directives) + return _apply_directives(stream, directives, ctxt, **vars) class OtherwiseDirective(Directive): @@ -654,7 +673,7 @@ Directive.__init__(self, None, template, namespaces, lineno, offset) self.filename = template.filepath - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): info = ctxt._choice_stack and ctxt._choice_stack[-1] if not info: raise TemplateRuntimeError('an "otherwise" directive can only be ' @@ -664,7 +683,7 @@ return [] info[0] = True - return _apply_directives(stream, ctxt, directives) + return _apply_directives(stream, directives, ctxt, **vars) class WithDirective(Directive): @@ -684,20 +703,20 @@ def __init__(self, value, template, namespaces=None, lineno=-1, offset=-1): Directive.__init__(self, None, template, namespaces, lineno, offset) - self.vars = [] - value = value.strip() + self.vars = [] + value = value.strip() try: - ast = _parse(value, 'exec').node - for node in ast.nodes: - if isinstance(node, compiler.ast.Discard): - continue - elif not isinstance(node, compiler.ast.Assign): - raise TemplateSyntaxError('only assignment allowed in ' - 'value of the "with" directive', - template.filepath, lineno, offset) - self.vars.append(([_assignment(n) for n in node.nodes], - Expression(node.expr, template.filepath, - lineno, lookup=template.lookup))) + ast = _parse(value, 'exec').node + for node in ast.nodes: + if isinstance(node, compiler.ast.Discard): + continue + elif not isinstance(node, compiler.ast.Assign): + raise TemplateSyntaxError('only assignment allowed in ' + 'value of the "with" directive', + template.filepath, lineno, offset) + self.vars.append(([_assignment(n) for n in node.nodes], + Expression(node.expr, template.filepath, + lineno, lookup=template.lookup))) except SyntaxError, err: err.msg += ' in expression "%s" of "%s" directive' % (value, self.tagname) @@ -711,15 +730,16 @@ namespaces, pos) attach = classmethod(attach) - def __call__(self, stream, ctxt, directives): + def __call__(self, stream, directives, ctxt, **vars): frame = {} ctxt.push(frame) for targets, expr in self.vars: value = expr.evaluate(ctxt.data) for assign in targets: assign(frame, value) - ctxt.replace(frame) - for event in _apply_directives(stream, ctxt, directives): + ctxt.pop() + ctxt.push(frame) + for event in _apply_directives(stream, directives, ctxt, **vars): yield event ctxt.pop() diff --git a/genshi/template/eval.py b/genshi/template/eval.py --- a/genshi/template/eval.py +++ b/genshi/template/eval.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -23,6 +23,7 @@ from sets import ImmutableSet as frozenset from sets import Set as set import sys +from textwrap import dedent from genshi.core import Markup from genshi.template.base import TemplateRuntimeError @@ -37,7 +38,8 @@ """Abstract base class for the `Expression` and `Suite` classes.""" __slots__ = ['source', 'code', 'ast', '_globals'] - def __init__(self, source, filename=None, lineno=-1, lookup='lenient'): + def __init__(self, source, filename=None, lineno=-1, lookup='strict', + xform=None): """Create the code object, either from a string, or from an AST node. :param source: either a string containing the source code, or an AST @@ -46,14 +48,18 @@ the code :param lineno: the number of the line on which the code was found :param lookup: the lookup class that defines how variables are looked - up in the context. Can be either `LenientLookup` (the - default), `StrictLookup`, or a custom lookup class + up in the context; can be either "strict" (the default), + "lenient", or a custom lookup class + :param xform: the AST transformer that should be applied to the code; + if `None`, the appropriate transformation is chosen + depending on the mode """ if isinstance(source, basestring): self.source = source node = _parse(source, mode=self.mode) else: - assert isinstance(source, ast.Node) + assert isinstance(source, ast.Node), \ + 'Expected string or AST node, but got %r' % source self.source = '?' if self.mode == 'eval': node = ast.Expression(source) @@ -62,12 +68,27 @@ self.ast = node self.code = _compile(node, self.source, mode=self.mode, - filename=filename, lineno=lineno) + filename=filename, lineno=lineno, xform=xform) if lookup is None: lookup = LenientLookup elif isinstance(lookup, basestring): lookup = {'lenient': LenientLookup, 'strict': StrictLookup}[lookup] - self._globals = lookup.globals() + self._globals = lookup.globals + + def __getstate__(self): + state = {'source': self.source, 'ast': self.ast, + 'lookup': self._globals.im_self} + c = self.code + state['code'] = (c.co_nlocals, c.co_stacksize, c.co_flags, c.co_code, + c.co_consts, c.co_names, c.co_varnames, c.co_filename, + c.co_name, c.co_firstlineno, c.co_lnotab, (), ()) + return state + + def __setstate__(self, state): + self.source = state['source'] + self.ast = state['ast'] + self.code = new.code(0, *state['code']) + self._globals = state['lookup'].globals def __eq__(self, other): return (type(other) == type(self)) and (self.code == other.code) @@ -133,8 +154,7 @@ :return: the result of the evaluation """ __traceback_hide__ = 'before_and_this' - _globals = self._globals - _globals['data'] = data + _globals = self._globals(data) return eval(self.code, _globals, data) @@ -155,8 +175,7 @@ :param data: a mapping containing the data to execute in """ __traceback_hide__ = 'before_and_this' - _globals = self._globals - _globals['data'] = data + _globals = self._globals(data) exec self.code in _globals, data @@ -242,14 +261,16 @@ class LookupBase(object): """Abstract base class for variable lookup implementations.""" - def globals(cls): + def globals(cls, data): """Construct the globals dictionary to use as the execution context for the expression or suite. """ return { + '__data__': data, '_lookup_name': cls.lookup_name, '_lookup_attr': cls.lookup_attr, - '_lookup_item': cls.lookup_item + '_lookup_item': cls.lookup_item, + 'UndefinedError': UndefinedError, } globals = classmethod(globals) @@ -259,18 +280,23 @@ if val is UNDEFINED: val = BUILTINS.get(name, val) if val is UNDEFINED: - return cls.undefined(name) + val = cls.undefined(name) return val lookup_name = classmethod(lookup_name) def lookup_attr(cls, obj, key): __traceback_hide__ = True - if hasattr(obj, key): - return getattr(obj, key) try: - return obj[key] - except (KeyError, TypeError): - return cls.undefined(key, owner=obj) + val = getattr(obj, key) + except AttributeError: + if hasattr(obj.__class__, key): + raise + else: + try: + val = obj[key] + except (KeyError, TypeError): + val = cls.undefined(key, owner=obj) + return val lookup_attr = classmethod(lookup_attr) def lookup_item(cls, obj, key): @@ -283,7 +309,7 @@ if isinstance(key, basestring): val = getattr(obj, key, UNDEFINED) if val is UNDEFINED: - return cls.undefined(key, owner=obj) + val = cls.undefined(key, owner=obj) return val raise lookup_item = classmethod(lookup_item) @@ -358,12 +384,24 @@ def _parse(source, mode='eval'): + source = source.strip() + if mode == 'exec': + lines = [line.expandtabs() for line in source.splitlines()] + if lines: + first = lines[0] + rest = dedent('\n'.join(lines[1:])).rstrip() + if first.rstrip().endswith(':') and not rest[0].isspace(): + rest = '\n'.join([' %s' % line for line in rest.splitlines()]) + source = '\n'.join([first, rest]) if isinstance(source, unicode): source = '\xef\xbb\xbf' + source.encode('utf-8') return parse(source, mode) -def _compile(node, source=None, mode='eval', filename=None, lineno=-1): - xform = {'eval': ExpressionASTTransformer}.get(mode, TemplateASTTransformer) +def _compile(node, source=None, mode='eval', filename=None, lineno=-1, + xform=None): + if xform is None: + xform = {'eval': ExpressionASTTransformer}.get(mode, + TemplateASTTransformer) tree = xform().visit(node) if isinstance(filename, unicode): # unicode file names not allowed for code objects @@ -376,10 +414,17 @@ if mode == 'eval': gen = ExpressionCodeGenerator(tree) - name = '' % (repr(source or '?')) + name = '' % (source or '?') else: gen = ModuleCodeGenerator(tree) - name = '' + lines = source.splitlines() + if not lines: + extract = '' + else: + extract = lines[0] + if len(lines) > 1: + extract += ' ...' + name = '' % (extract) gen.optimized = True code = gen.getCode() @@ -416,7 +461,8 @@ node = node.__class__(*args) if lineno is not None: node.lineno = lineno - if isinstance(node, (ast.Class, ast.Function, ast.GenExpr, ast.Lambda)): + if isinstance(node, (ast.Class, ast.Function, ast.Lambda)) or \ + sys.version_info > (2, 4) and isinstance(node, ast.GenExpr): node.filename = '' # workaround for bug in pycodegen return node @@ -443,7 +489,7 @@ def visitClass(self, node): return self._clone(node, node.name, [self.visit(x) for x in node.bases], - node.doc, node.code + node.doc, self.visit(node.code) ) def visitFunction(self, node): @@ -636,7 +682,7 @@ """ def __init__(self): - self.locals = [CONSTANTS, set()] + self.locals = [CONSTANTS] def visitConst(self, node): if isinstance(node.value, str): @@ -647,18 +693,19 @@ return node def visitAssName(self, node): - if self.locals: + if len(self.locals) > 1: self.locals[-1].add(node.name) return node def visitAugAssign(self, node): - if isinstance(node.node, ast.Name): + if isinstance(node.node, ast.Name) \ + and node.node.name not in flatten(self.locals): name = node.node.name - node.node = ast.Subscript(ast.Name('data'), 'OP_APPLY', + node.node = ast.Subscript(ast.Name('__data__'), 'OP_APPLY', [ast.Const(name)]) node.expr = self.visit(node.expr) return ast.If([ - (ast.Compare(ast.Const(name), [('in', ast.Name('data'))]), + (ast.Compare(ast.Const(name), [('in', ast.Name('__data__'))]), ast.Stmt([node]))], ast.Stmt([ast.Raise(ast.CallFunc(ast.Name('UndefinedError'), [ast.Const(name)]), @@ -667,6 +714,8 @@ return ASTTransformer.visitAugAssign(self, node) def visitClass(self, node): + if len(self.locals) > 1: + self.locals[-1].add(node.name) self.locals.append(set()) try: return ASTTransformer.visitClass(self, node) @@ -681,6 +730,8 @@ self.locals.pop() def visitFunction(self, node): + if len(self.locals) > 1: + self.locals[-1].add(node.name) self.locals.append(set(node.argnames)) try: return ASTTransformer.visitFunction(self, node) @@ -711,12 +762,11 @@ def visitName(self, node): # If the name refers to a local inside a lambda, list comprehension, or # generator expression, leave it alone - for frame in self.locals: - if node.name in frame: - return node - # Otherwise, translate the name ref into a context lookup - func_args = [ast.Name('data'), ast.Const(node.name)] - return ast.CallFunc(ast.Name('_lookup_name'), func_args) + if node.name not in flatten(self.locals): + # Otherwise, translate the name ref into a context lookup + func_args = [ast.Name('__data__'), ast.Const(node.name)] + node = ast.CallFunc(ast.Name('_lookup_name'), func_args) + return node class ExpressionASTTransformer(TemplateASTTransformer): diff --git a/genshi/template/interpolation.py b/genshi/template/interpolation.py --- a/genshi/template/interpolation.py +++ b/genshi/template/interpolation.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -30,8 +30,7 @@ NAMECHARS = NAMESTART + '.0123456789' PREFIX = '$' -def interpolate(text, basedir=None, filename=None, lineno=-1, offset=0, - lookup='lenient'): +def interpolate(text, filepath=None, lineno=-1, offset=0, lookup='strict'): """Parse the given string and extract expressions. This function is a generator that yields `TEXT` events for literal strings, @@ -45,9 +44,8 @@ TEXT u'bar' :param text: the text to parse - :param basedir: base directory of the file in which the text was found - (optional) - :param filename: basename of the file in which the text was found (optional) + :param filepath: absolute path to the file in which the text was found + (optional) :param lineno: the line number at which the text was found (optional) :param offset: the column number at which the text starts in the source (optional) @@ -57,9 +55,6 @@ :raise TemplateSyntaxError: when a syntax error in an expression is encountered """ - filepath = filename - if filepath and basedir: - filepath = os.path.join(basedir, filepath) pos = [filepath, lineno, offset] textbuf = [] @@ -73,7 +68,7 @@ if chunk: try: expr = Expression(chunk.strip(), pos[0], pos[1], - lookup=lookup) + lookup=lookup) yield EXPR, expr, tuple(pos) except SyntaxError, err: raise TemplateSyntaxError(err, filepath, pos[1], diff --git a/genshi/template/loader.py b/genshi/template/loader.py --- a/genshi/template/loader.py +++ b/genshi/template/loader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -77,12 +77,15 @@ """ def __init__(self, search_path=None, auto_reload=False, default_encoding=None, max_cache_size=25, default_class=None, - variable_lookup='lenient', allow_exec=True, callback=None): + variable_lookup='strict', allow_exec=True, callback=None): """Create the template laoder. :param search_path: a list of absolute path names that should be searched for template files, or a string containing - a single absolute path + a single absolute path; alternatively, any item on + the list may be a ''load function'' that is passed + a filename and returns a file-like object and some + metadata :param auto_reload: whether to check the last modification time of template files, and reload them if they have changed :param default_encoding: the default encoding to assume when loading @@ -91,8 +94,8 @@ cache :param default_class: the default `Template` subclass to use when instantiating templates - :param variable_lookup: the variable lookup mechanism; either "lenient" - (the default), "strict", or a custom lookup + :param variable_lookup: the variable lookup mechanism; either "strict" + (the default), "lenient", or a custom lookup class :param allow_exec: whether to allow Python code blocks in templates :param callback: (optional) a callback function that is invoked after a @@ -109,7 +112,7 @@ self.search_path = search_path if self.search_path is None: self.search_path = [] - elif isinstance(self.search_path, basestring): + elif not isinstance(self.search_path, (list, tuple)): self.search_path = [self.search_path] self.auto_reload = auto_reload @@ -124,15 +127,15 @@ raise TypeError('The "callback" parameter needs to be callable') self.callback = callback self._cache = LRUCache(max_cache_size) - self._mtime = {} + self._uptodate = {} self._lock = threading.RLock() def load(self, filename, relative_to=None, cls=None, encoding=None): """Load the template with the given name. - If the `filename` parameter is relative, this method searches the search - path trying to locate a template matching the given name. If the file - name is an absolute path, the search path is ignored. + If the `filename` parameter is relative, this method searches the + search path trying to locate a template matching the given name. If the + file name is an absolute path, the search path is ignored. If the requested template is not found, a `TemplateNotFound` exception is raised. Otherwise, a `Template` object is returned that represents @@ -155,26 +158,27 @@ :param encoding: the encoding of the template to load; defaults to the ``default_encoding`` of the loader instance :return: the loaded `Template` instance - :raises TemplateNotFound: if a template with the given name could not be - found + :raises TemplateNotFound: if a template with the given name could not + be found """ if cls is None: cls = self.default_class - if encoding is None: - encoding = self.default_encoding if relative_to and not os.path.isabs(relative_to): filename = os.path.join(os.path.dirname(relative_to), filename) filename = os.path.normpath(filename) + cachekey = filename self._lock.acquire() try: # First check the cache to avoid reparsing the same file try: - tmpl = self._cache[filename] - if not self.auto_reload or \ - os.path.getmtime(tmpl.filepath) == self._mtime[filename]: + tmpl = self._cache[cachekey] + if not self.auto_reload: return tmpl - except KeyError: + uptodate = self._uptodate[cachekey] + if uptodate is not None and uptodate(): + return tmpl + except KeyError, OSError: pass search_path = self.search_path @@ -190,41 +194,135 @@ # template is on the search path dirname = os.path.dirname(relative_to) if dirname not in search_path: - search_path = search_path + [dirname] + search_path = list(search_path) + [dirname] isabs = True elif not search_path: # Uh oh, don't know where to look for the template raise TemplateError('Search path for templates not configured') - for dirname in search_path: - filepath = os.path.join(dirname, filename) + for loadfunc in search_path: + if isinstance(loadfunc, basestring): + loadfunc = directory(loadfunc) try: - fileobj = open(filepath, 'U') + filepath, filename, fileobj, uptodate = loadfunc(filename) + except IOError: + continue + else: try: if isabs: # If the filename of either the included or the # including template is absolute, make sure the # included template gets an absolute path, too, - # so that nested include work properly without a + # so that nested includes work properly without a # search path - filename = os.path.join(dirname, filename) - dirname = '' - tmpl = cls(fileobj, basedir=dirname, filename=filename, - loader=self, encoding=encoding, - lookup=self.variable_lookup, - allow_exec=self.allow_exec) + filename = filepath + tmpl = self._instantiate(cls, fileobj, filepath, + filename, encoding=encoding) if self.callback: self.callback(tmpl) - self._cache[filename] = tmpl - self._mtime[filename] = os.path.getmtime(filepath) + self._cache[cachekey] = tmpl + self._uptodate[cachekey] = uptodate finally: - fileobj.close() + if hasattr(fileobj, 'close'): + fileobj.close() return tmpl - except IOError: - continue raise TemplateNotFound(filename, search_path) finally: self._lock.release() + + def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): + """Instantiate and return the `Template` object based on the given + class and parameters. + + This function is intended for subclasses to override if they need to + implement special template instantiation logic. Code that just uses + the `TemplateLoader` should use the `load` method instead. + + :param cls: the class of the template object to instantiate + :param fileobj: a readable file-like object containing the template + source + :param filepath: the absolute path to the template file + :param filename: the path to the template file relative to the search + path + :param encoding: the encoding of the template to load; defaults to the + ``default_encoding`` of the loader instance + :return: the loaded `Template` instance + :rtype: `Template` + """ + if encoding is None: + encoding = self.default_encoding + return cls(fileobj, filepath=filepath, filename=filename, loader=self, + encoding=encoding, lookup=self.variable_lookup, + allow_exec=self.allow_exec) + + def directory(path): + """Loader factory for loading templates from a local directory. + + :param path: the path to the local directory containing the templates + :return: the loader function to load templates from the given directory + :rtype: ``function`` + """ + def _load_from_directory(filename): + filepath = os.path.join(path, filename) + fileobj = open(filepath, 'U') + mtime = os.path.getmtime(filepath) + def _uptodate(): + return mtime == os.path.getmtime(filepath) + return filepath, filename, fileobj, _uptodate + return _load_from_directory + directory = staticmethod(directory) + + def package(name, path): + """Loader factory for loading templates from egg package data. + + :param name: the name of the package containing the resources + :param path: the path inside the package data + :return: the loader function to load templates from the given package + :rtype: ``function`` + """ + from pkg_resources import resource_stream + def _load_from_package(filename): + filepath = os.path.join(path, filename) + return filepath, filename, resource_stream(name, filepath), None + return _load_from_package + package = staticmethod(package) + + def prefixed(**delegates): + """Factory for a load function that delegates to other loaders + depending on the prefix of the requested template path. + + The prefix is stripped from the filename when passing on the load + request to the delegate. + + >>> load = prefixed( + ... app1 = lambda filename: ('app1', filename, None, None), + ... app2 = lambda filename: ('app2', filename, None, None) + ... ) + >>> print load('app1/foo.html') + ('app1', 'app1/foo.html', None, None) + >>> print load('app2/bar.html') + ('app2', 'app2/bar.html', None, None) + + :param delegates: mapping of path prefixes to loader functions + :return: the loader function + :rtype: ``function`` + """ + def _dispatch_by_prefix(filename): + for prefix, delegate in delegates.items(): + if filename.startswith(prefix): + if isinstance(delegate, basestring): + delegate = directory(delegate) + filepath, _, fileobj, uptodate = delegate( + filename[len(prefix):].lstrip('/\\') + ) + return filepath, filename, fileobj, uptodate + raise TemplateNotFound(filename, delegates.keys()) + return _dispatch_by_prefix + prefixed = staticmethod(prefixed) + +directory = TemplateLoader.directory +package = TemplateLoader.package +prefixed = TemplateLoader.prefixed diff --git a/genshi/template/markup.py b/genshi/template/markup.py --- a/genshi/template/markup.py +++ b/genshi/template/markup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -14,23 +14,17 @@ """Markup templating engine.""" from itertools import chain -import sys -from textwrap import dedent -from genshi.core import Attrs, Namespace, Stream, StreamEventKind +from genshi.core import Attrs, Markup, Namespace, Stream, StreamEventKind from genshi.core import START, END, START_NS, END_NS, TEXT, PI, COMMENT from genshi.input import XMLParser from genshi.template.base import BadDirectiveError, Template, \ TemplateSyntaxError, _apply_directives, \ - INCLUDE, SUB + EXEC, INCLUDE, SUB from genshi.template.eval import Suite from genshi.template.interpolation import interpolate from genshi.template.directives import * - -if sys.version_info < (2, 4): - _ctxt2dict = lambda ctxt: ctxt.frames[0] -else: - _ctxt2dict = lambda ctxt: ctxt +from genshi.template.text import NewTextTemplate __all__ = ['MarkupTemplate'] __docformat__ = 'restructuredtext en' @@ -47,8 +41,6 @@
        • 1
        • 2
        • 3
        """ - EXEC = StreamEventKind('EXEC') - """Stream event kind representing a Python code suite to execute.""" DIRECTIVE_NAMESPACE = Namespace('http://genshi.edgewall.org/') XINCLUDE_NAMESPACE = Namespace('http://www.w3.org/2001/XInclude') @@ -65,17 +57,16 @@ ('content', ContentDirective), ('attrs', AttrsDirective), ('strip', StripDirective)] + serializer = 'xml' + _number_conv = Markup - def __init__(self, source, basedir=None, filename=None, loader=None, - encoding=None, lookup='lenient', allow_exec=True): - Template.__init__(self, source, basedir=basedir, filename=filename, - loader=loader, encoding=encoding, lookup=lookup, - allow_exec=allow_exec) + def _init_filters(self): + Template._init_filters(self) # Make sure the include filter comes after the match filter - if loader: + if self.loader: self.filters.remove(self._include) - self.filters += [self._exec, self._match] - if loader: + self.filters += [self._match] + if self.loader: self.filters.append(self._include) def _parse(self, source, encoding): @@ -83,8 +74,8 @@ dirmap = {} # temporary mapping of directives to elements ns_prefix = {} depth = 0 - in_fallback = 0 - include_href = None + fallbacks = [] + includes = [] if not isinstance(source, Stream): source = XMLParser(source, filename=self.filename, @@ -133,8 +124,8 @@ directives.append((cls, value, ns_prefix.copy(), pos)) else: if value: - value = list(interpolate(value, self.basedir, - pos[0], pos[1], pos[2], + value = list(interpolate(value, self.filepath, + pos[1], pos[2], lookup=self.lookup)) if len(value) == 1 and value[0][0] is TEXT: value = value[0][1] @@ -155,9 +146,11 @@ raise TemplateSyntaxError('Include misses required ' 'attribute "href"', self.filepath, *pos[1:]) + includes.append((include_href, new_attrs.get('parse'))) streams.append([]) elif tag.localname == 'fallback': - in_fallback += 1 + streams.append([]) + fallbacks.append(streams[-1]) else: stream.append((kind, (tag, new_attrs), pos)) @@ -167,12 +160,26 @@ elif kind is END: depth -= 1 - if in_fallback and data == self.XINCLUDE_NAMESPACE['fallback']: - in_fallback -= 1 + if fallbacks and data == self.XINCLUDE_NAMESPACE['fallback']: + assert streams.pop() is fallbacks[-1] elif data == self.XINCLUDE_NAMESPACE['include']: - fallback = streams.pop() + fallback = None + if len(fallbacks) == len(includes): + fallback = fallbacks.pop() + streams.pop() # discard anything between the include tags + # and the fallback element stream = streams[-1] - stream.append((INCLUDE, (include_href, fallback), pos)) + href, parse = includes.pop() + try: + cls = { + 'xml': MarkupTemplate, + 'text': NewTextTemplate + }[parse or 'xml'] + except KeyError: + raise TemplateSyntaxError('Invalid value for "parse" ' + 'attribute of include', + self.filepath, *pos[1:]) + stream.append((INCLUDE, (href, cls, fallback), pos)) else: stream.append((kind, data, pos)) @@ -191,19 +198,7 @@ raise TemplateSyntaxError('Python code blocks not allowed', self.filepath, *pos[1:]) try: - # As Expat doesn't report whitespace between the PI target - # and the data, we have to jump through some hoops here to - # get correctly indented Python code - # Unfortunately, we'll still probably not get the line - # number quite right - lines = [line.expandtabs() for line in data[1].splitlines()] - first = lines[0] - rest = dedent('\n'.join(lines[1:])).rstrip() - if first.rstrip().endswith(':') and not rest[0].isspace(): - rest = '\n'.join([' ' + line for line - in rest.splitlines()]) - source = '\n'.join([first, rest]) - suite = Suite(source, self.filepath, pos[1], + suite = Suite(data[1], self.filepath, pos[1], lookup=self.lookup) except SyntaxError, err: raise TemplateSyntaxError(err, self.filepath, @@ -212,9 +207,8 @@ stream.append((EXEC, suite, pos)) elif kind is TEXT: - for kind, data, pos in interpolate(data, self.basedir, pos[0], - pos[1], pos[2], - lookup=self.lookup): + for kind, data, pos in interpolate(data, self.filepath, pos[1], + pos[2], lookup=self.lookup): stream.append((kind, data, pos)) elif kind is COMMENT: @@ -227,17 +221,7 @@ assert len(streams) == 1 return streams[0] - def _exec(self, stream, ctxt): - """Internal stream filter that executes code in ```` - processing instructions. - """ - for event in stream: - if event[0] is EXEC: - event[1].execute(ctxt.data) - else: - yield event - - def _match(self, stream, ctxt, match_templates=None): + def _match(self, stream, ctxt, match_templates=None, **vars): """Internal stream filter that applies any defined match templates to the stream. """ @@ -268,10 +252,13 @@ yield event continue - for idx, (test, path, template, namespaces, directives) in \ - enumerate(match_templates): + for idx, (test, path, template, hints, namespaces, directives) \ + in enumerate(match_templates): if test(event, namespaces, ctxt) is True: + if 'match_once' in hints: + del match_templates[idx] + idx -= 1 # Let the remaining match templates know about the event so # they get a chance to update their internal state @@ -280,35 +267,39 @@ # Consume and store all events until an end event # corresponding to this start event is encountered - content = chain([event], - self._match(_strip(stream), ctxt, - [match_templates[idx]]), - tail) - content = list(self._include(content, ctxt)) + pre_match_templates = match_templates[:idx + 1] + if 'match_once' not in hints and 'not_recursive' in hints: + pre_match_templates.pop() + inner = _strip(stream) + if pre_match_templates: + inner = self._match(inner, ctxt, pre_match_templates) + content = self._include(chain([event], inner, tail), ctxt) + if 'not_buffered' not in hints: + content = list(content) - for test in [mt[0] for mt in match_templates]: - test(tail[0], namespaces, ctxt, updateonly=True) + if tail: + for test in [mt[0] for mt in match_templates]: + test(tail[0], namespaces, ctxt, updateonly=True) # Make the select() function available in the body of the # match template def select(path): return Stream(content).select(path, namespaces, ctxt) - ctxt.push(dict(select=select)) + vars = dict(select=select) # Recursively process the output - template = _apply_directives(template, ctxt, directives) - for event in self._match(self._eval(self._flatten(template, - ctxt), - ctxt), ctxt, - match_templates[:idx] + - match_templates[idx + 1:]): + template = _apply_directives(template, directives, ctxt, + **vars) + for event in self._match( + self._exec( + self._eval( + self._flatten(template, ctxt, **vars), + ctxt, **vars), + ctxt, **vars), + ctxt, match_templates[idx + 1:], **vars): yield event - ctxt.pop() break else: # no matches yield event - - -EXEC = MarkupTemplate.EXEC diff --git a/genshi/template/plugin.py b/genshi/template/plugin.py --- a/genshi/template/plugin.py +++ b/genshi/template/plugin.py @@ -23,7 +23,7 @@ from genshi.template.base import Template from genshi.template.loader import TemplateLoader from genshi.template.markup import MarkupTemplate -from genshi.template.text import TextTemplate +from genshi.template.text import TextTemplate, NewTextTemplate __all__ = ['ConfigurationError', 'AbstractTemplateEnginePlugin', 'MarkupTemplateEnginePlugin', 'TextTemplateEnginePlugin'] @@ -62,7 +62,7 @@ if loader_callback and not callable(loader_callback): raise ConfigurationError('loader callback must be a function') - lookup_errors = options.get('genshi.lookup_errors', 'lenient') + lookup_errors = options.get('genshi.lookup_errors', 'strict') if lookup_errors not in ('lenient', 'strict'): raise ConfigurationError('Unknown lookup errors mode "%s"' % lookup_errors) @@ -97,7 +97,7 @@ return self.loader.load(templatename) - def _get_render_options(self, format=None): + def _get_render_options(self, format=None, fragment=False): if format is None: format = self.default_format kwargs = {'method': format} @@ -107,7 +107,7 @@ def render(self, info, format=None, fragment=False, template=None): """Render the template to a string using the provided info.""" - kwargs = self._get_render_options(format=format) + kwargs = self._get_render_options(format=format, fragment=fragment) return self.transform(info, template).render(**kwargs) def transform(self, info, template): @@ -140,10 +140,10 @@ raise ConfigurationError('Unknown output format %r' % format) self.default_format = format - def _get_render_options(self, format=None): + def _get_render_options(self, format=None, fragment=False): kwargs = super(MarkupTemplateEnginePlugin, - self)._get_render_options(format) - if self.default_doctype: + self)._get_render_options(format, fragment) + if self.default_doctype and not fragment: kwargs['doctype'] = self.default_doctype return kwargs @@ -162,3 +162,15 @@ template_class = TextTemplate extension = '.txt' default_format = 'text' + + def __init__(self, extra_vars_func=None, options=None): + if options is None: + options = {} + + new_syntax = options.get('genshi.new_text_syntax') + if isinstance(new_syntax, basestring): + new_syntax = new_syntax.lower() in ('1', 'on', 'yes', 'true') + if new_syntax: + self.template_class = NewTextTemplate + + AbstractTemplateEnginePlugin.__init__(self, extra_vars_func, options) diff --git a/genshi/template/tests/directives.py b/genshi/template/tests/directives.py --- a/genshi/template/tests/directives.py +++ b/genshi/template/tests/directives.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -382,6 +382,7 @@ """) self.assertEqual(""" Hi, you! + """, str(tmpl.generate())) def test_function_with_star_args(self): @@ -482,9 +483,36 @@ try: list(tmpl.generate(foo=12)) self.fail('Expected TemplateRuntimeError') - except TemplateRuntimeError, e: + except TypeError, e: + assert (str(e) == "iteration over non-sequence" or + str(e) == "'int' object is not iterable") + exc_type, exc_value, exc_traceback = sys.exc_info() + frame = exc_traceback.tb_next + frames = [] + while frame.tb_next: + frame = frame.tb_next + frames.append(frame) + self.assertEqual("", + frames[-1].tb_frame.f_code.co_name) + self.assertEqual('test.html', + frames[-1].tb_frame.f_code.co_filename) + if sys.version_info[:2] >= (2, 4): + self.assertEqual(2, frames[-1].tb_lineno) + + def test_for_with_empty_value(self): + """ + Verify an empty 'for' value is an error + """ + try: + MarkupTemplate(""" + + empty + + """, filename='test.html') + self.fail('ExpectedTemplateSyntaxError') + except TemplateSyntaxError, e: self.assertEqual('test.html', e.filename) - if sys.version_info[:2] >= (2, 4): + if sys.version_info[:2] > (2,4): self.assertEqual(2, e.lineno) @@ -620,6 +648,32 @@ """, str(tmpl.generate())) + def test_recursive_match_3(self): + tmpl = MarkupTemplate(""" + + ${select('*|text()')} + + +
          ${select('*')}
        +         + + ${select('*|text()')} + + + + + 1 + 2 + + +         + """) + self.assertEqual(""" + +
          12
        +         +         """, str(tmpl.generate())) + def test_not_match_self(self): """ See http://genshi.edgewall.org/ticket/77 @@ -842,6 +896,54 @@ """, str(tmpl.generate())) + def test_match_with_once_attribute(self): + tmpl = MarkupTemplate(""" + +
        + ${select("*")} +
        +         + +

        Foo

        + + +

        Bar

        + + """) + self.assertEqual(""" + +
        +

        Foo

        +
        + + +

        Bar

        + + """, str(tmpl.generate())) + + def test_match_with_recursive_attribute(self): + tmpl = MarkupTemplate(""" + +
        + ${select('*')} +
        +         + + + + + +         """) + self.assertEqual(""" + +
        + + + +
        +         +         """, str(tmpl.generate())) + # FIXME #def test_match_after_step(self): # tmpl = MarkupTemplate("""
        @@ -857,6 +959,21 @@ #
        """, str(tmpl.generate())) +class ContentDirectiveTestCase(unittest.TestCase): + """Tests for the `py:content` template directive.""" + + def test_as_element(self): + try: + tmpl = MarkupTemplate(""" + Foo + """, filename='test.html') + self.fail('Expected TemplateSyntaxError') + except TemplateSyntaxError, e: + self.assertEqual('test.html', e.filename) + if sys.version_info[:2] >= (2, 4): + self.assertEqual(2, e.lineno) + + class ReplaceDirectiveTestCase(unittest.TestCase): """Tests for the `py:replace` template directive.""" @@ -875,6 +992,14 @@ if sys.version_info[:2] >= (2, 4): self.assertEqual(2, e.lineno) + def test_as_element(self): + tmpl = MarkupTemplate("""
        + +
        """, filename='test.html') + self.assertEqual("""
        + Test +
        """, str(tmpl.generate(title='Test'))) + class StripDirectiveTestCase(unittest.TestCase): """Tests for the `py:strip` template directive.""" @@ -968,6 +1093,22 @@ here are two semicolons: ;;
    """, str(tmpl.generate())) + def test_ast_transformation(self): + """ + Verify that the usual template expression AST transformations are + applied despite the code being compiled to a `Suite` object. + """ + tmpl = MarkupTemplate("""
    + + $bar + +
    """) + self.assertEqual("""
    + + 42 + +
    """, str(tmpl.generate(foo={'bar': 42}))) + def test_unicode_expr(self): tmpl = MarkupTemplate("""
    @@ -979,6 +1120,16 @@ 一二三四五六日
    """, str(tmpl.generate())) + + def test_with_empty_value(self): + """ + Verify that an empty py:with works (useless, but legal) + """ + tmpl = MarkupTemplate("""
    + Text
    """) + + self.assertEqual("""
    + Text
    """, str(tmpl.generate())) def suite(): @@ -990,6 +1141,7 @@ suite.addTest(unittest.makeSuite(ForDirectiveTestCase, 'test')) suite.addTest(unittest.makeSuite(IfDirectiveTestCase, 'test')) suite.addTest(unittest.makeSuite(MatchDirectiveTestCase, 'test')) + suite.addTest(unittest.makeSuite(ContentDirectiveTestCase, 'test')) suite.addTest(unittest.makeSuite(ReplaceDirectiveTestCase, 'test')) suite.addTest(unittest.makeSuite(StripDirectiveTestCase, 'test')) suite.addTest(unittest.makeSuite(WithDirectiveTestCase, 'test')) diff --git a/genshi/template/tests/eval.py b/genshi/template/tests/eval.py --- a/genshi/template/tests/eval.py +++ b/genshi/template/tests/eval.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -12,6 +12,8 @@ # history and logs, available at http://genshi.edgewall.org/log/. import doctest +import pickle +from StringIO import StringIO import sys import unittest @@ -32,6 +34,14 @@ self.assertEqual(hash(expr), hash(Expression('x,y'))) self.assertNotEqual(hash(expr), hash(Expression('y, x'))) + def test_pickle(self): + expr = Expression('1 < 2') + buf = StringIO() + pickle.dump(expr, buf, 2) + buf.seek(0) + unpickled = pickle.load(buf) + assert unpickled.evaluate({}) is True + def test_name_lookup(self): self.assertEqual('bar', Expression('foo').evaluate({'foo': 'bar'})) self.assertEqual(id, Expression('id').evaluate({})) @@ -321,7 +331,8 @@ self.assertEqual([0, 1, 2, 3], expr.evaluate({'numbers': range(5)})) def test_access_undefined(self): - expr = Expression("nothing", filename='index.html', lineno=50) + expr = Expression("nothing", filename='index.html', lineno=50, + lookup='lenient') retval = expr.evaluate({}) assert isinstance(retval, Undefined) self.assertEqual('nothing', retval._name) @@ -332,23 +343,45 @@ def __repr__(self): return '' something = Something() - expr = Expression('something.nil', filename='index.html', lineno=50) + expr = Expression('something.nil', filename='index.html', lineno=50, + lookup='lenient') retval = expr.evaluate({'something': something}) assert isinstance(retval, Undefined) self.assertEqual('nil', retval._name) assert retval._owner is something + def test_getattr_exception(self): + class Something(object): + def prop_a(self): + raise NotImplementedError + prop_a = property(prop_a) + def prop_b(self): + raise AttributeError + prop_b = property(prop_b) + self.assertRaises(NotImplementedError, + Expression('s.prop_a').evaluate, {'s': Something()}) + self.assertRaises(AttributeError, + Expression('s.prop_b').evaluate, {'s': Something()}) + def test_getitem_undefined_string(self): class Something(object): def __repr__(self): return '' something = Something() - expr = Expression('something["nil"]', filename='index.html', lineno=50) + expr = Expression('something["nil"]', filename='index.html', lineno=50, + lookup='lenient') retval = expr.evaluate({'something': something}) assert isinstance(retval, Undefined) self.assertEqual('nil', retval._name) assert retval._owner is something + def test_getitem_exception(self): + class Something(object): + def __getitem__(self, key): + raise NotImplementedError + self.assertRaises(NotImplementedError, + Expression('s["foo"]').evaluate, {'s': Something()}) + def test_error_access_undefined(self): expr = Expression("nothing", filename='index.html', lineno=50, lookup='strict') @@ -420,6 +453,29 @@ class SuiteTestCase(unittest.TestCase): + def test_pickle(self): + suite = Suite('foo = 42') + buf = StringIO() + pickle.dump(suite, buf, 2) + buf.seek(0) + unpickled = pickle.load(buf) + data = {} + unpickled.execute(data) + self.assertEqual(42, data['foo']) + + def test_internal_shadowing(self): + # The context itself is stored in the global execution scope of a suite + # It used to get stored under the name 'data', which meant the + # following test would fail, as the user defined 'data' variable + # shadowed the Genshi one. We now use the name '__data__' to avoid + # conflicts + suite = Suite("""data = [] +bar = foo +""") + data = {'foo': 42} + suite.execute(data) + self.assertEqual(42, data['bar']) + def test_assign(self): suite = Suite("foo = 42") data = {} @@ -434,7 +490,8 @@ self.assertEqual(None, data['donothing']()) def test_def_with_multiple_statements(self): - suite = Suite("""def donothing(): + suite = Suite(""" +def donothing(): if True: return foo """) @@ -443,6 +500,35 @@ assert 'donothing' in data self.assertEqual('bar', data['donothing']()) + def test_def_using_nonlocal(self): + suite = Suite(""" +values = [] +def add(value): + if value not in values: + values.append(value) +add('foo') +add('bar') +""") + data = {} + suite.execute(data) + self.assertEqual(['foo', 'bar'], data['values']) + + def test_def_nested(self): + suite = Suite(""" +def doit(): + values = [] + def add(value): + if value not in values: + values.append(value) + add('foo') + add('bar') + return values +x = doit() +""") + data = {} + suite.execute(data) + self.assertEqual(['foo', 'bar'], data['x']) + def test_delete(self): suite = Suite("""foo = 42 del foo @@ -457,6 +543,28 @@ suite.execute(data) assert 'plain' in data + def test_class_in_def(self): + suite = Suite(""" +def create(): + class Foobar(object): + def __str__(self): + return 'foobar' + return Foobar() +x = create() +""") + data = {} + suite.execute(data) + self.assertEqual('foobar', str(data['x'])) + + def test_class_with_methods(self): + suite = Suite("""class plain(object): + def donothing(): + pass +""") + data = {} + suite.execute(data) + assert 'plain' in data + def test_import(self): suite = Suite("from itertools import ifilter") data = {} @@ -525,6 +633,25 @@ def test_local_augmented_assign(self): Suite("x = 1; x += 42; assert x == 43").execute({}) + def test_augmented_assign_in_def(self): + d = {} + Suite("""def foo(): + i = 1 + i += 1 + return i +x = foo()""").execute(d) + self.assertEqual(2, d['x']) + + def test_augmented_assign_in_loop_in_def(self): + d = {} + Suite("""def foo(): + i = 0 + for n in range(5): + i += n + return i +x = foo()""").execute(d) + self.assertEqual(10, d['x']) + def test_assign_in_list(self): suite = Suite("[d['k']] = 'foo',; assert d['k'] == 'foo'") d = {"k": "bar"} diff --git a/genshi/template/tests/interpolation.py b/genshi/template/tests/interpolation.py --- a/genshi/template/tests/interpolation.py +++ b/genshi/template/tests/interpolation.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2007 Edgewall Software +# Copyright (C) 2007-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which diff --git a/genshi/template/tests/loader.py b/genshi/template/tests/loader.py --- a/genshi/template/tests/loader.py +++ b/genshi/template/tests/loader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -104,6 +104,34 @@
    Included
    """, tmpl.generate().render()) + def test_relative_include_samesubdir(self): + file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') + try: + file1.write("""
    Included tmpl1.html
    """) + finally: + file1.close() + + os.mkdir(os.path.join(self.dirname, 'sub')) + file2 = open(os.path.join(self.dirname, 'sub', 'tmpl1.html'), 'w') + try: + file2.write("""
    Included sub/tmpl1.html
    """) + finally: + file2.close() + + file3 = open(os.path.join(self.dirname, 'sub', 'tmpl2.html'), 'w') + try: + file3.write(""" + + """) + finally: + file3.close() + + loader = TemplateLoader([self.dirname]) + tmpl = loader.load('sub/tmpl2.html') + self.assertEqual(""" +
    Included sub/tmpl1.html
    + """, tmpl.generate().render()) + def test_relative_include_without_search_path(self): file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') try: @@ -172,6 +200,67 @@
    Included
    """, tmpl2.generate().render()) + def test_relative_absolute_template_preferred(self): + file1 = open(os.path.join(self.dirname, 'tmpl1.html'), 'w') + try: + file1.write("""
    Included
    """) + finally: + file1.close() + + os.mkdir(os.path.join(self.dirname, 'sub')) + file2 = open(os.path.join(self.dirname, 'sub', 'tmpl1.html'), 'w') + try: + file2.write("""
    Included from sub
    """) + finally: + file2.close() + + file3 = open(os.path.join(self.dirname, 'sub', 'tmpl2.html'), 'w') + try: + file3.write(""" + + """) + finally: + file3.close() + + loader = TemplateLoader() + tmpl = loader.load(os.path.abspath(os.path.join(self.dirname, 'sub', + 'tmpl2.html'))) + self.assertEqual(""" +
    Included from sub
    + """, tmpl.generate().render()) + + def test_abspath_caching(self): + abspath = os.path.join(self.dirname, 'abs') + os.mkdir(abspath) + file1 = open(os.path.join(abspath, 'tmpl1.html'), 'w') + try: + file1.write(""" + + """) + finally: + file1.close() + + file2 = open(os.path.join(abspath, 'tmpl2.html'), 'w') + try: + file2.write("""
    Included from abspath.
    """) + finally: + file2.close() + + searchpath = os.path.join(self.dirname, 'searchpath') + os.mkdir(searchpath) + file3 = open(os.path.join(searchpath, 'tmpl2.html'), 'w') + try: + file3.write("""
    Included from searchpath.
    """) + finally: + file3.close() + + loader = TemplateLoader(searchpath) + tmpl1 = loader.load(os.path.join(abspath, 'tmpl1.html')) + self.assertEqual(""" +
    Included from searchpath.
    + """, tmpl1.generate().render()) + assert 'tmpl2.html' in loader._cache + def test_load_with_default_encoding(self): f = open(os.path.join(self.dirname, 'tmpl.html'), 'w') try: @@ -219,6 +308,108 @@

    Hello, hello

    """, tmpl.generate().render()) + def test_prefix_delegation_to_directories(self): + """ + Test prefix delegation with the following layout: + + templates/foo.html + sub1/templates/tmpl1.html + sub2/templates/tmpl2.html + + Where sub1 and sub2 are prefixes, and both tmpl1.html and tmpl2.html + incldue foo.html. + """ + dir1 = os.path.join(self.dirname, 'templates') + os.mkdir(dir1) + file1 = open(os.path.join(dir1, 'foo.html'), 'w') + try: + file1.write("""
    Included foo
    """) + finally: + file1.close() + + dir2 = os.path.join(self.dirname, 'sub1', 'templates') + os.makedirs(dir2) + file2 = open(os.path.join(dir2, 'tmpl1.html'), 'w') + try: + file2.write(""" + from sub1 + """) + finally: + file2.close() + + dir3 = os.path.join(self.dirname, 'sub2', 'templates') + os.makedirs(dir3) + file3 = open(os.path.join(dir3, 'tmpl2.html'), 'w') + try: + file3.write("""
    tmpl2
    """) + finally: + file3.close() + + loader = TemplateLoader([dir1, TemplateLoader.prefixed( + sub1 = dir2, + sub2 = dir3 + )]) + tmpl = loader.load('sub1/tmpl1.html') + self.assertEqual(""" +
    Included foo
    from sub1 + """, tmpl.generate().render()) + + def test_prefix_delegation_to_directories_with_subdirs(self): + """ + Test prefix delegation with the following layout: + + templates/foo.html + sub1/templates/tmpl1.html + sub1/templates/tmpl2.html + sub1/templates/bar/tmpl3.html + + Where sub1 is a prefix, and tmpl1.html includes all the others. + """ + dir1 = os.path.join(self.dirname, 'templates') + os.mkdir(dir1) + file1 = open(os.path.join(dir1, 'foo.html'), 'w') + try: + file1.write("""
    Included foo
    """) + finally: + file1.close() + + dir2 = os.path.join(self.dirname, 'sub1', 'templates') + os.makedirs(dir2) + file2 = open(os.path.join(dir2, 'tmpl1.html'), 'w') + try: + file2.write(""" + from sub1 + from sub1 + from sub1 + """) + finally: + file2.close() + + file3 = open(os.path.join(dir2, 'tmpl2.html'), 'w') + try: + file3.write("""
    tmpl2
    """) + finally: + file3.close() + + dir3 = os.path.join(self.dirname, 'sub1', 'templates', 'bar') + os.makedirs(dir3) + file4 = open(os.path.join(dir3, 'tmpl3.html'), 'w') + try: + file4.write("""
    bar/tmpl3
    """) + finally: + file4.close() + + loader = TemplateLoader([dir1, TemplateLoader.prefixed( + sub1 = os.path.join(dir2), + sub2 = os.path.join(dir3) + )]) + tmpl = loader.load('sub1/tmpl1.html') + self.assertEqual(""" +
    Included foo
    from sub1 +
    tmpl2
    from sub1 +
    bar/tmpl3
    from sub1 + """, tmpl.generate().render()) + def suite(): suite = unittest.TestSuite() diff --git a/genshi/template/tests/markup.py b/genshi/template/tests/markup.py --- a/genshi/template/tests/markup.py +++ b/genshi/template/tests/markup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -13,6 +13,7 @@ import doctest import os +import pickle import shutil from StringIO import StringIO import sys @@ -22,7 +23,7 @@ from genshi.core import Markup from genshi.input import XML from genshi.template.base import BadDirectiveError, TemplateSyntaxError -from genshi.template.loader import TemplateLoader +from genshi.template.loader import TemplateLoader, TemplateNotFound from genshi.template.markup import MarkupTemplate @@ -39,6 +40,15 @@ tmpl = MarkupTemplate(stream) self.assertEqual(' 42 42', str(tmpl.generate(var=42))) + def test_pickle(self): + stream = XML('$var') + tmpl = MarkupTemplate(stream) + buf = StringIO() + pickle.dump(tmpl, buf, 2) + buf.seek(0) + unpickled = pickle.load(buf) + self.assertEqual('42', str(unpickled.generate(var=42))) + def test_interpolate_mixed3(self): tmpl = MarkupTemplate(' ${var} $var') self.assertEqual(' 42 42', str(tmpl.generate(var=42))) @@ -270,7 +280,7 @@ finally: shutil.rmtree(dirname) - def test_dynamic_inlude_href(self): + def test_dynamic_include_href(self): dirname = tempfile.mkdtemp(suffix='genshi_test') try: file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w') @@ -296,7 +306,7 @@ finally: shutil.rmtree(dirname) - def test_select_inluded_elements(self): + def test_select_included_elements(self): dirname = tempfile.mkdtemp(suffix='genshi_test') try: file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w') @@ -351,6 +361,23 @@ finally: shutil.rmtree(dirname) + def test_error_when_include_not_found(self): + dirname = tempfile.mkdtemp(suffix='genshi_test') + try: + file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w') + try: + file2.write(""" + + """) + finally: + file2.close() + + loader = TemplateLoader([dirname], auto_reload=True) + tmpl = loader.load('tmpl2.html') + self.assertRaises(TemplateNotFound, tmpl.generate().render) + finally: + shutil.rmtree(dirname) + def test_fallback_when_include_not_found(self): dirname = tempfile.mkdtemp(suffix='genshi_test') try: @@ -371,6 +398,26 @@ finally: shutil.rmtree(dirname) + def test_fallback_when_auto_reload_true(self): + dirname = tempfile.mkdtemp(suffix='genshi_test') + try: + file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w') + try: + file2.write(""" + + Missing + """) + finally: + file2.close() + + loader = TemplateLoader([dirname], auto_reload=True) + tmpl = loader.load('tmpl2.html') + self.assertEqual(""" + Missing + """, tmpl.generate().render()) + finally: + shutil.rmtree(dirname) + def test_include_in_fallback(self): dirname = tempfile.mkdtemp(suffix='genshi_test') try: @@ -397,7 +444,7 @@ loader = TemplateLoader([dirname]) tmpl = loader.load('tmpl3.html') self.assertEqual(""" -
    Included
    +
    Included
    """, tmpl.generate().render()) finally: shutil.rmtree(dirname) @@ -422,7 +469,36 @@ loader = TemplateLoader([dirname]) tmpl = loader.load('tmpl3.html') self.assertEqual(""" - Missing + Missing + """, tmpl.generate().render()) + finally: + shutil.rmtree(dirname) + + def test_nested_include_in_fallback(self): + dirname = tempfile.mkdtemp(suffix='genshi_test') + try: + file1 = open(os.path.join(dirname, 'tmpl2.html'), 'w') + try: + file1.write("""
    Included
    """) + finally: + file1.close() + + file2 = open(os.path.join(dirname, 'tmpl3.html'), 'w') + try: + file2.write(""" + + + + + + """) + finally: + file2.close() + + loader = TemplateLoader([dirname]) + tmpl = loader.load('tmpl3.html') + self.assertEqual(""" +
    Included
    """, tmpl.generate().render()) finally: shutil.rmtree(dirname) @@ -530,6 +606,91 @@ """) tmpl = MarkupTemplate(xml, filename='test.html', allow_exec=True) + def test_exec_in_match(self): + xml = (""" + + + ${title} + +

    moot text

    + """) + tmpl = MarkupTemplate(xml, filename='test.html', allow_exec=True) + self.assertEqual(""" + + wakka wakka wakka + + """, tmpl.generate().render()) + + def test_with_in_match(self): + xml = (""" + +

    ${select('text()')}

    + ${select('.')} +     +

    ${foo}

    + """) + tmpl = MarkupTemplate(xml, filename='test.html') + self.assertEqual(""" + +

    bar

    +

    bar

    + + """, tmpl.generate().render()) + + def test_nested_include_matches(self): + # See ticket #157 + dirname = tempfile.mkdtemp(suffix='genshi_test') + try: + file1 = open(os.path.join(dirname, 'tmpl1.html'), 'w') + try: + file1.write(""" +
    Some content.
    +""") + finally: + file1.close() + + file2 = open(os.path.join(dirname, 'tmpl2.html'), 'w') + try: + file2.write(""" + +

    Some full html document that includes file1.html

    + + +""") + finally: + file2.close() + + file3 = open(os.path.join(dirname, 'tmpl3.html'), 'w') + try: + file3.write(""" +
    + Some added stuff. + ${select('*|text()')} +
    + + +""") + finally: + file3.close() + + loader = TemplateLoader([dirname]) + tmpl = loader.load('tmpl3.html') + self.assertEqual(""" + + +

    Some full html document that includes file1.html

    +
    + Some added stuff. + Some content. +
    + + +""", tmpl.generate().render()) + finally: + shutil.rmtree(dirname) + def suite(): suite = unittest.TestSuite() diff --git a/genshi/template/tests/plugin.py b/genshi/template/tests/plugin.py --- a/genshi/template/tests/plugin.py +++ b/genshi/template/tests/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2007 Edgewall Software # Copyright (C) 2006 Matthew Good # All rights reserved. # @@ -18,7 +18,7 @@ from genshi.core import Stream from genshi.output import DocType -from genshi.template import MarkupTemplate, TextTemplate +from genshi.template import MarkupTemplate, TextTemplate, NewTextTemplate from genshi.template.plugin import ConfigurationError, \ MarkupTemplateEnginePlugin, \ TextTemplateEnginePlugin @@ -145,6 +145,23 @@ """, output) + def test_render_fragment_with_doctype(self): + plugin = MarkupTemplateEnginePlugin(options={ + 'genshi.default_doctype': 'html-strict', + }) + tmpl = plugin.load_template(PACKAGE + '.templates.test_no_doctype') + output = plugin.render({'message': 'Hello'}, template=tmpl, + fragment=True) + self.assertEqual(""" + + Test + + +

    Test

    +

    Hello

    + +""", output) + def test_helper_functions(self): plugin = MarkupTemplateEnginePlugin() tmpl = plugin.load_template(PACKAGE + '.templates.functions') @@ -185,6 +202,15 @@ }) self.assertEqual('iso-8859-15', plugin.default_encoding) + def test_init_with_new_syntax(self): + plugin = TextTemplateEnginePlugin(options={ + 'genshi.new_text_syntax': 'yes', + }) + self.assertEqual(NewTextTemplate, plugin.template_class) + tmpl = plugin.load_template(PACKAGE + '.templates.new_syntax') + output = plugin.render({'foo': True}, template=tmpl) + self.assertEqual('bar', output) + def test_load_template_from_file(self): plugin = TextTemplateEnginePlugin() tmpl = plugin.load_template(PACKAGE + '.templates.test') diff --git a/genshi/template/tests/templates/new_syntax.txt b/genshi/template/tests/templates/new_syntax.txt new file mode 100644 --- /dev/null +++ b/genshi/template/tests/templates/new_syntax.txt @@ -0,0 +1,1 @@ +{% if foo %}bar{% end %} \ No newline at end of file diff --git a/genshi/template/tests/templates/test_no_doctype.html b/genshi/template/tests/templates/test_no_doctype.html new file mode 100644 --- /dev/null +++ b/genshi/template/tests/templates/test_no_doctype.html @@ -0,0 +1,13 @@ + + + + Test + + + +

    Test

    +

    $message

    + + + diff --git a/genshi/template/tests/text.py b/genshi/template/tests/text.py --- a/genshi/template/tests/text.py +++ b/genshi/template/tests/text.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -17,11 +17,12 @@ import tempfile import unittest +from genshi.template.base import TemplateSyntaxError from genshi.template.loader import TemplateLoader -from genshi.template.text import TextTemplate +from genshi.template.text import OldTextTemplate, NewTextTemplate -class TextTemplateTestCase(unittest.TestCase): +class OldTextTemplateTestCase(unittest.TestCase): """Tests for text template processing.""" def setUp(self): @@ -31,19 +32,19 @@ shutil.rmtree(self.dirname) def test_escaping(self): - tmpl = TextTemplate('\\#escaped') + tmpl = OldTextTemplate('\\#escaped') self.assertEqual('#escaped', str(tmpl.generate())) def test_comment(self): - tmpl = TextTemplate('## a comment') + tmpl = OldTextTemplate('## a comment') self.assertEqual('', str(tmpl.generate())) def test_comment_escaping(self): - tmpl = TextTemplate('\\## escaped comment') + tmpl = OldTextTemplate('\\## escaped comment') self.assertEqual('## escaped comment', str(tmpl.generate())) def test_end_with_args(self): - tmpl = TextTemplate(""" + tmpl = OldTextTemplate(""" #if foo bar #end 'if foo'""") @@ -51,16 +52,16 @@ def test_latin1_encoded(self): text = u'$foo\xf6$bar'.encode('iso-8859-1') - tmpl = TextTemplate(text, encoding='iso-8859-1') + tmpl = OldTextTemplate(text, encoding='iso-8859-1') self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) def test_unicode_input(self): text = u'$foo\xf6$bar' - tmpl = TextTemplate(text) + tmpl = OldTextTemplate(text) self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) def test_empty_lines1(self): - tmpl = TextTemplate("""Your items: + tmpl = OldTextTemplate("""Your items: #for item in items * ${item} @@ -70,10 +71,10 @@ * 0 * 1 * 2 -""", tmpl.generate(items=range(3)).render('text')) +""", tmpl.generate(items=range(3)).render()) def test_empty_lines2(self): - tmpl = TextTemplate("""Your items: + tmpl = OldTextTemplate("""Your items: #for item in items * ${item} @@ -87,7 +88,7 @@ * 2 -""", tmpl.generate(items=range(3)).render('text')) +""", tmpl.generate(items=range(3)).render()) def test_include(self): file1 = open(os.path.join(self.dirname, 'tmpl1.txt'), 'w') @@ -105,16 +106,159 @@ file2.close() loader = TemplateLoader([self.dirname]) - tmpl = loader.load('tmpl2.txt', cls=TextTemplate) + tmpl = loader.load('tmpl2.txt', cls=OldTextTemplate) self.assertEqual("""----- Included data below this line ----- Included ----- Included data above this line -----""", tmpl.generate().render()) - + + +class NewTextTemplateTestCase(unittest.TestCase): + """Tests for text template processing.""" + + def setUp(self): + self.dirname = tempfile.mkdtemp(suffix='markup_test') + + def tearDown(self): + shutil.rmtree(self.dirname) + + def test_escaping(self): + tmpl = NewTextTemplate('\\{% escaped %}') + self.assertEqual('{% escaped %}', str(tmpl.generate())) + + def test_comment(self): + tmpl = NewTextTemplate('{# a comment #}') + self.assertEqual('', str(tmpl.generate())) + + def test_comment_escaping(self): + tmpl = NewTextTemplate('\\{# escaped comment #}') + self.assertEqual('{# escaped comment #}', str(tmpl.generate())) + + def test_end_with_args(self): + tmpl = NewTextTemplate(""" +{% if foo %} + bar +{% end 'if foo' %}""") + self.assertEqual('\n', str(tmpl.generate(foo=False))) + + def test_latin1_encoded(self): + text = u'$foo\xf6$bar'.encode('iso-8859-1') + tmpl = NewTextTemplate(text, encoding='iso-8859-1') + self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) + + def test_unicode_input(self): + text = u'$foo\xf6$bar' + tmpl = NewTextTemplate(text) + self.assertEqual(u'x\xf6y', unicode(tmpl.generate(foo='x', bar='y'))) + + def test_empty_lines1(self): + tmpl = NewTextTemplate("""Your items: + +{% for item in items %}\ + * ${item} +{% end %}""") + self.assertEqual("""Your items: + + * 0 + * 1 + * 2 +""", tmpl.generate(items=range(3)).render()) + + def test_empty_lines2(self): + tmpl = NewTextTemplate("""Your items: + +{% for item in items %}\ + * ${item} + +{% end %}""") + self.assertEqual("""Your items: + + * 0 + + * 1 + + * 2 + +""", tmpl.generate(items=range(3)).render()) + + def test_exec_with_trailing_space(self): + """ + Verify that a code block with trailing space does not cause a syntax + error (see ticket #127). + """ + NewTextTemplate(u""" + {% python + bar = 42 + $} + """) + + def test_exec_import(self): + tmpl = NewTextTemplate(u"""{% python from datetime import timedelta %} + ${timedelta(days=2)} + """) + self.assertEqual(""" + 2 days, 0:00:00 + """, str(tmpl.generate())) + + def test_exec_def(self): + tmpl = NewTextTemplate(u"""{% python + def foo(): + return 42 + %} + ${foo()} + """) + self.assertEqual(u""" + 42 + """, str(tmpl.generate())) + + def test_include(self): + file1 = open(os.path.join(self.dirname, 'tmpl1.txt'), 'w') + try: + file1.write("Included") + finally: + file1.close() + + file2 = open(os.path.join(self.dirname, 'tmpl2.txt'), 'w') + try: + file2.write("""----- Included data below this line ----- +{% include tmpl1.txt %} +----- Included data above this line -----""") + finally: + file2.close() + + loader = TemplateLoader([self.dirname]) + tmpl = loader.load('tmpl2.txt', cls=NewTextTemplate) + self.assertEqual("""----- Included data below this line ----- +Included +----- Included data above this line -----""", tmpl.generate().render()) + + def test_include_expr(self): + file1 = open(os.path.join(self.dirname, 'tmpl1.txt'), 'w') + try: + file1.write("Included") + finally: + file1.close() + + file2 = open(os.path.join(self.dirname, 'tmpl2.txt'), 'w') + try: + file2.write("""----- Included data below this line ----- + {% include ${'%s.txt' % ('tmpl1',)} %} + ----- Included data above this line -----""") + finally: + file2.close() + + loader = TemplateLoader([self.dirname]) + tmpl = loader.load('tmpl2.txt', cls=NewTextTemplate) + self.assertEqual("""----- Included data below this line ----- + Included + ----- Included data above this line -----""", tmpl.generate().render()) + + def suite(): suite = unittest.TestSuite() - suite.addTest(doctest.DocTestSuite(TextTemplate.__module__)) - suite.addTest(unittest.makeSuite(TextTemplateTestCase, 'test')) + suite.addTest(doctest.DocTestSuite(NewTextTemplate.__module__)) + suite.addTest(unittest.makeSuite(OldTextTemplateTestCase, 'test')) + suite.addTest(unittest.makeSuite(NewTextTemplateTestCase, 'test')) return suite if __name__ == '__main__': diff --git a/genshi/template/text.py b/genshi/template/text.py --- a/genshi/template/text.py +++ b/genshi/template/text.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2006-2007 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -11,23 +11,235 @@ # individuals. For the exact contribution history, see the revision # history and logs, available at http://genshi.edgewall.org/log/. -"""Plain text templating engine.""" +"""Plain text templating engine. + +This module implements two template language syntaxes, at least for a certain +transitional period. `OldTextTemplate` (aliased to just `TextTemplate`) defines +a syntax that was inspired by Cheetah/Velocity. `NewTextTemplate` on the other +hand is inspired by the syntax of the Django template language, which has more +explicit delimiting of directives, and is more flexible with regards to +white space and line breaks. + +In a future release, `OldTextTemplate` will be phased out in favor of +`NewTextTemplate`, as the names imply. Therefore the new syntax is strongly +recommended for new projects, and existing projects may want to migrate to the +new syntax to remain compatible with future Genshi releases. +""" import re -from genshi.template.base import BadDirectiveError, Template, INCLUDE, SUB +from genshi.core import TEXT +from genshi.template.base import BadDirectiveError, Template, \ + TemplateSyntaxError, EXEC, INCLUDE, SUB +from genshi.template.eval import Suite from genshi.template.directives import * -from genshi.template.directives import Directive, _apply_directives +from genshi.template.directives import Directive from genshi.template.interpolation import interpolate -__all__ = ['TextTemplate'] +__all__ = ['NewTextTemplate', 'OldTextTemplate', 'TextTemplate'] __docformat__ = 'restructuredtext en' -class TextTemplate(Template): - """Implementation of a simple text-based template engine. +class NewTextTemplate(Template): + r"""Implementation of a simple text-based template engine. This class will + replace `OldTextTemplate` in a future release. - >>> tmpl = TextTemplate('''Dear $name, + It uses a more explicit delimiting style for directives: instead of the old + style which required putting directives on separate lines that were prefixed + with a ``#`` sign, directives and commenbtsr are enclosed in delimiter pairs + (by default ``{% ... %}`` and ``{# ... #}``, respectively). + + Variable substitution uses the same interpolation syntax as for markup + languages: simple references are prefixed with a dollar sign, more complex + expression enclosed in curly braces. + + >>> tmpl = NewTextTemplate('''Dear $name, + ... + ... {# This is a comment #} + ... We have the following items for you: + ... {% for item in items %} + ... * ${'Item %d' % item} + ... {% end %} + ... ''') + >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + Dear Joe, + + + We have the following items for you: + + * Item 1 + + * Item 2 + + * Item 3 + + + + By default, no spaces or line breaks are removed. If a line break should + not be included in the output, prefix it with a backslash: + + >>> tmpl = NewTextTemplate('''Dear $name, + ... + ... {# This is a comment #}\ + ... We have the following items for you: + ... {% for item in items %}\ + ... * $item + ... {% end %}\ + ... ''') + >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + Dear Joe, + + We have the following items for you: + * 1 + * 2 + * 3 + + + Backslashes are also used to escape the start delimiter of directives and + comments: + + >>> tmpl = NewTextTemplate('''Dear $name, + ... + ... \{# This is a comment #} + ... We have the following items for you: + ... {% for item in items %}\ + ... * $item + ... {% end %}\ + ... ''') + >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() + Dear Joe, + + {# This is a comment #} + We have the following items for you: + * 1 + * 2 + * 3 + + + :since: version 0.5 + """ + directives = [('def', DefDirective), + ('when', WhenDirective), + ('otherwise', OtherwiseDirective), + ('for', ForDirective), + ('if', IfDirective), + ('choose', ChooseDirective), + ('with', WithDirective)] + serializer = 'text' + + _DIRECTIVE_RE = r'((? offset: + text = _escape_sub(_escape_repl, source[offset:start]) + for kind, data, pos in interpolate(text, self.filepath, lineno, + lookup=self.lookup): + stream.append((kind, data, pos)) + lineno += len(text.splitlines()) + + lineno += len(source[start:end].splitlines()) + command, value = mo.group(2, 3) + + if command == 'include': + pos = (self.filename, lineno, 0) + value = list(interpolate(value, self.filepath, lineno, 0, + lookup=self.lookup)) + if len(value) == 1 and value[0][0] is TEXT: + value = value[0][1] + stream.append((INCLUDE, (value, None, []), pos)) + + elif command == 'python': + if not self.allow_exec: + raise TemplateSyntaxError('Python code blocks not allowed', + self.filepath, lineno) + try: + suite = Suite(value, self.filepath, lineno, + lookup=self.lookup) + except SyntaxError, err: + raise TemplateSyntaxError(err, self.filepath, + lineno + (err.lineno or 1) - 1) + pos = (self.filename, lineno, 0) + stream.append((EXEC, suite, pos)) + + elif command == 'end': + depth -= 1 + if depth in dirmap: + directive, start_offset = dirmap.pop(depth) + substream = stream[start_offset:] + stream[start_offset:] = [(SUB, ([directive], substream), + (self.filepath, lineno, 0))] + + elif command: + cls = self._dir_by_name.get(command) + if cls is None: + raise BadDirectiveError(command) + directive = cls, value, None, (self.filepath, lineno, 0) + dirmap[depth] = (directive, len(stream)) + depth += 1 + + offset = end + + if offset < len(source): + text = _escape_sub(_escape_repl, source[offset:]) + for kind, data, pos in interpolate(text, self.filepath, lineno, + lookup=self.lookup): + stream.append((kind, data, pos)) + + return stream + + +class OldTextTemplate(Template): + """Legacy implementation of the old syntax text-based templates. This class + is provided in a transition phase for backwards compatibility. New code + should use the `NewTextTemplate` class and the improved syntax it provides. + + >>> tmpl = OldTextTemplate('''Dear $name, ... ... We have the following items for you: ... #for item in items @@ -36,7 +248,7 @@ ... ... All the best, ... Foobar''') - >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render('text') + >>> print tmpl.generate(name='Joe', items=[1, 2, 3]).render() Dear Joe, We have the following items for you: @@ -54,6 +266,7 @@ ('if', IfDirective), ('choose', ChooseDirective), ('with', WithDirective)] + serializer = 'text' _DIRECTIVE_RE = re.compile(r'(?:^[ \t]*(? offset: text = source[offset:start] - for kind, data, pos in interpolate(text, self.basedir, - self.filename, lineno, + for kind, data, pos in interpolate(text, self.filepath, lineno, lookup=self.lookup): stream.append((kind, data, pos)) lineno += len(text.splitlines()) @@ -98,7 +310,7 @@ (self.filepath, lineno, 0))] elif command == 'include': pos = (self.filename, lineno, 0) - stream.append((INCLUDE, (value.strip(), []), pos)) + stream.append((INCLUDE, (value.strip(), None, []), pos)) elif command != '#': cls = self._dir_by_name.get(command) if cls is None: @@ -111,9 +323,11 @@ if offset < len(source): text = source[offset:].replace('\\#', '#') - for kind, data, pos in interpolate(text, self.basedir, - self.filename, lineno, + for kind, data, pos in interpolate(text, self.filepath, lineno, lookup=self.lookup): stream.append((kind, data, pos)) return stream + + +TextTemplate = OldTextTemplate diff --git a/genshi/tests/core.py b/genshi/tests/core.py --- a/genshi/tests/core.py +++ b/genshi/tests/core.py @@ -14,10 +14,14 @@ import doctest import pickle from StringIO import StringIO +try: + from cStringIO import StringIO as cStringIO +except ImportError: + cStringIO = StringIO import unittest from genshi import core -from genshi.core import Markup, Namespace, QName, escape, unescape +from genshi.core import Markup, Attrs, Namespace, QName, escape, unescape from genshi.input import XML, ParseError @@ -35,6 +39,18 @@ xml = XML('
  • Über uns
    • ') self.assertEqual('
  • Über uns
    • ', xml.render(encoding='ascii')) + def test_render_output_stream_utf8(self): + xml = XML('
  • Über uns
    • ') + strio = cStringIO() + self.assertEqual(None, xml.render(out=strio)) + self.assertEqual('
  • Über uns
    • ', strio.getvalue()) + + def test_render_output_stream_unicode(self): + xml = XML('
  • Über uns
    • ') + strio = StringIO() + self.assertEqual(None, xml.render(encoding=None, out=strio)) + self.assertEqual(u'
  • Über uns
    • ', strio.getvalue()) + def test_pickle(self): xml = XML('
  • Foo
    • ') buf = StringIO() @@ -46,6 +62,10 @@ class MarkupTestCase(unittest.TestCase): + def test_new_with_encoding(self): + markup = Markup('Döner', encoding='utf-8') + self.assertEquals("", repr(markup)) + def test_repr(self): markup = Markup('foo') self.assertEquals("", repr(markup)) @@ -91,6 +111,16 @@ assert type(markup) is Markup self.assertEquals('& boo', markup) + def test_mod_mapping(self): + markup = Markup('%(foo)s') % {'foo': '&'} + assert type(markup) is Markup + self.assertEquals('&', markup) + + def test_mod_noescape(self): + markup = Markup('%(amp)s') % {'amp': Markup('&')} + assert type(markup) is Markup + self.assertEquals('&', markup) + def test_mul(self): markup = Markup('foo') * 2 assert type(markup) is Markup @@ -134,6 +164,18 @@ self.assertEquals("", repr(pickle.load(buf))) +class AttrsTestCase(unittest.TestCase): + + def test_pickle(self): + attrs = Attrs([("attr1", "foo"), ("attr2", "bar")]) + buf = StringIO() + pickle.dump(attrs, buf, 2) + buf.seek(0) + unpickled = pickle.load(buf) + self.assertEquals("Attrs([('attr1', 'foo'), ('attr2', 'bar')])", + repr(unpickled)) + + class NamespaceTestCase(unittest.TestCase): def test_pickle(self): @@ -165,12 +207,18 @@ self.assertEqual("QName(u'http://www.example.org/namespace}elem')", repr(QName('http://www.example.org/namespace}elem'))) + def test_leading_curly_brace(self): + qname = QName('{http://www.example.org/namespace}elem') + self.assertEquals('http://www.example.org/namespace', qname.namespace) + self.assertEquals('elem', qname.localname) + def suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(StreamTestCase, 'test')) suite.addTest(unittest.makeSuite(MarkupTestCase, 'test')) suite.addTest(unittest.makeSuite(NamespaceTestCase, 'test')) + suite.addTest(unittest.makeSuite(AttrsTestCase, 'test')) suite.addTest(unittest.makeSuite(QNameTestCase, 'test')) suite.addTest(doctest.DocTestSuite(core)) return suite diff --git a/genshi/tests/output.py b/genshi/tests/output.py --- a/genshi/tests/output.py +++ b/genshi/tests/output.py @@ -23,6 +23,15 @@ class XMLSerializerTestCase(unittest.TestCase): + def test_xml_serialiser_with_decl(self): + stream = Stream([(Stream.XML_DECL, ('1.0', None, -1), (None, -1, -1))]) + output = stream.render(XMLSerializer, doctype='xhtml') + self.assertEqual('\n' + '\n', + output) + def test_doctype_in_stream(self): stream = Stream([(Stream.DOCTYPE, DocType.HTML_STRICT, (None, -1, -1))]) output = stream.render(XMLSerializer) @@ -219,7 +228,7 @@ def test_xml_space(self): text = ' Do not mess \n\n with me ' output = XML(text).render(XHTMLSerializer) - self.assertEqual(text, output) + self.assertEqual(' Do not mess \n\n with me ', output) def test_empty_script(self): text = """ diff --git a/genshi/tests/path.py b/genshi/tests/path.py --- a/genshi/tests/path.py +++ b/genshi/tests/path.py @@ -76,7 +76,7 @@ path = Path('//*') self.assertEqual('', repr(path)) - self.assertEqual('', path.select(xml).render()) + self.assertEqual('', path.select(xml).render()) def test_1step_attribute(self): path = Path('@foo') diff --git a/genshi/util.py b/genshi/util.py --- a/genshi/util.py +++ b/genshi/util.py @@ -15,6 +15,11 @@ import htmlentitydefs import re +try: + set +except NameError: + from sets import ImmutableSet as frozenset + from sets import Set as set __docformat__ = 'restructuredtext en' @@ -152,7 +157,7 @@ """ retval = [] for item in items: - if isinstance(item, (list, tuple)): + if isinstance(item, (frozenset, list, set, tuple)): retval += flatten(item) else: retval.append(item) @@ -223,7 +228,7 @@ return ref return _STRIPENTITIES_RE.sub(_replace_entity, text) -_STRIPTAGS_RE = re.compile(r'<[^>]*?>') +_STRIPTAGS_RE = re.compile(r'(|<[^>]*>)') def striptags(text): """Return a copy of the text with any XML/HTML tags removed. @@ -234,6 +239,11 @@ >>> striptags('Foo
    ') 'Foo' + HTML/XML comments are stripped, too: + + >>> striptags('test') + 'test' + :param text: the string to remove tags from :return: the text with tags removed """ diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (C) 2006 Edgewall Software +# Copyright (C) 2006-2008 Edgewall Software # All rights reserved. # # This software is licensed as described in the file COPYING, which @@ -14,7 +14,7 @@ from distutils.cmd import Command from distutils.command.build_ext import build_ext -from distutils.errors import CCompilerError +from distutils.errors import CCompilerError, DistutilsPlatformError import doctest from glob import glob import os @@ -25,102 +25,33 @@ Feature = None import sys - -class build_doc(Command): - description = 'Builds the documentation' - user_options = [ - ('force', None, - "force regeneration even if no reStructuredText files have changed"), - ('without-apidocs', None, - "whether to skip the generation of API documentaton"), - ] - boolean_options = ['force', 'without-apidocs'] - - def initialize_options(self): - self.force = False - self.without_apidocs = False - - def finalize_options(self): - pass - - def run(self): - from docutils.core import publish_cmdline - from docutils.nodes import raw - from docutils.parsers import rst - - docutils_conf = os.path.join('doc', 'conf', 'docutils.ini') - epydoc_conf = os.path.join('doc', 'conf', 'epydoc.ini') - - try: - from pygments import highlight - from pygments.lexers import get_lexer_by_name - from pygments.formatters import HtmlFormatter - - def code_block(name, arguments, options, content, lineno, - content_offset, block_text, state, state_machine): - lexer = get_lexer_by_name(arguments[0]) - html = highlight('\n'.join(content), lexer, HtmlFormatter()) - return [raw('', html, format='html')] - code_block.arguments = (1, 0, 0) - code_block.options = {'language' : rst.directives.unchanged} - code_block.content = 1 - rst.directives.register_directive('code-block', code_block) - except ImportError: - print 'Pygments not installed, syntax highlighting disabled' - - for source in glob('doc/*.txt'): - dest = os.path.splitext(source)[0] + '.html' - if self.force or not os.path.exists(dest) or \ - os.path.getmtime(dest) < os.path.getmtime(source): - print 'building documentation file %s' % dest - publish_cmdline(writer_name='html', - argv=['--config=%s' % docutils_conf, source, - dest]) - - if not self.without_apidocs: - try: - from epydoc import cli - old_argv = sys.argv[1:] - sys.argv[1:] = [ - '--config=%s' % epydoc_conf, - '--no-private', # epydoc bug, not read from config - '--simple-term', - '--verbose' - ] - cli.cli() - sys.argv[1:] = old_argv - - except ImportError: - print 'epydoc not installed, skipping API documentation.' - - -class test_doc(Command): - description = 'Tests the code examples in the documentation' - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - for filename in glob('doc/*.txt'): - print 'testing documentation file %s' % filename - doctest.testfile(filename, False, optionflags=doctest.ELLIPSIS) +sys.path.append(os.path.join('doc', 'common')) +try: + from doctools import build_doc, test_doc +except ImportError: + build_doc = test_doc = None class optional_build_ext(build_ext): # This class allows C extension building to fail. + def run(self): + try: + build_ext.run(self) + except DistutilsPlatformError: + self._unavailable() + def build_extension(self, ext): try: build_ext.build_extension(self, ext) except CCompilerError, x: - print '*' * 70 - print """WARNING: + self._unavailable() + + def _unavailable(self): + print '*' * 70 + print """WARNING: An optional C extension could not be compiled, speedups will not be available.""" - print '*' * 70 + print '*' * 70 if Feature: @@ -137,7 +68,7 @@ setup( name = 'Genshi', version = '0.5', - description = 'A toolkit for stream-based generation of output for the web', + description = 'A toolkit for generation of output for the web', long_description = \ """Genshi is a Python library that provides an integrated set of components for parsing, generating, and processing HTML, XML or