genshi/mirror: genshi/filters/transform.py comparison

comparison genshi/filters/transform.py @ 744:cd6624cf2f7c trunk

Lots of `Transformer` cleanup: - Content-insertion transformations (before, after, etc.) now accept a callable. - `.prepend()` now ''only'' operates on elements. Previously it also operated on `OUTSIDE` marked events. - Where it makes sense, transformations are now ''consistently'' applied to individually selected objects in the document, rather than on any contiguous selection. This means that adjacent selected elements will be treated individually rather than as a whole. - Transformations should now consistently work on the context node. - `.substitute()` now defaults to a count of 0 (ie. all) rather than 1. This is to be consistent with Python's regex substitution. - `ATTR` events now have a `kind` of `ATTR` in addition to having this as their `mark`. - Added the `BREAK` `mark`. This allows cuts of otherwise seamlessly joined objects to be operated on. - Added a full test suite.

author	athomas
date	Mon, 09 Jun 2008 06:39:46 +0000
parents	ea2566b2f226
children	8bb31ed1072e

comparison

equal deleted inserted replaced

-:f4f8ffefbd49
+:cd6624cf2f7c
 from genshi.builder import Element
 from genshi.core import Stream, Attrs, QName, TEXT, START, END, _ensure, Markup
 from genshi.path import Path
 __all__ = ['Transformer', 'StreamBuffer', 'InjectorTransformation', 'ENTER',
-'EXIT', 'INSIDE', 'OUTSIDE']
+'EXIT', 'INSIDE', 'OUTSIDE', 'BREAK']
 class TransformMark(str):
 """A mark on a transformation stream."""
 __slots__ = []
 """Stream augmentation mark indicating a selected element attribute."""
 EXIT = TransformMark('EXIT')
 """Stream augmentation mark indicating that a selected element is being
 exited."""
+BREAK = TransformMark('BREAK')
+"""Stream augmentation mark indicating a break between two otherwise contiguous
+blocks of marked events.
+This is used primarily by the cut() transform to provide later transforms with
+an opportunity to operate on the cut buffer.
+"""
+class PushBackStream(object):
+"""Allows a single event to be pushed back onto the stream and re-consumed.
+"""
+def __init__(self, stream):
+self.stream = iter(stream)
+self.peek = None
+def push(self, event):
+assert self.peek is None
+self.peek = event
+def __iter__(self):
+while True:
+if self.peek is not None:
+peek = self.peek
+self.peek = None
+yield peek
+else:
+try:
+event = self.stream.next()
+yield event
+except StopIteration:
+if self.peek is None:
+raise
 class Transformer(object):
 """Stream filter that can apply a variety of different transformations to
 a stream.
 :param path: an XPath expression (as string) or a `Path` instance
 """
 self.transforms = [SelectTransformation(path)]
-def __call__(self, stream):
+def __call__(self, stream, keep_marks=False):
 """Apply the transform filter to the marked stream.
 :param stream: the marked event stream to filter
+:param keep_marks: Do not strip transformer selection marks from the
+stream. Useful for testing.
 :return: the transformed stream
 :rtype: `Stream`
 """
 transforms = self._mark(stream)
 for link in self.transforms:
 transforms = link(transforms)
-return Stream(self._unmark(transforms),
+if not keep_marks:
+transforms = self._unmark(transforms)
+return Stream(transforms,
 serializer=getattr(stream, 'serializer', None))
 def apply(self, function):
 """Apply a transformation to the stream.
 ...             '<body>Some <em>body</em> text.</body></html>')
 >>> print html | Transformer('.//title/text()').replace('New Title')
 <html><head><title>New Title</title></head><body>Some <em>body</em>
 text.</body></html>
-:param content: Either an iterable of events or a string to insert.
+:param content: Either a callable, an iterable of events, or a string
+to insert.
 :rtype: `Transformer`
 """
 return self.apply(ReplaceTransformation(content))
 def before(self, content):
 ...             '<body>Some <em>body</em> text.</body></html>')
 >>> print html | Transformer('.//em').before('emphasised ')
 <html><head><title>Some Title</title></head><body>Some emphasised
 <em>body</em> text.</body></html>
-:param content: Either an iterable of events or a string to insert.
+:param content: Either a callable, an iterable of events, or a string
+to insert.
 :rtype: `Transformer`
 """
 return self.apply(BeforeTransformation(content))
 def after(self, content):
 ...             '<body>Some <em>body</em> text.</body></html>')
 >>> print html | Transformer('.//em').after(' rock')
 <html><head><title>Some Title</title></head><body>Some <em>body</em>
 rock text.</body></html>
-:param content: Either an iterable of events or a string to insert.
+:param content: Either a callable, an iterable of events, or a string
+to insert.
 :rtype: `Transformer`
 """
 return self.apply(AfterTransformation(content))
 def prepend(self, content):
 ...             '<body>Some <em>body</em> text.</body></html>')
 >>> print html | Transformer('.//body').prepend('Some new body text. ')
 <html><head><title>Some Title</title></head><body>Some new body text.
 Some <em>body</em> text.</body></html>
-:param content: Either an iterable of events or a string to insert.
+:param content: Either a callable, an iterable of events, or a string
+to insert.
 :rtype: `Transformer`
 """
 return self.apply(PrependTransformation(content))
 def append(self, content):
 ...             '<body>Some <em>body</em> text.</body></html>')
 >>> print html | Transformer('.//body').append(' Some new body text.')
 <html><head><title>Some Title</title></head><body>Some <em>body</em>
 text. Some new body text.</body></html>
-:param content: Either an iterable of events or a string to insert.
+:param content: Either a callable, an iterable of events, or a string
+to insert.
 :rtype: `Transformer`
 """
 return self.apply(AppendTransformation(content))
 #{ Attribute manipulation
 #{ Buffer operations
 def copy(self, buffer, accumulate=False):
 """Copy selection into buffer.
-The buffer is replaced by each contiguous selection before being passed
+The buffer is replaced by each *contiguous* selection before being passed
 to the next transformation. If accumulate=True, further selections will
 be appended to the buffer rather than replacing it.
 >>> from genshi.builder import tag
 >>> buffer = StreamBuffer()
 :param buffer: the `StreamBuffer` in which the selection should be
 stored
 :rtype: `Transformer`
-:note: this transformation will buffer the entire input stream
+note: Copy (and cut) copy each individual selected object into the
+buffer before passing to the next transform. For example, the
+XPath ``*|text()`` will select all elements and text, each
+instance of which will be copied to the buffer individually
+before passing to the next transform. This has implications for
+how ``StreamBuffer`` objects can be used, so some
+experimentation may be required.
 """
 return self.apply(CopyTransformation(buffer, accumulate))
 def cut(self, buffer, accumulate=False):
 """Copy selection into buffer and remove the selection from the stream.
 for event in stream:
 yield OUTSIDE, event
 def _unmark(self, stream):
 for mark, event in stream:
-if event[0] is not None:
+kind = event[0]
+if not (kind is None or kind is ATTR or kind is BREAK):
 yield event
 class SelectTransformation(object):
 """Select and mark events that match an XPath expression."""
 else:
 yield OUTSIDE, event
 elif isinstance(result, Attrs):
 # XXX  Selected *attributes* are given a "kind" of None to
 # indicate they are not really part of the stream.
-yield ATTR, (None, (QName(event[1][0] + '@*'), result), event[2])
+yield ATTR, (ATTR, (QName(event[1][0] + '@*'), result), event[2])
 yield None, event
+elif isinstance(result, tuple):
+yield OUTSIDE, result
 elif result:
+# XXX Assume everything else is "text"?
 yield None, (TEXT, unicode(result), (None, -1, -1))
 else:
 yield None, event
 """Apply the transform filter to the marked stream.
 :param stream: the marked event stream to filter
 """
 for mark, event in stream:
-if mark not in (INSIDE, OUTSIDE):
+yield mark, event
-yield mark, event
+if mark is ENTER:
+for mark, event in stream:
+if mark is EXIT:
+yield mark, event
+break
 class RemoveTransformation(object):
 """Remove selection from the stream."""
 if mark:
 element = list(self.element.generate())
 for prefix in element[:-1]:
 yield None, prefix
 yield mark, event
-while True:
+start = mark
-try:
+stopped = False
-mark, event = stream.next()
+for mark, event in stream:
-except StopIteration:
+if start is ENTER and mark is EXIT:
-yield None, element[-1]
+yield mark, event
+stopped = True
+break
 if not mark:
 break
 yield mark, event
+else:
+stopped = True
 yield None, element[-1]
-yield mark, event
+if not stopped:
+yield mark, event
 else:
 yield mark, event
 class TraceTransformation(object):
 yield event
 class FilterTransformation(object):
 """Apply a normal stream filter to the selection. The filter is called once
-for each contiguous block of marked events."""
+for each selection."""
 def __init__(self, filter):
 """Create the transform.
 :param filter: The stream filter to apply.
 yield OUTSIDE, event
 del queue[:]
 queue = []
 for mark, event in stream:
-if mark:
+if mark is ENTER:
 queue.append(event)
-else:
+for mark, event in stream:
+queue.append(event)
+if mark is EXIT:
+break
 for queue_event in flush(queue):
 yield queue_event
-yield None, event
+elif mark is OUTSIDE:
-for event in flush(queue):
+stopped = True
-yield event
+queue.append(event)
+for mark, event in stream:
+if mark is not OUTSIDE:
+break
+queue.append(event)
+else:
+stopped = True
+for queue_event in flush(queue):
+yield queue_event
+if not stopped:
+yield None, event
+else:
+yield mark, event
+for queue_event in flush(queue):
+yield queue_event
 class MapTransformation(object):
 """Apply a function to the `data` element of events of ``kind`` in the
 selection.
 class SubstituteTransformation(object):
 """Replace text matching a regular expression.
 Refer to the documentation for ``re.sub()`` for details.
 """
-def __init__(self, pattern, replace, count=1):
+def __init__(self, pattern, replace, count=0):
 """Create the transform.
 :param pattern: A regular expression object, or string.
 :param replace: Replacement pattern.
 :param count: Number of replacements to make in each text fragment.
 injected.
 """
 self.content = content
 def _inject(self):
-for event in _ensure(self.content):
+content = self.content
+if callable(content):
+content = content()
+for event in _ensure(content):
 yield None, event
 class ReplaceTransformation(InjectorTransformation):
 """Replace selection with content."""
 def __call__(self, stream):
 """Apply the transform filter to the marked stream.
 :param stream: The marked event stream to filter
 """
+stream = PushBackStream(stream)
 for mark, event in stream:
 if mark is not None:
+start = mark
 for subevent in self._inject():
 yield subevent
-while True:
+for mark, event in stream:
-mark, event = stream.next()
+if start is ENTER:
-if mark is None:
+if mark is EXIT:
-yield mark, event
+break
+elif mark != start:
+stream.push((mark, event))
 break
 else:
 yield mark, event
 def __call__(self, stream):
 """Apply the transform filter to the marked stream.
 :param stream: The marked event stream to filter
 """
+stream = PushBackStream(stream)
 for mark, event in stream:
 if mark is not None:
+start = mark
 for subevent in self._inject():
 yield subevent
 yield mark, event
-while True:
+for mark, event in stream:
-mark, event = stream.next()
+if mark != start and start is not ENTER:
-if not mark:
+stream.push((mark, event))
 break
 yield mark, event
-yield mark, event
+if start is ENTER and mark is EXIT:
+break
+else:
+yield mark, event
 class AfterTransformation(InjectorTransformation):
 """Insert content after selection."""
 def __call__(self, stream):
 """Apply the transform filter to the marked stream.
 :param stream: The marked event stream to filter
 """
+stream = PushBackStream(stream)
 for mark, event in stream:
 yield mark, event
 if mark:
-while True:
+start = mark
-try:
+for mark, event in stream:
-mark, event = stream.next()
+if start is not ENTER and mark != start:
-except StopIteration:
+stream.push((mark, event))
-break
-if not mark:
 break
 yield mark, event
+if start is ENTER and mark is EXIT:
+break
 for subevent in self._inject():
 yield subevent
-yield mark, event
 class PrependTransformation(InjectorTransformation):
 """Prepend content to the inside of selected elements."""
 :param stream: The marked event stream to filter
 """
 for mark, event in stream:
 yield mark, event
-if mark in (ENTER, OUTSIDE):
+if mark is ENTER:
 for subevent in self._inject():
 yield subevent
 class AppendTransformation(InjectorTransformation):
 :param stream: The marked event stream to filter
 """
 for mark, event in stream:
 yield mark, event
 if mark is ENTER:
-while True:
+for mark, event in stream:
-mark, event = stream.next()
 if mark is EXIT:
 break
 yield mark, event
 for subevent in self._inject():
 yield subevent
 :param event: the markup event to add
 """
 self.events.append(event)
 def reset(self):
-"""Reset the buffer so that it's empty."""
+"""Empty the buffer of events."""
 del self.events[:]
 class CopyTransformation(object):
 """Copy selected events into a buffer for later insertion."""
 def __call__(self, stream):
 """Apply the transformation to the marked stream.
 :param stream: the marked event stream to filter
 """
-stream = iter(stream)
+stream = PushBackStream(stream)
 for mark, event in stream:
 if mark:
 if not self.accumulate:
 self.buffer.reset()
-events = []
+events = [(mark, event)]
-while mark:
+self.buffer.append(event)
+start = mark
+for mark, event in stream:
+if start is not ENTER and mark != start:
+stream.push((mark, event))
+break
 events.append((mark, event))
 self.buffer.append(event)
-mark, event = stream.next()
+if start is ENTER and mark is EXIT:
+break
 for i in events:
 yield i
-yield mark, event
+else:
+yield mark, event
 class CutTransformation(object):
 """Cut selected events into a buffer for later insertion and remove the
 selection.
 """Create the cut transformation.
 :param buffer: the `StreamBuffer` in which the selection should be
 stored
 """
-if not accumulate:
-buffer.reset()
 self.buffer = buffer
 self.accumulate = accumulate
 def __call__(self, stream):
 """Apply the transform filter to the marked stream.
 :param stream: the marked event stream to filter
 """
-attributes = None
+attributes = []
-stream = iter(stream)
+stream = PushBackStream(stream)
+broken = False
+if not self.accumulate:
+self.buffer.reset()
 for mark, event in stream:
 if mark:
+# Send a BREAK event if there was no other event sent between
 if not self.accumulate:
+if not broken and self.buffer:
+yield BREAK, (BREAK, None, None)
 self.buffer.reset()
-while mark:
+self.buffer.append(event)
-if mark is ATTR:
+start = mark
-attributes = [name for name, _ in data[1]]
+if mark is ATTR:
+attributes.extend([name for name, _ in event[1][1]])
+for mark, event in stream:
+if start is mark is ATTR:
+attributes.extend([name for name, _ in event[1][1]])
+# Handle non-element contiguous selection
+if start is not ENTER and mark != start:
+# Operating on the attributes of a START event
+if start is ATTR:
+kind, data, pos = event
+assert kind is START
+data = (data[0], data[1] - attributes)
+attributes = None
+stream.push((mark, (kind, data, pos)))
+else:
+stream.push((mark, event))
+break
 self.buffer.append(event)
-mark, event = stream.next()
+if start is ENTER and mark is EXIT:
-# If we've cut attributes, the associated element should START
+break
-# immediately after.
+broken = False
-if attributes:
+else:
-assert kind is START
+broken = True
-data = (data[0], data[1] - attributes)
+yield mark, event
-attributes = None
+if not broken and self.buffer:
+yield BREAK, (BREAK, None, None)
-yield mark, event

Mercurial > genshi > mirror

comparison genshi/filters/transform.py @ 744:cd6624cf2f7c trunk