From 02d81f74b5f22e932917c23785d22170db2f67e3 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 7 Apr 2007 16:31:42 +0200 Subject: [PATCH] [svn] implemented and documented jinja streaming interface --HG-- branch : trunk --- docs/src/designerdoc.txt | 19 +++++++ docs/src/index.txt | 2 + docs/src/objects.txt | 14 +++++ docs/src/streaming.txt | 100 ++++++++++++++++++++++++++++++++++++ jinja/datastructure.py | 57 ++++++++++++++++++++ jinja/defaults.py | 5 +- jinja/environment.py | 4 +- jinja/translators/python.py | 26 +++++++++- jinja/utils.py | 11 +++- 9 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 docs/src/streaming.txt diff --git a/docs/src/designerdoc.txt b/docs/src/designerdoc.txt index 3bb5719..a705e68 100644 --- a/docs/src/designerdoc.txt +++ b/docs/src/designerdoc.txt @@ -208,6 +208,25 @@ available per default: *new in Jinja 1.1* +`flush` + + Jinja 1.1 includes a new stream interface which can be used to speed + up the generation of big templates by streaming them to the client. + Per default Jinja flushes after 40 actions automatically. If you + want to force a flush at a special point you can use `flush()` to + do so. (eg, end of a loop iteration etc) + + .. sourcecode:: jinja + + {% for items in seq %} + {% for item in items %} +
  • {{ item|e }}
    • + {% endfor %} + {{ flush() }} + {% endfor %} + + *new in Jinja 1.1* + Loops ===== diff --git a/docs/src/index.txt b/docs/src/index.txt index f56f0e7..3a81e17 100644 --- a/docs/src/index.txt +++ b/docs/src/index.txt @@ -18,6 +18,8 @@ Welcome in the Jinja documentation. - `Global Objects `_ + - `Streaming Interface `_ + - `Context and Environment `_ - `Translators `_ diff --git a/docs/src/objects.txt b/docs/src/objects.txt index 5d0bddc..bb03f0e 100644 --- a/docs/src/objects.txt +++ b/docs/src/objects.txt @@ -109,6 +109,20 @@ access. You can mark both attributes and methods as unsafe: delete.jinja_unsafe_call = True +Bypassing Automatic Filtering +============================= + +With Jinja 1.1 it's possible to assign filters to any variable printed. Of +course there are ways to bypass that in order to make sure an object does +not get escaped etc. + +In order to disable automatic filtering for an object you have to make +sure that it's either an subclass of `unicode` or implements a +`__unicode__` method. `__str__` will work too as long as the return value +only contains ASCII values. Additionally you have to add an attribute to +that object named `jinja_no_finalization` and set that to `True`. + + .. _Filters: filters.txt .. _Tests: tests.txt .. _context object: contextenv.txt diff --git a/docs/src/streaming.txt b/docs/src/streaming.txt new file mode 100644 index 0000000..ef91b0e --- /dev/null +++ b/docs/src/streaming.txt @@ -0,0 +1,100 @@ +=================== +Streaming Interface +=================== + +With Jinja 1.1 onwards it's possible to stream the template output. This is +usually a bad idea because it's slower than `render()` but there are some +situations where it's useful. + +If you for example generate a file with a couple of megabytes you may want +to pass the stream to the WSGI interface in order to keep the amount of +memory used low and deliver the output to the browser as fast as possible. + +The streaming works quite simple. Whenever an item is returned by the +internal generator it's passed to a `TemplateStream` which buffers 40 events +before yielding them. Because WSGI usually slows down on too many flushings +this is the integrated solution for this problem. The template designer +can further control that behavior by placing a ``{{ flush() }}`` call in +the template. Whenever the `TemplateStream` encounters an object marked as +flushable it stops buffering and yields the item. + +Note that some constructs are not part of the stream. For example blocks +that are rendered using ``{{ super() }}`` are yielded as one flushable item. + +This restriction exists because template designers may want to operate on +the return value of a macro, block etc. + +The following constructs yield their subcontents as one event: ``blocks`` +that are not the current one (eg, blocks called using ``{{ super() }}``), +macros, filtered sections. + +The TemplateStream +================== + +You can get a new `TemplateStream` by calling the `stream()` function on +a template like you would do for rendering. The `TemplateStream` behaves +like a normal generator/iterator. However, as long as the stream is not +started you can modify the buffer threshold. Once the stream started +streaming it looses the `threshold` attribute. + +Explicit flushing: + +.. sourcecode:: pycon + + >>> tmpl = evironment.from_string("
      {% for item in seq %}" + ... "
    • {{ item }}
      • {{ flush() }}{% endfor %}
    ") + >>> s = tmpl.stream(seq=range(3)) + >>> s.next() + u'
    • 0
      • ' + >>> s.next() + u'
    • 1
      • ' + >>> s.next() + u'
    • 2
      • ' + >>> s.next() + u'
    ' + >>> + +Implicit flushing after 6 events: + +.. sourcecode:: pycon + + >>> tmpl = environment.from_string("
      {% for item in seq %}" + ... "
    • {{ item }}
      • {% endfor %}
    ") + >>> s = tmpl.stream(seq=range(6)) + >>> s.threshold = 6 + >>> s.next() + u'
    • 0
    • 1' + >>> s.next() + u'
    • 2
    • 3' + >>> s.next() + u'
    • 4
    • 5' + >>> s.next() + u'
    ' + +General `TemplateStream` behavior: + +.. sourcecode:: pycon + + >>> s = tmpl.stream(seq=range(6)) + >>> s.started + False + >>> s.threshold + 40 + >>> s.next() + u'
    • 0
    • 1
    • 2
    • 3
    • 4
    • 5
    ' + >>> s.started + True + >>> s.threshold + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'TemplateStream' object has no attribute 'threshold' + + +Stream Control +============== + +The stream control is designed so that it's completely transparent. When used +in non stream mode the invisible flush tokens disappear. In order to flush +the stream after calling a specific function all you have to do is to wrap +the return value in a `jinja.datastructure.Flush` object. This however +bypasses the automatic filtering system and converts the value to unicode. diff --git a/jinja/datastructure.py b/jinja/datastructure.py index 85186d2..e59cda9 100644 --- a/jinja/datastructure.py +++ b/jinja/datastructure.py @@ -172,6 +172,14 @@ class TemplateData(Markup): """ +class Flush(TemplateData): + """ + After a string marked as Flush the stream will stop buffering. + """ + + jinja_no_finalization = True + + class Context(object): """ Dict like object. @@ -458,3 +466,52 @@ class TokenStream(object): def push(self, lineno, token, data): """Push an yielded token back to the stream.""" self._pushed.append((lineno, token, data)) + + +class TemplateStream(object): + """ + Pass it a template generator and it will buffer a few items + before yielding them as one item. Useful when working with WSGI + because a Jinja template yields far too many items per default. + + The `TemplateStream` class looks for the invisble `Flush` + markers sent by the template to find out when it should stop + buffering. + """ + + def __init__(self, gen): + self._next = gen.next + self._threshold = None + self.threshold = 40 + + def __iter__(self): + return self + + def started(self): + return self._threshold is not None + started = property(started) + + def next(self): + if self._threshold is None: + self._threshold = t = self.threshold + del self.threshold + else: + t = self._threshold + buf = [] + size = 0 + push = buf.append + next = self._next + + try: + while True: + item = next() + if item: + push(item) + size += 1 + if (size and item.__class__ is Flush) or size >= t: + raise StopIteration() + except StopIteration: + pass + if not size: + raise StopIteration() + return u''.join(buf) diff --git a/jinja/defaults.py b/jinja/defaults.py index 320a54f..7d13c60 100644 --- a/jinja/defaults.py +++ b/jinja/defaults.py @@ -11,7 +11,7 @@ from jinja.filters import FILTERS as DEFAULT_FILTERS from jinja.tests import TESTS as DEFAULT_TESTS from jinja.utils import debug_context, safe_range, generate_lorem_ipsum, \ - watch_changes + watch_changes, flush __all__ = ['DEFAULT_FILTERS', 'DEFAULT_TESTS', 'DEFAULT_NAMESPACE'] @@ -21,5 +21,6 @@ DEFAULT_NAMESPACE = { 'range': safe_range, 'debug': debug_context, 'lipsum': generate_lorem_ipsum, - 'watchchanges': watch_changes + 'watchchanges': watch_changes, + 'flush': flush } diff --git a/jinja/environment.py b/jinja/environment.py index b390c01..64c0d4a 100644 --- a/jinja/environment.py +++ b/jinja/environment.py @@ -24,7 +24,7 @@ __all__ = ['Environment'] class Environment(object): """ - The jinja environment. + The Jinja environment. The core component of Jinja is the `Environment`. It contains important shared variables like configuration, filters, tests, @@ -320,6 +320,8 @@ class Environment(object): """ if value is Undefined or value is None: return u'' + elif getattr(value, 'jinja_no_finalization', False): + return value val = self.to_unicode(value) if self.default_filters: val = self.apply_filters(val, ctx, self.default_filters) diff --git a/jinja/translators/python.py b/jinja/translators/python.py index ede4737..67c2d9a 100644 --- a/jinja/translators/python.py +++ b/jinja/translators/python.py @@ -27,10 +27,18 @@ from jinja.nodes import get_nodes from jinja.parser import Parser from jinja.exceptions import TemplateSyntaxError from jinja.translators import Translator +from jinja.datastructure import TemplateStream from jinja.utils import translate_exception, capture_generator, \ RUNTIME_EXCEPTION_OFFSET +try: + GeneratorExit +except NameError: + class GeneratorExit(Exception): + pass + + #: regular expression for the debug symbols _debug_re = re.compile(r'^\s*\# DEBUG\(filename=(?P.*?), ' r'lineno=(?P\d+)\)$') @@ -83,6 +91,14 @@ class Template(object): def render(self, *args, **kwargs): """Render a template.""" + return capture_generator(self._generate(*args, **kwargs)) + + def stream(self, *args, **kwargs): + """Render a template as stream.""" + return TemplateStream(self._generate(*args, **kwargs)) + + def _generate(self, *args, **kwargs): + """Template generation helper""" # if there is no generation function we execute the code # in a new namespace and save the generation function and # debug information. @@ -93,7 +109,8 @@ class Template(object): self._debug_info = ns['debug_info'] ctx = self.environment.context_class(self.environment, *args, **kwargs) try: - return capture_generator(self.generate_func(ctx)) + for item in self.generate_func(ctx): + yield item except: if not self.environment.friendly_traceback: raise @@ -103,6 +120,13 @@ class Template(object): # or two (python2.4 and lower)). After that we call a function # that creates a new traceback that is easier to debug. exc_type, exc_value, traceback = sys.exc_info() + + # if an exception is a GeneratorExit we just reraise it. If we + # run on top of python2.3 or python2.4 a fake GeneratorExit + # class is added for this module so that we don't get a NameError + if exc_type is GeneratorExit: + raise + for _ in xrange(RUNTIME_EXCEPTION_OFFSET): traceback = traceback.tb_next traceback = translate_exception(self, exc_type, exc_value, diff --git a/jinja/utils.py b/jinja/utils.py index b3dce7b..45b19d1 100644 --- a/jinja/utils.py +++ b/jinja/utils.py @@ -18,7 +18,7 @@ import cgi from types import MethodType, FunctionType from compiler.ast import CallFunc, Name, Const from jinja.nodes import Trans -from jinja.datastructure import Context, TemplateData +from jinja.datastructure import Context, Flush from jinja.exceptions import SecurityException, TemplateNotFound #: the python2.4 version of deque is missing the remove method @@ -200,6 +200,13 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): return u'\n'.join([u'

    %s

    ' % escape(x) for x in result]) +def flush(): + """ + Yield a flush marker. + """ + return Flush() + + def watch_changes(env, context, iterable, *attributes): """ Wise replacement for ``{% ifchanged %}``. @@ -255,7 +262,7 @@ def buffereater(f): (macros, filter sections etc) """ def wrapped(*args, **kwargs): - return TemplateData(capture_generator(f(*args, **kwargs))) + return Flush(capture_generator(f(*args, **kwargs))) return wrapped -- 2.26.2