From ee2c18ef16ceb0f100c5c8211002cc9c309f2944 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Fri, 20 Apr 2007 22:39:04 +0200 Subject: [PATCH] [svn] added c implementation of the jinja context class. --HG-- branch : trunk --- CHANGES | 6 +- THANKS | 8 + TODO | 7 +- docs/src/contextenv.txt | 173 ---------------- docs/src/designerdoc.txt | 2 +- docs/src/devintro.txt | 5 + docs/src/index.txt | 49 +++-- docs/src/objects.txt | 2 + docs/src/recipies.txt | 13 ++ docs/src/translators.txt | 6 +- jinja/_native.py | 91 +++++++++ jinja/_speedups.c | 392 ++++++++++++++++++++++++++++++++++++ jinja/datastructure.py | 111 +++------- jinja/environment.py | 13 +- jinja/lexer.py | 4 +- jinja/nodes.py | 24 ++- jinja/parser.py | 47 ++++- jinja/translators/python.py | 113 ++++++++--- setup.py | 31 ++- tests/conftest.py | 2 +- tests/test_macros.py | 3 +- tests/test_various.py | 10 + 22 files changed, 791 insertions(+), 321 deletions(-) create mode 100644 THANKS delete mode 100644 docs/src/contextenv.txt create mode 100644 jinja/_native.py create mode 100644 jinja/_speedups.c diff --git a/CHANGES b/CHANGES index de01f65..9e5e64c 100644 --- a/CHANGES +++ b/CHANGES @@ -22,7 +22,8 @@ Version 1.1 - some small bugfixes. -- improved security system regarding function calls. +- improved security system regarding function calls and variable + assignment in for loops. - added `lipsum` function to generate random text. @@ -64,6 +65,9 @@ Version 1.1 - fixed a bug in the parser that didn't unescape keyword arguments. (thanks to Alexey Melchakov for reporting) +- You can now use the environment to just tokenize a template. This can + be useful for syntax highlighting or other purposes. + Version 1.0 ----------- diff --git a/THANKS b/THANKS new file mode 100644 index 0000000..e9d865c --- /dev/null +++ b/THANKS @@ -0,0 +1,8 @@ +Thanks To +========= + +All the people listed here helped improving Jinja a lot, provided +patches, helped working out solutions etc. Thanks to all of you! + +- Axel Böhm +- Alexey Melchakov diff --git a/TODO b/TODO index e77d7c8..1659c39 100644 --- a/TODO +++ b/TODO @@ -2,7 +2,8 @@ TODO List for Jinja =================== -- Requirements in Jinja (blocks and set directives) outside of renderable - blocks should become part of the module not the generate function. - - Improve the context lookup (maybe with an optional C extension) + +- `include` and `extends` should work with dynamic data too. In order to + support this the blocks should be stored as importable functions in the + generated source. diff --git a/docs/src/contextenv.txt b/docs/src/contextenv.txt deleted file mode 100644 index 6257d10..0000000 --- a/docs/src/contextenv.txt +++ /dev/null @@ -1,173 +0,0 @@ -======================= -Context and Environment -======================= - -The two central objects in Jinja are the `Environment` and `Context`. Both -are designed to be subclassed by applications if they need to extend Jinja. - -Environment -=========== - -The initialization parameters are already covered in the `Quickstart`_ thus -not repeated here. - -But beside those configurable instance variables there are some functions used -in the template evaluation code you may want to override: - -**def** `to_unicode` *(self, value)*: - - Called to convert variables to unicode. Per default this checks if the - value is already unicode. If not it's converted to unicode using the - charset defined on the environment. - - Also `None` is converted into an empty string per default. - -**def** `get_translator` *(self, context)*: - - Return the translator used for i18n. A translator is an object that - provides the two functions ``gettext(string)`` and - ``ngettext(singular, plural, n)``. Both of those functions have to - behave like the `ugettext` and `nugettext` functions described in the - python `gettext documentation`_. - - If you don't provide a translator a default one is used to switch - between singular and plural forms. - - Have a look at the `i18n`_ section for more information. - -**def** `get_translations` *(self, name)*: - - Get the translations for the template `name`. Only works if a loader - is present. See the `i18n`_ section for more details. - -**def** `get_translations_for_string` *(self, string)*: - - Get the translations for the string `string`. This works also if no - loader is present and can be used to lookup translation strings from - templates that are loaded from dynamic resources like databases. - -**def** `apply_filters` *(self, value, context, filters)*: - - Now this function is a bit tricky and you usually don't have to override - it. It's used to apply filters on a value. The Jinja expression - ``{{ foo|escape|replace('a', 'b') }}`` calls the function with the - value of `foo` as first parameter, the current context as second and - a list of filters as third. The list looks like this: - - .. sourcecode:: python - - [('escape', ()), ('replace', (u'a', u'b'))] - - As you can see the filter `escape` is called without arguments whereas - `replace` is called with the two literal strings ``a`` and ``b``, both - unicode. The filters for the names are stored on ``self.filters`` in a - dict. Missing filters should raise a `FilterNotFound` exception. - - **Warning** this is a Jinja internal method. The actual implementation - and function signature might change. - -**def** `perform_test` *(self, context, testname, args, value, invert)*: - - Like `apply_filters` you usually don't override this one. It's the - callback function for tests (``foo is bar`` / ``foo is not bar``). - - The first parameter is the current contex, the second the name of - the test to perform. the third a tuple of arguments, the fourth is - the value to test. The last one is `True` if the test was performed - with the `is not` operator, `False` if with the `is` operator. - - Missing tests should raise a `TestNotFound` exception. - - **Warning** this is a Jinja internal method. The actual implementation - and function signature might change. - -**def** `get_attribute` *(self, obj, attribute)*: - - Get `attribute` from the object provided. The default implementation - performs security tests. - - **Warning** this is a Jinja internal method. The actual implementation - and function signature might change. - -**def** `get_attributes` *(self, obj, attributes)*: - - Get some attributes from the object. If `attributes` is an empty - sequence the object itself is returned unchanged. - -**def** `call_function` *(self, f, context, args, kwargs, dyn_args, dyn_kwargs)*: - - Call a function `f` with the arguments `args`, `kwargs`, `dyn_args` and - `dyn_kwargs` where `args` is a tuple and `kwargs` a dict. If `dyn_args` - is not `None` you have to add it to the arguments, if `dyn_kwargs` is - not `None` you have to update the `kwargs` with it. - - The default implementation performs some security checks. - - **Warning** this is a Jinja internal method. The actual implementation - and function signature might change. - -**def** `call_function_simple` *(self, f, context)*: - - Like `call_function` but without arguments. - - **Warning** this is a Jinja internal method. The actual implementation - and function signature might change. - -**def** `finish_var` *(self, value, ctx)*: - - Postprocess a variable before it's sent to the template. - - **Warning** this is a Jinja internal method. The actual implementation - and function signature might change. - -.. admonition:: Note - - The Enviornment class is defined in `jinja.environment.Environment` - but imported into the `jinja` package because it's often used. - -Context -======= - -Jinja wraps the variables passed to the template in a special class called a -context. This context supports variables on multiple layers and lazy (deferred) -objects. Often your application has a request object, database connection -object or something similar you want to access in filters, functions etc. - -Beacause of that you can easily subclass a context to add additional variables -or to change the way it behaves. - -**def** `pop` *(self)*: - - Pop the outermost layer and return it. - -**def** `push` *(self, data=None)*: - - Push a dict to the stack or an empty layer. - - Has to return the pushed object. - -**def** `to_dict` *(self)*: - - Flatten the context and convert it into a dict. - -**def** `__getitem__` *(self, name)*: - - Resolve an item. Per default this also resolves `Deferred` objects. - -**def** `__setitem__` *(self, name, value)*: - - Set an item in the outermost layer. - -**def** `__delitem__` *(self, name)*: - - Delete an item in the outermost layer. Do not raise exceptions if - the value does not exist. - -**def** `__contains__` *(self, name)*: - - Return `True` if `name` exists in the context. - - -.. _i18n: i18n.txt -.. _Quickstart: devintro.txt -.. _gettext documentation: http://docs.python.org/lib/module-gettext.html diff --git a/docs/src/designerdoc.txt b/docs/src/designerdoc.txt index dcdb5ab..3040e14 100644 --- a/docs/src/designerdoc.txt +++ b/docs/src/designerdoc.txt @@ -784,7 +784,7 @@ The following keywords exist and cannot be used as identifiers: `and`, `block`, `cycle`, `elif`, `else`, `endblock`, `endfilter`, `endfor`, `endif`, `endmacro`, `endraw`, `endtrans`, `extends`, `filter`, `for`, `if`, `in`, `include`, `is`, `macro`, `not`, `or`, `pluralize`, - `raw`, `recursive`, `set`, `trans` + `print`, `raw`, `recursive`, `set`, `trans` If you want to use such a name you have to prefix or suffix it or use alternative names: diff --git a/docs/src/devintro.txt b/docs/src/devintro.txt index 11cf2f0..bcfb046 100644 --- a/docs/src/devintro.txt +++ b/docs/src/devintro.txt @@ -57,6 +57,11 @@ addition to the initialization values: syntax tree. This tree of nodes is used by the `translators`_ to convert the template into executable source- or bytecode. +``lex(source, filename)`` Tokenize the given sourcecode and return a + generator of tuples in the form + ``(lineno, token, value)``. The filename is just + used in the exceptions raised. + **New in Jinja 1.1** ``from_string(source)`` Load and parse a template source and translate it into eval-able Python code. This code is wrapped within a `Template` class that allows you to diff --git a/docs/src/index.txt b/docs/src/index.txt index f911449..d591fd1 100644 --- a/docs/src/index.txt +++ b/docs/src/index.txt @@ -6,41 +6,54 @@ Welcome in the Jinja documentation. - `Installing Jinja `_ -- Application Developer Documentation: +- **Application Developer Documentation**: - - `Quickstart `_ + - `Quickstart `_ - getting started with Jinja - - `Template Loaders `_ + - `Template Loaders `_ - documentation for the different + loader types and how to write custom ones. - - `Filter Functions `_ + - `Filter Functions `_ - information about how to write + custom filter functions. - - `Test Functions `_ + - `Test Functions `_ - information about how to write + custom test functions. - - `Global Objects `_ + - `Global Objects `_ - information about the special global + namespace in Jinja templates. - - `Streaming Interface `_ + - `Streaming Interface `_ - using Jinja for big templates + by streaming the output. - - `Context and Environment `_ + - `Internationalization `_ - how to internationalize applications + using Jinja templates. - - `Translators `_ + - `Alternative Syntax `_ - changing the default Jinja + block / variable / comment delimiters. - - `Framework Integration `_ + - `API Documentation `_ - API documentation for public Jinja + objects like `Environment`. - - `Debugging Support `_ + - `Translators `_ - explanation about the Jinja template + translation interface. - - `Internationalization `_ + - `Framework Integration `_ - integrating Jinja into + python frameworks. - - `Alternative Syntax `_ + - `Debugging Support `_ - debugging Jinja templates. - - `Developer Recipies `_ + - `Developer Recipies `_ - tips and tricks for application + developers. -- Template Designer Documentation: +- **Template Designer Documentation**: - - `Syntax Reference `_ + - `Syntax Reference `_ - the designer documentation. Put + this under your pillow. - - `Differences To Django `_ + - `Differences To Django `_ - coming from django? Then this + document is for you. - - `Designer Recipies `_ + - `Designer Recipies `_ - various tips and tricks for designers. - `Changelog `_ diff --git a/docs/src/objects.txt b/docs/src/objects.txt index bb03f0e..6d9add9 100644 --- a/docs/src/objects.txt +++ b/docs/src/objects.txt @@ -74,6 +74,8 @@ The function is always called with the same arguments. The first one is the current environment, the second the context and the third is the name of the variable. In this example ``recent_comments``. +The value is cached until rendering/streaming finished. + Unsafe Methods / Attributes =========================== diff --git a/docs/src/recipies.txt b/docs/src/recipies.txt index f8fa284..08ce8bb 100644 --- a/docs/src/recipies.txt +++ b/docs/src/recipies.txt @@ -169,3 +169,16 @@ Or if you use the `capture` filter in `clean` mode:
{{ title }}
+ + +Vim Syntax Highlighting +======================= + +Because of the similar syntax to django you can use the django highlighting +plugin for jinja too. There is however a Jinja syntax highlighting plugin +too which supports all of the syntax elements. + +You can download it from the vim webpage: `jinja.vim`_ + + +.. _jinja.vim: http://www.vim.org/scripts/script.php?script_id=1856 diff --git a/docs/src/translators.txt b/docs/src/translators.txt index 2522152..dd77746 100644 --- a/docs/src/translators.txt +++ b/docs/src/translators.txt @@ -5,6 +5,6 @@ Translators Jinja translates the template sourcecode into executable python code behind the secenes. This is done by the python translator which is currently the only shipped translator. Because the interface isn't stable it's also not -recommended yet to write other translators. However for the next Jinja version -a JavaScript translator is planned which allows you to translate Jinja -templates into executable JavaScript code. +recommended yet to write other translators. However for one of the next Jinja +versions a JavaScript translator is planned which allows you to translate +Jinja templates into executable JavaScript code. diff --git a/jinja/_native.py b/jinja/_native.py new file mode 100644 index 0000000..c5f5786 --- /dev/null +++ b/jinja/_native.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" + jinja._native + ~~~~~~~~~~~~~ + + This module implements the native base classes in case of not + having a jinja with the _speedups module compiled. + + :copyright: 2007 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +from jinja.datastructure import Deferred, Undefined + + +class BaseContext(object): + + def __init__(self, silent, globals, initial): + self.silent = silent + self.current = current = {} + self.stack = [globals, initial, current] + self.globals = globals + self.initial = initial + + def pop(self): + """ + Pop the last layer from the stack and return it. + """ + rv = self.stack.pop() + self.current = self.stack[-1] + return rv + + def push(self, data=None): + """ + Push a new dict or empty layer to the stack and return that layer + """ + data = data or {} + self.stack.append(data) + self.current = self.stack[-1] + return data + + def __getitem__(self, name): + """ + Resolve one item. Restrict the access to internal variables + such as ``'::cycle1'``. Resolve deferreds. + """ + if not name.startswith('::'): + # because the stack is usually quite small we better + # use [::-1] which is faster than reversed() in such + # a situation. + for d in self.stack[::-1]: + if name in d: + rv = d[name] + if rv.__class__ is Deferred: + rv = rv(self, name) + # never touch the globals! + if d is self.globals: + self.initial[name] = rv + else: + d[name] = rv + return rv + if self.silent: + return Undefined + raise TemplateRuntimeError('%r is not defined' % name) + + def __setitem__(self, name, value): + """ + Set a variable in the outermost layer. + """ + self.current[name] = value + + def __delitem__(self, name): + """ + Delete an variable in the outermost layer. + """ + if name in self.current: + del self.current[name] + + def __contains__(self, name): + """ + Check if the context contains a given variable. + """ + for layer in self.stack: + if name in layer: + return True + return False + + def __len__(self): + """ + Size of the stack. + """ + return len(self.stack) diff --git a/jinja/_speedups.c b/jinja/_speedups.c new file mode 100644 index 0000000..2f654d7 --- /dev/null +++ b/jinja/_speedups.c @@ -0,0 +1,392 @@ +/** + * jinja._speedups + * ~~~~~~~~~~~~~~~ + * + * This module implements the BaseContext, a c implementation of the + * Context baseclass. If this extension is not compiled the datastructure + * module implements a class in python. + * + * :copyright: 2007 by Armin Ronacher. + * :license: BSD, see LICENSE for more details. + */ + +#include +#include + +static PyObject *Undefined, *TemplateRuntimeError; +static PyTypeObject *DeferredType; + +struct StackLayer { + PyObject *dict; /* current value, a dict */ + struct StackLayer *prev; /* lower struct layer or NULL */ +}; + +typedef struct { + PyObject_HEAD + struct StackLayer *globals; /* the dict for the globals */ + struct StackLayer *initial; /* initial values */ + struct StackLayer *current; /* current values */ + long stacksize; /* current size of the stack */ + int silent; /* boolean value for silent failure */ +} BaseContext; + +static int +init_constants(void) +{ + PyObject *datastructure = PyImport_ImportModule("jinja.datastructure"); + if (!datastructure) + return 0; + PyObject *exceptions = PyImport_ImportModule("jinja.exceptions"); + if (!exceptions) { + Py_DECREF(datastructure); + return 0; + } + Undefined = PyObject_GetAttrString(datastructure, "Undefined"); + PyObject *deferred = PyObject_GetAttrString(datastructure, "Deferred"); + DeferredType = deferred->ob_type; + TemplateRuntimeError = PyObject_GetAttrString(exceptions, "TemplateRuntimeError"); + Py_DECREF(datastructure); + Py_DECREF(exceptions); + return 1; +} + +static void +BaseContext_dealloc(BaseContext *self) +{ + struct StackLayer *current = self->current, *tmp; + while (current) { + tmp = current; + Py_XDECREF(current->dict); + current->dict = NULL; + current = tmp->prev; + PyMem_Free(tmp); + } + self->ob_type->tp_free((PyObject*)self); +} + +static int +BaseContext_init(BaseContext *self, PyObject *args, PyObject *kwds) +{ + PyObject *silent = NULL, *globals = NULL, *initial = NULL; + + static char *kwlist[] = {"silent", "globals", "initial", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOO", kwlist, + &silent, &globals, &initial)) + return -1; + if (!PyDict_Check(globals) || !PyDict_Check(initial)) { + PyErr_SetString(PyExc_TypeError, "stack layers must be a dicts."); + return -1; + } + + self->silent = PyObject_IsTrue(silent); + + self->globals = PyMem_Malloc(sizeof(struct StackLayer)); + self->globals->dict = globals; + Py_INCREF(globals); + self->globals->prev = NULL; + + self->initial = PyMem_Malloc(sizeof(struct StackLayer)); + self->initial->dict = initial; + Py_INCREF(initial); + self->initial->prev = self->globals; + + self->current = PyMem_Malloc(sizeof(struct StackLayer)); + self->current->dict = PyDict_New(); + if (!self->current->dict) + return -1; + Py_INCREF(self->current->dict); + self->current->prev = self->initial; + + self->stacksize = 3; + return 0; +} + +static PyObject* +BaseContext_pop(BaseContext *self) +{ + if (self->stacksize <= 3) { + PyErr_SetString(PyExc_IndexError, "stack too small."); + return NULL; + } + PyObject *result = self->current->dict; + struct StackLayer *tmp = self->current; + self->current = tmp->prev; + PyMem_Free(tmp); + self->stacksize--; + return result; +} + +static PyObject* +BaseContext_push(BaseContext *self, PyObject *args) +{ + PyObject *value = NULL; + if (!PyArg_ParseTuple(args, "|O:push", &value)) + return NULL; + if (!value) { + value = PyDict_New(); + if (!value) + return NULL; + } + else if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, "dict required."); + return NULL; + } + else + Py_INCREF(value); + struct StackLayer *new = malloc(sizeof(struct StackLayer)); + new->dict = value; + new->prev = self->current; + self->current = new; + self->stacksize++; + Py_INCREF(value); + return value; +} + +static PyObject* +BaseContext_getstack(BaseContext *self, void *closure) +{ + PyObject *result = PyList_New(self->stacksize); + if (!result) + return NULL; + struct StackLayer *current = self->current; + int idx = 0; + while (current) { + PyList_SetItem(result, idx++, current->dict); + Py_INCREF(current->dict); + current = current->prev; + } + PyList_Reverse(result); + return result; +} + +static PyObject* +BaseContext_getcurrent(BaseContext *self, void *closure) +{ + Py_INCREF(self->current->dict); + return self->current->dict; +} + +static PyObject* +BaseContext_getinitial(BaseContext *self, void *closure) +{ + Py_INCREF(self->initial->dict); + return self->initial->dict; +} + +static PyObject* +BaseContext_getglobals(BaseContext *self, void *closure) +{ + Py_INCREF(self->globals->dict); + return self->globals->dict; +} + +static int +BaseContext_readonly(BaseContext *self, PyObject *value, void *closure) +{ + PyErr_SetString(PyExc_AttributeError, "can't set attribute"); + return -1; +} + +static PyObject* +BaseContext_getitem(BaseContext *self, PyObject *item) +{ + if (!PyString_Check(item)) { + Py_INCREF(Py_False); + return Py_False; + } + + /* disallow access to internal jinja values */ + char *name = PyString_AS_STRING(item); + if (strlen(name) >= 2 && name[0] == ':' && name[1] == ':') { + Py_INCREF(Py_False); + return Py_False; + } + + PyObject *result; + struct StackLayer *current = self->current; + while (current) { + result = PyDict_GetItemString(current->dict, name); + if (!result) { + current = current->prev; + continue; + } + Py_INCREF(result); + if (PyObject_TypeCheck(result, DeferredType)) { + PyObject *args = PyTuple_New(2); + if (!args || !PyTuple_SetItem(args, 0, (PyObject*)self) || + !PyTuple_SetItem(args, 1, item)) + return NULL; + + PyObject *resolved = PyObject_CallObject(result, args); + if (!resolved) + return NULL; + + /* never touch the globals */ + Py_DECREF(result); + Py_INCREF(resolved); + PyObject *namespace; + if (current == self->globals) + namespace = self->initial->dict; + else + namespace = current->dict; + PyDict_SetItemString(namespace, name, resolved); + return resolved; + } + return result; + } + + if (self->silent) { + Py_INCREF(Undefined); + return Undefined; + } + PyErr_Format(TemplateRuntimeError, "'%s' is not defined", name); + return NULL; +} + +static int +BaseContext_contains(BaseContext *self, PyObject *item) +{ + if (!PyString_Check(item)) + return 0; + + char *name = PyString_AS_STRING(item); + if (strlen(name) >= 2 && name[0] == ':' && name[1] == ':') + return 0; + + struct StackLayer *current = self->current; + while (current) { + if (!PyMapping_HasKeyString(current->dict, name)) { + current = current->prev; + continue; + } + return 1; + } + + return 0; +} + +static int +BaseContext_setitem(BaseContext *self, PyObject *item, PyObject *value) +{ + char *name = PyString_AS_STRING(item); + if (!value) + return PyDict_DelItemString(self->current->dict, name); + return PyDict_SetItemString(self->current->dict, name, value); +} + +static PyObject* +BaseContext_length(BaseContext *self) +{ + return PyInt_FromLong(self->stacksize); +} + +static PyGetSetDef BaseContext_getsetters[] = { + {"stack", (getter)BaseContext_getstack, (setter)BaseContext_readonly, + "a read only copy of the internal stack", NULL}, + {"current", (getter)BaseContext_getcurrent, (setter)BaseContext_readonly, + "reference to the current layer on the stack", NULL}, + {"initial", (getter)BaseContext_getinitial, (setter)BaseContext_readonly, + "reference to the initial layer on the stack", NULL}, + {"globals", (getter)BaseContext_getglobals, (setter)BaseContext_readonly, + "reference to the global layer on the stack", NULL}, + {NULL} /* Sentinel */ +}; + +static PyMemberDef BaseContext_members[] = { + {NULL} /* Sentinel */ +}; + +static PyMethodDef BaseContext_methods[] = { + {"pop", (PyCFunction)BaseContext_pop, METH_NOARGS, + "Pop the highest layer from the stack"}, + {"push", (PyCFunction)BaseContext_push, METH_VARARGS, + "Push one layer to the stack"}, + {NULL} /* Sentinel */ +}; + +static PySequenceMethods BaseContext_as_sequence[] = { + 0, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ + 0, /* sq_ass_slice */ + BaseContext_contains, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +static PyMappingMethods BaseContext_as_mapping[] = { + (lenfunc)BaseContext_length, + (binaryfunc)BaseContext_getitem, + (objobjargproc)BaseContext_setitem +}; + +static PyTypeObject BaseContextType = { + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ + "jinja._speedups.BaseContext", /* tp_name */ + sizeof(BaseContext), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)BaseContext_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &BaseContext_as_sequence, /* tp_as_sequence */ + &BaseContext_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BaseContext_methods, /* tp_methods */ + BaseContext_members, /* tp_members */ + BaseContext_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BaseContext_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew /* tp_new */ +}; + +static PyMethodDef module_methods[] = { + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC +init_speedups(void) +{ + PyObject *module; + + if (PyType_Ready(&BaseContextType) < 0) + return; + + if (!init_constants()) + return; + + module = Py_InitModule3("_speedups", module_methods, ""); + if (!module) + return; + + Py_INCREF(&BaseContextType); + PyModule_AddObject(module, "BaseContext", (PyObject*)&BaseContextType); +} diff --git a/jinja/datastructure.py b/jinja/datastructure.py index 4047d72..bf7325d 100644 --- a/jinja/datastructure.py +++ b/jinja/datastructure.py @@ -180,107 +180,58 @@ class Flush(TemplateData): jinja_no_finalization = True -class Context(object): +# import these here because those modules import Deferred and Undefined +# from this module. +try: + # try to use the c implementation of the base context if available + from jinja._speedups import BaseContext +except ImportError: + # if there is no c implementation we go with a native python one + from jinja._native import BaseContext + + +class Context(BaseContext): """ Dict like object containing the variables for the template. """ def __init__(self, _environment_, *args, **kwargs): - self.environment = _environment_ - self._stack = [_environment_.globals, dict(*args, **kwargs), {}] - self.globals, self.initial, self.current = self._stack - self._translate_func = None + super(Context, self).__init__(_environment_.silent, + _environment_.globals, + dict(*args, **kwargs)) - # cache object used for filters and tests + self._translate_func = None self.cache = {} - - def translate_func(self): - """ - Return the translator for this context. - """ - if self._translate_func is None: - translator = self.environment.get_translator(self) - def translate(s, p=None, n=None, r=None): - if p is None: - return translator.gettext(s) % (r or {}) - return translator.ngettext(s, p, r[n]) % (r or {}) - self._translate_func = translate - return self._translate_func - translate_func = property(translate_func, doc=translate_func.__doc__) - - def pop(self): - """ - Pop the last layer from the stack and return it. - """ - rv = self._stack.pop() - self.current = self._stack[-1] - return rv - - def push(self, data=None): - """ - Push a new dict or empty layer to the stack and return that layer - """ - data = data or {} - self._stack.append(data) - self.current = self._stack[-1] - return data + self.environment = _environment_ def to_dict(self): """ Convert the context into a dict. This skips the globals. """ result = {} - for layer in self._stack[1:]: + for layer in self.stack[1:]: for key, value in layer.iteritems(): if key.startswith('::'): continue result[key] = value return result - def __getitem__(self, name): - """ - Resolve one item. Restrict the access to internal variables - such as ``'::cycle1'``. Resolve deferreds. - """ - if not name.startswith('::'): - # because the stack is usually quite small we better use [::-1] - # which is faster than reversed() somehow. - for d in self._stack[::-1]: - if name in d: - rv = d[name] - if rv.__class__ is Deferred: - rv = rv(self, name) - # never touch the globals! - if d is self.globals: - self.initial[name] = rv - else: - d[name] = rv - return rv - if self.environment.silent: - return Undefined - raise TemplateRuntimeError('%r is not defined' % name) - - def __setitem__(self, name, value): - """ - Set a variable in the outermost layer. - """ - self.current[name] = value - - def __delitem__(self, name): - """ - Delete an variable in the outermost layer. - """ - if name in self.current: - del self.current[name] - - def __contains__(self, name): + def translate_func(self): """ - Check if the context contains a given variable. + Return a translation function for this context. It takes + 4 parameters. The singular string, the optional plural one, + the indicator number which is used to select the correct + plural form and a dict with values which should be inserted. """ - for layer in self._stack: - if name in layer: - return True - return False + if self._translate_func is None: + translator = self.environment.get_translator(self) + def translate(s, p=None, n=None, r=None): + if p is None: + return translator.gettext(s) % (r or {}) + return translator.ngettext(s, p, r[n]) % (r or {}) + self._translate_func = translate + return self._translate_func + translate_func = property(translate_func, doc=translate_func.__doc__) def __repr__(self): """ diff --git a/jinja/environment.py b/jinja/environment.py index e1a057c..a1a08e1 100644 --- a/jinja/environment.py +++ b/jinja/environment.py @@ -167,6 +167,17 @@ class Environment(object): parser = Parser(self, source, filename) return parser.parse() + def lex(self, source, filename=None): + """ + Lex the given sourcecode and return a generator that yields tokens. + The stream returned is not usable for Jinja but can be used if + Jinja templates should be processed by other tools (for example + syntax highlighting etc) + + The tuples are returned in the form ``(lineno, token, value)``. + """ + return self.lexer.tokeniter(source, filename) + def from_string(self, source): """ Load and parse a template source and translate it into eval-able @@ -300,7 +311,7 @@ class Environment(object): """ if dyn_args is not None: args += tuple(dyn_args) - elif dyn_kwargs is not None: + if dyn_kwargs is not None: kwargs.update(dyn_kwargs) if getattr(f, 'jinja_unsafe_call', False) or \ getattr(f, 'alters_data', False): diff --git a/jinja/lexer.py b/jinja/lexer.py index 7d488db..0475d25 100644 --- a/jinja/lexer.py +++ b/jinja/lexer.py @@ -51,7 +51,7 @@ number_re = re.compile(r'\d+(\.\d+)*') operator_re = re.compile('(%s)' % '|'.join([ isinstance(x, unicode) and str(x) or re.escape(x) for x in [ # math operators - '+', '-', '*', '//', '/', '%', + '+', '-', '**', '*', '//', '/', '%', # braces and parenthesis '[', ']', '(', ')', '{', '}', # attribute access and comparison / logical operators @@ -64,7 +64,7 @@ keywords = set(['and', 'block', 'cycle', 'elif', 'else', 'endblock', 'endfilter', 'endfor', 'endif', 'endmacro', 'endraw', 'endtrans', 'extends', 'filter', 'for', 'if', 'in', 'include', 'is', 'macro', 'not', 'or', 'pluralize', 'raw', - 'recursive', 'set', 'trans']) + 'recursive', 'set', 'trans', 'print', 'call', 'endcall']) class Failure(object): diff --git a/jinja/nodes.py b/jinja/nodes.py index 9156b3b..cba063d 100644 --- a/jinja/nodes.py +++ b/jinja/nodes.py @@ -220,6 +220,26 @@ class Macro(Node): ) +class Call(Node): + """ + A node that represents am extended macro call. + """ + + def __init__(self, lineno, expr, body): + self.lineno = lineno + self.expr = expr + self.body = body + + def get_items(self): + return [self.expr, self.body] + + def __repr__(self): + return 'Call(%r, %r)' % ( + self.expr, + self.body + ) + + class Set(Node): """ Allow defining own variables. @@ -322,7 +342,9 @@ class Include(Node): return [self.template] def __repr__(self): - return 'Include(%r)' % self.template + return 'Include(%r)' % ( + self.template + ) class Trans(Node): diff --git a/jinja/parser.py b/jinja/parser.py index cb3e062..f93a1d5 100644 --- a/jinja/parser.py +++ b/jinja/parser.py @@ -41,6 +41,7 @@ switch_if = StateTest.expect_name('else', 'elif', 'endif') end_of_if = StateTest.expect_name('endif') end_of_filter = StateTest.expect_name('endfilter') end_of_macro = StateTest.expect_name('endmacro') +end_of_call = StateTest.expect_name('endcall') end_of_block_tag = StateTest.expect_name('endblock') end_of_trans = StateTest.expect_name('endtrans') @@ -77,6 +78,7 @@ class Parser(object): 'filter': self.handle_filter_directive, 'print': self.handle_print_directive, 'macro': self.handle_macro_directive, + 'call': self.handle_call_directive, 'block': self.handle_block_directive, 'extends': self.handle_extends_directive, 'include': self.handle_include_directive, @@ -262,6 +264,19 @@ class Parser(object): args = None return nodes.Macro(lineno, ast.name, args, body) + def handle_call_directive(self, lineno, gen): + """ + Handle {% call foo() %}...{% endcall %} + """ + expr = self.parse_python(lineno, gen, '(%s)').expr + if expr.__class__ is not ast.CallFunc: + raise TemplateSyntaxError('call requires a function or macro ' + 'call as only argument.', lineno, + self.filename) + body = self.subparse(end_of_call, True) + self.close_remaining_block() + return nodes.Call(lineno, expr, body) + def handle_block_directive(self, lineno, gen): """ Handle block directives used for inheritance. @@ -305,7 +320,8 @@ class Parser(object): raise TemplateSyntaxError('extends requires a string', lineno, self.filename) if self.extends is not None: - raise TemplateSyntaxError('extends called twice', lineno) + raise TemplateSyntaxError('extends called twice', lineno, + self.filename) self.extends = nodes.Extends(lineno, tokens[0][2][1:-1]) def handle_include_directive(self, lineno, gen): @@ -313,10 +329,13 @@ class Parser(object): Handle the include directive used for template inclusion. """ tokens = list(gen) - if len(tokens) != 1 or tokens[0][1] != 'string': - raise TemplateSyntaxError('include requires a string', lineno, - self.filename) - return nodes.Include(lineno, tokens[0][2][1:-1]) + # hardcoded include (faster because copied into the bytecode) + if len(tokens) == 1 and tokens[0][1] == 'string': + return nodes.Include(lineno, str(tokens[0][2][1:-1])) + raise TemplateSyntaxError('invalid syntax for include ' + 'directive. Requires a hardcoded ' + 'string', lineno, + self.filename) def handle_trans_directive(self, lineno, gen): """ @@ -338,8 +357,9 @@ class Parser(object): try: gen.next() except StopIteration: - #XXX: what about escapes? - return nodes.Trans(lineno, data[1:-1], None, + # XXX: this looks fishy + data = data[1:-1].encode('utf-8').decode('string-escape') + return nodes.Trans(lineno, data.decode('utf-8'), None, None, None) raise TemplateSyntaxError('string based translations ' 'require at most one argument.', @@ -560,6 +580,7 @@ class Parser(object): 'as identifier.' % node.name, node.lineno, self.filename) node.name = node.name[:-1] + # same for attributes elif node.__class__ is ast.Getattr: if not node.attrname.endswith('_'): raise TemplateSyntaxError('illegal use of keyword %r ' @@ -567,6 +588,18 @@ class Parser(object): node.name, node.lineno, self.filename) node.attrname = node.attrname[:-1] + # if we have a ForLoop we ensure that nobody patches existing + # object using "for foo.bar in seq" + elif node.__class__ is nodes.ForLoop: + def check(node): + if node.__class__ not in (ast.AssName, ast.AssTuple): + raise TemplateSyntaxError('can\'t assign to ' + 'expression.', node.lineno, + self.filename) + for n in node.getChildNodes(): + check(n) + check(node.item) + # now set the filename and continue working on the childnodes node.filename = self.filename todo.extend(node.getChildNodes()) return nodes.Template(self.filename, body, self.extends) diff --git a/jinja/translators/python.py b/jinja/translators/python.py index 4072f10..62bcd39 100644 --- a/jinja/translators/python.py +++ b/jinja/translators/python.py @@ -44,6 +44,12 @@ except NameError: """For python2.3/python2.4 compatibility""" +try: + set +except NameError: + from sets import Set as set + + def _to_tuple(args): """ Return a tuple repr without nested repr. @@ -173,6 +179,7 @@ class PythonTranslator(Translator): nodes.Cycle: self.handle_cycle, nodes.Print: self.handle_print, nodes.Macro: self.handle_macro, + nodes.Call: self.handle_call, nodes.Set: self.handle_set, nodes.Filter: self.handle_filter, nodes.Block: self.handle_block, @@ -386,10 +393,13 @@ class PythonTranslator(Translator): # bootstrapping code lines = [ + '# Essential imports\n' 'from __future__ import division\n' 'from jinja.datastructure import Undefined, LoopContext, ' 'CycleContext, SuperBlock\n' 'from jinja.utils import buffereater\n' + 'from jinja.exceptions import TemplateRuntimeError\n\n' + '# Local aliases for some speedup\n' '%s\n' '__name__ = %r\n\n' 'def generate(context):\n' @@ -412,7 +422,7 @@ class PythonTranslator(Translator): # add body lines and "generator hook" lines.extend(body_lines) - lines.append(' if False:\n yield None') + lines.append(' if 0: yield None') # add the missing blocks block_items = blocks.items() @@ -428,8 +438,7 @@ class PythonTranslator(Translator): '\ndef %s(context):' % func_name, ' ctx_push = context.push', ' ctx_pop = context.pop', - ' if False:', - ' yield None' + ' if 0: yield None' ]) lines.append(self.indent(self.nodeinfo(item, True))) lines.append(self.handle_block(item, idx + 1)) @@ -441,10 +450,15 @@ class PythonTranslator(Translator): # blocks must always be defined. even if it's empty. some # features depend on it - lines.append('\nblocks = {\n%s\n}' % ',\n'.join(dict_lines)) + lines.append('\n# Block mapping and debug information') + if dict_lines: + lines.append('blocks = {\n%s\n}' % ',\n'.join(dict_lines)) + else: + lines.append('blocks = {}') # now get the real source lines and map the debugging symbols debug_mapping = [] + file_mapping = {} last = None offset = -1 sourcelines = ('\n'.join(lines)).splitlines() @@ -454,10 +468,16 @@ class PythonTranslator(Translator): m = _debug_re.search(line) if m is not None: d = m.groupdict() - this = (d['filename'] or None, int(d['lineno'])) + filename = d['filename'] or None + if filename in file_mapping: + file_id = file_mapping[filename] + else: + file_id = file_mapping[filename] = 'F%d' % \ + len(file_mapping) + this = (file_id, int(d['lineno'])) # if it's the same as the line before we ignore it if this != last: - debug_mapping.append((idx - offset,) + this) + debug_mapping.append('(%r, %s, %r)' % ((idx - offset,) + this)) last = this # for each debug symbol the line number and so the offset # changes by one. @@ -465,15 +485,26 @@ class PythonTranslator(Translator): else: result.append(line) - result.append('\ndebug_info = %r' % debug_mapping) + # now print file mapping and debug info + file_mapping = file_mapping.items() + file_mapping.sort(lambda a, b: cmp(a[1], b[1])) + for filename, file_id in file_mapping: + result.append('%s = %r' % (file_id, filename)) + result.append('debug_info = [%s]' % ', '.join(debug_mapping)) return '\n'.join(result) def handle_template_text(self, node): """ Handle data around nodes. """ + # if we have a ascii only string we go with the + # bytestring. otherwise we go with the unicode object + try: + data = str(node.text) + except UnicodeError: + data = node.text return self.indent(self.nodeinfo(node)) + '\n' +\ - self.indent('yield %r' % node.text) + self.indent('yield %r' % data) def handle_node_list(self, node): """ @@ -533,10 +564,8 @@ class PythonTranslator(Translator): # call recursive for loop! if node.recursive: write('context[\'loop\'].pop()') - write('if False:') - self.indention += 1 - write('yield None') - self.indention -= 2 + write('if 0: yield None') + self.indention -= 1 write('context[\'loop\'] = LoopContext(None, context[\'loop\'], ' 'buffereater(forloop))') write('for item in forloop(%s):' % self.handle_node(node.seq)) @@ -631,37 +660,67 @@ class PythonTranslator(Translator): buf = [] write = lambda x: buf.append(self.indent(x)) - write('def macro(*args):') + write('def macro(*args, **kw):') self.indention += 1 write(self.nodeinfo(node)) + # collect macro arguments + arg_items = [] + caller_overridden = False if node.arguments: write('argcount = len(args)') - tmp = [] for idx, (name, n) in enumerate(node.arguments): - tmp.append('\'%s\': (argcount > %d and (args[%d],) ' + arg_items.append('\'%s\': (argcount > %d and (args[%d],) ' 'or (%s,))[0]' % ( name, idx, idx, n is None and 'Undefined' or self.handle_node(n) )) - write('ctx_push({%s})' % ', '.join(tmp)) + if name == 'caller': + caller_overridden = True + if caller_overridden: + write('kw.pop(\'caller\', None)') else: - write('ctx_push()') + arg_items.append('\'caller\': kw.pop(\'caller\', Undefined)') + write('ctx_push({%s})' % ', '.join(arg_items)) + + # disallow any keyword arguments + write('if kw:') + self.indention += 1 + write('raise TemplateRuntimeError(\'%s got an unexpected keyword ' + 'argument %%r\' %% iter(kw).next())' % node.name) + self.indention -= 1 write(self.nodeinfo(node.body)) buf.append(self.handle_node(node.body)) write('ctx_pop()') - write('if False:') - self.indention += 1 - write('yield False') - self.indention -= 2 + write('if 0: yield None') + self.indention -= 1 buf.append(self.indent('context[%r] = buffereater(macro)' % node.name)) return '\n'.join(buf) + def handle_call(self, node): + """ + Handle extended macro calls. + """ + buf = [] + write = lambda x: buf.append(self.indent(x)) + + write('def call(**kwargs):') + self.indention += 1 + write('ctx_push(kwargs)') + buf.append(self.handle_node(node.body)) + write('ctx_pop()') + write('if 0: yield None') + self.indention -= 1 + write('yield ' + self.handle_call_func(node.expr, + {'caller': 'buffereater(call)'})) + + return '\n'.join(buf) + def handle_set(self, node): """ Handle variable assignments. @@ -687,10 +746,8 @@ class PythonTranslator(Translator): write(self.nodeinfo(node.body)) buf.append(self.handle_node(node.body)) write('ctx_pop()') - write('if False:') - self.indention += 1 - write('yield None') - self.indention -= 2 + write('if 0: yield None') + self.indention -= 1 write('yield %s' % self.filter('buffereater(filtered)()', node.filters)) return '\n'.join(buf) @@ -874,7 +931,7 @@ class PythonTranslator(Translator): """ return self.filter(self.handle_node(node.nodes[0]), node.nodes[1:]) - def handle_call_func(self, node): + def handle_call_func(self, node, extra_kwargs=None): """ Handle function calls. """ @@ -890,7 +947,9 @@ class PythonTranslator(Translator): kwargs[arg.name] = self.handle_node(arg.expr) else: args.append(self.handle_node(arg)) - if not (args or kwargs or star_args or dstar_args): + if extra_kwargs: + kwargs.update(extra_kwargs) + if not (args or kwargs or star_args or dstar_args or extra_kwargs): return 'call_function_simple(%s, context)' % \ self.handle_node(node.node) return 'call_function(%s, context, %s, {%s}, %s, %s)' % ( diff --git a/setup.py b/setup.py index 4943932..e45aa0d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,10 @@ import jinja import os import ez_setup ez_setup.use_setuptools() -from setuptools import setup + +from distutils.command.build_ext import build_ext +from distutils.errors import CCompilerError +from setuptools import setup, Extension, Feature from inspect import getdoc @@ -16,6 +19,22 @@ def list_files(path): yield fn +class optional_build_ext(build_ext): + + def build_extension(self, ext): + try: + build_ext.build_extension(self, ext) + except CCompilerError, e: + print '=' * 79 + print 'INFORMATION' + print ' the speedup extension could not be compiled, jinja will' + print ' fall back to the native python classes.' + print '=' * 79 + + + + + setup( name = 'Jinja', version = '1.0', @@ -51,5 +70,13 @@ setup( [python.templating.engines] jinja = jinja.plugin:BuffetPlugin ''', - extras_require = {'plugin': ['setuptools>=0.6a2']} + extras_require = {'plugin': ['setuptools>=0.6a2']}, + features = {'speedups': Feature( + 'optional C-speed enhancements', + standard = True, + ext_modules = [ + Extension('jinja._speedups', ['jinja/_speedups.c']) + ] + )}, + cmdclass = {'build_ext': optional_build_ext} ) diff --git a/tests/conftest.py b/tests/conftest.py index aa060fa..651aca8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,7 +36,7 @@ class GlobalLoader(object): loader = GlobalLoader(globals()) -simple_env = Environment(trim_blocks=True, loader=loader) +simple_env = Environment(trim_blocks=True, friendly_traceback=False, loader=loader) class Module(py.test.collect.Module): diff --git a/tests/test_macros.py b/tests/test_macros.py index c28499b..35cb291 100644 --- a/tests/test_macros.py +++ b/tests/test_macros.py @@ -35,10 +35,11 @@ def test_simple(env): def test_kwargs_failure(env): + from jinja.exceptions import TemplateRuntimeError tmpl = env.from_string(KWARGSFAILURE) try: tmpl.render() - except TypeError, e: + except TemplateRuntimeError, e: pass else: raise AssertionError('kwargs failure test failed') diff --git a/tests/test_various.py b/tests/test_various.py index b7336e2..df08bd4 100644 --- a/tests/test_various.py +++ b/tests/test_various.py @@ -34,6 +34,8 @@ UNPACKING = '''{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor RAW = '''{% raw %}{{ FOO }} and {% BAR %}{% endraw %}''' +CALL = '''{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}''' + def test_keywords(env): env.from_string(KEYWORDS) @@ -59,3 +61,11 @@ def test_cache_dict(): d["d"] = 4 assert len(d) == 3 assert 'a' in d and 'c' in d and 'd' in d and 'b' not in d + + +def test_call(): + from jinja import Environment + env = Environment() + env.globals['foo'] = lambda a, b, c, e, g: a + b + c + e + g + tmpl = env.from_string(CALL) + assert tmpl.render() == 'abdfh' -- 2.26.2