Merge pull request #137 from PaulMcMillan/improve_whitespace_docs
[jinja2.git] / ext / django2jinja / django2jinja.py
1 # -*- coding: utf-8 -*-
2 """
3     Django to Jinja
4     ~~~~~~~~~~~~~~~
5
6     Helper module that can convert django templates into Jinja2 templates.
7
8     This file is not intended to be used as stand alone application but to
9     be used as library.  To convert templates you basically create your own
10     writer, add extra conversion logic for your custom template tags,
11     configure your django environment and run the `convert_templates`
12     function.
13
14     Here a simple example::
15
16         # configure django (or use settings.configure)
17         import os
18         os.environ['DJANGO_SETTINGS_MODULE'] = 'yourapplication.settings'
19         from yourapplication.foo.templatetags.bar import MyNode
20
21         from django2jinja import Writer, convert_templates
22
23         def write_my_node(writer, node):
24             writer.start_variable()
25             writer.write('myfunc(')
26             for idx, arg in enumerate(node.args):
27                 if idx:
28                     writer.write(', ')
29                 writer.node(arg)
30             writer.write(')')
31             writer.end_variable()
32
33         writer = Writer()
34         writer.node_handlers[MyNode] = write_my_node
35         convert_templates('/path/to/output/folder', writer=writer)
36     
37     Here is an example hos to automatically translate your django
38     variables to jinja2::
39         
40         import re
41         # List of tuple (Match pattern, Replace pattern, Exclusion pattern)
42         
43         var_re  = ((re.compile(r"(u|user)\.is_authenticated"), r"\1.is_authenticated()", None),
44                   (re.compile(r"\.non_field_errors"), r".non_field_errors()", None),
45                   (re.compile(r"\.label_tag"), r".label_tag()", None),
46                   (re.compile(r"\.as_dl"), r".as_dl()", None),
47                   (re.compile(r"\.as_table"), r".as_table()", None),
48                   (re.compile(r"\.as_widget"), r".as_widget()", None),
49                   (re.compile(r"\.as_hidden"), r".as_hidden()", None),
50                   
51                   (re.compile(r"\.get_([0-9_\w]+)_url"), r".get_\1_url()", None),
52                   (re.compile(r"\.url"), r".url()", re.compile(r"(form|calendar).url")),
53                   (re.compile(r"\.get_([0-9_\w]+)_display"), r".get_\1_display()", None),
54                   (re.compile(r"loop\.counter"), r"loop.index", None),
55                   (re.compile(r"loop\.revcounter"), r"loop.revindex", None),
56                   (re.compile(r"request\.GET\.([0-9_\w]+)"), r"request.GET.get('\1', '')", None),
57                   (re.compile(r"request\.get_host"), r"request.get_host()", None),
58                   
59                   (re.compile(r"\.all(?!_)"), r".all()", None),
60                   (re.compile(r"\.all\.0"), r".all()[0]", None),
61                   (re.compile(r"\.([0-9])($|\s+)"), r"[\1]\2", None),
62                   (re.compile(r"\.items"), r".items()", None),
63         )
64         writer = Writer(var_re=var_re)
65         
66     For details about the writing process have a look at the module code.
67
68     :copyright: (c) 2009 by the Jinja Team.
69     :license: BSD.
70 """
71 import re
72 import os
73 import sys
74 from jinja2.defaults import *
75 from django.conf import settings
76 from django.template import defaulttags as core_tags, loader, TextNode, \
77      FilterExpression, libraries, Variable, loader_tags, TOKEN_TEXT, \
78      TOKEN_VAR
79 from django.template.debug import DebugVariableNode as VariableNode
80 from django.templatetags import i18n as i18n_tags
81 from StringIO import StringIO
82
83
84 _node_handlers = {}
85 _resolved_filters = {}
86 _newline_re = re.compile(r'(?:\r\n|\r|\n)')
87
88
89 # Django stores an itertools object on the cycle node.  Not only is this
90 # thread unsafe but also a problem for the converter which needs the raw
91 # string values passed to the constructor to create a jinja loop.cycle()
92 # call from it.
93 _old_cycle_init = core_tags.CycleNode.__init__
94 def _fixed_cycle_init(self, cyclevars, variable_name=None):
95     self.raw_cycle_vars = map(Variable, cyclevars)
96     _old_cycle_init(self, cyclevars, variable_name)
97 core_tags.CycleNode.__init__ = _fixed_cycle_init
98
99
100 def node(cls):
101     def proxy(f):
102         _node_handlers[cls] = f
103         return f
104     return proxy
105
106
107 def convert_templates(output_dir, extensions=('.html', '.txt'), writer=None,
108                       callback=None):
109     """Iterates over all templates in the template dirs configured and
110     translates them and writes the new templates into the output directory.
111     """
112     if writer is None:
113         writer = Writer()
114
115     def filter_templates(files):
116         for filename in files:
117             ifilename = filename.lower()
118             for extension in extensions:
119                 if ifilename.endswith(extension):
120                     yield filename
121
122     def translate(f, loadname):
123         template = loader.get_template(loadname)
124         original = writer.stream
125         writer.stream = f
126         writer.body(template.nodelist)
127         writer.stream = original
128
129     if callback is None:
130         def callback(template):
131             print template
132
133     for directory in settings.TEMPLATE_DIRS:
134         for dirname, _, files in os.walk(directory):
135             dirname = dirname[len(directory) + 1:]
136             for filename in filter_templates(files):
137                 source = os.path.normpath(os.path.join(dirname, filename))
138                 target = os.path.join(output_dir, dirname, filename)
139                 basetarget = os.path.dirname(target)
140                 if not os.path.exists(basetarget):
141                     os.makedirs(basetarget)
142                 callback(source)
143                 f = file(target, 'w')
144                 try:
145                     translate(f, source)
146                 finally:
147                     f.close()
148
149
150 class Writer(object):
151     """The core writer class."""
152
153     def __init__(self, stream=None, error_stream=None,
154                  block_start_string=BLOCK_START_STRING,
155                  block_end_string=BLOCK_END_STRING,
156                  variable_start_string=VARIABLE_START_STRING,
157                  variable_end_string=VARIABLE_END_STRING,
158                  comment_start_string=COMMENT_START_STRING,
159                  comment_end_string=COMMENT_END_STRING,
160                  initial_autoescape=True,
161                  use_jinja_autoescape=False,
162                  custom_node_handlers=None,
163                  var_re=[],
164                  env=None):
165         if stream is None:
166             stream = sys.stdout
167         if error_stream is None:
168             error_stream = sys.stderr
169         self.stream = stream
170         self.error_stream = error_stream
171         self.block_start_string = block_start_string
172         self.block_end_string = block_end_string
173         self.variable_start_string = variable_start_string
174         self.variable_end_string = variable_end_string
175         self.comment_start_string = comment_start_string
176         self.comment_end_string = comment_end_string
177         self.autoescape = initial_autoescape
178         self.spaceless = False
179         self.use_jinja_autoescape = use_jinja_autoescape
180         self.node_handlers = dict(_node_handlers,
181                                   **(custom_node_handlers or {}))
182         self._loop_depth = 0
183         self._filters_warned = set()
184         self.var_re = var_re
185         self.env = env
186
187     def enter_loop(self):
188         """Increments the loop depth so that write functions know if they
189         are in a loop.
190         """
191         self._loop_depth += 1
192
193     def leave_loop(self):
194         """Reverse of enter_loop."""
195         self._loop_depth -= 1
196
197     @property
198     def in_loop(self):
199         """True if we are in a loop."""
200         return self._loop_depth > 0
201
202     def write(self, s):
203         """Writes stuff to the stream."""
204         self.stream.write(s.encode(settings.FILE_CHARSET))
205
206     def print_expr(self, expr):
207         """Open a variable tag, write to the string to the stream and close."""
208         self.start_variable()
209         self.write(expr)
210         self.end_variable()
211
212     def _post_open(self):
213         if self.spaceless:
214             self.write('- ')
215         else:
216             self.write(' ')
217
218     def _pre_close(self):
219         if self.spaceless:
220             self.write(' -')
221         else:
222             self.write(' ')
223
224     def start_variable(self):
225         """Start a variable."""
226         self.write(self.variable_start_string)
227         self._post_open()
228
229     def end_variable(self, always_safe=False):
230         """End a variable."""
231         if not always_safe and self.autoescape and \
232            not self.use_jinja_autoescape:
233             self.write('|e')
234         self._pre_close()
235         self.write(self.variable_end_string)
236
237     def start_block(self):
238         """Starts a block."""
239         self.write(self.block_start_string)
240         self._post_open()
241
242     def end_block(self):
243         """Ends a block."""
244         self._pre_close()
245         self.write(self.block_end_string)
246
247     def tag(self, name):
248         """Like `print_expr` just for blocks."""
249         self.start_block()
250         self.write(name)
251         self.end_block()
252
253     def variable(self, name):
254         """Prints a variable.  This performs variable name transformation."""
255         self.write(self.translate_variable_name(name))
256
257     def literal(self, value):
258         """Writes a value as literal."""
259         value = repr(value)
260         if value[:2] in ('u"', "u'"):
261             value = value[1:]
262         self.write(value)
263
264     def filters(self, filters, is_block=False):
265         """Dumps a list of filters."""
266         want_pipe = not is_block
267         for filter, args in filters:
268             name = self.get_filter_name(filter)
269             if name is None:
270                 self.warn('Could not find filter %s' % name)
271                 continue
272             if name not in DEFAULT_FILTERS and \
273                name not in self._filters_warned:
274                 self._filters_warned.add(name)
275                 self.warn('Filter %s probably doesn\'t exist in Jinja' %
276                             name)
277             if not want_pipe:
278                 want_pipe = True
279             else:
280                 self.write('|')
281             self.write(name)
282             if args:
283                 self.write('(')
284                 for idx, (is_var, value) in enumerate(args):
285                     if idx:
286                         self.write(', ')
287                     if is_var:
288                         self.node(value)
289                     else:
290                         self.literal(value)
291                 self.write(')')
292
293     def get_location(self, origin, position):
294         """Returns the location for an origin and position tuple as name
295         and lineno.
296         """
297         if hasattr(origin, 'source'):
298             source = origin.source
299             name = '<unknown source>'
300         else:
301             source = origin.loader(origin.loadname, origin.dirs)[0]
302             name = origin.loadname
303         lineno = len(_newline_re.findall(source[:position[0]])) + 1
304         return name, lineno
305
306     def warn(self, message, node=None):
307         """Prints a warning to the error stream."""
308         if node is not None and hasattr(node, 'source'):
309             filename, lineno = self.get_location(*node.source)
310             message = '[%s:%d] %s' % (filename, lineno, message)
311         print >> self.error_stream, message
312
313     def translate_variable_name(self, var):
314         """Performs variable name translation."""
315         if self.in_loop and var == 'forloop' or var.startswith('forloop.'):
316             var = var[3:]
317         
318         for reg, rep, unless in self.var_re:
319             no_unless = unless and unless.search(var) or True
320             if reg.search(var) and no_unless:
321                 var = reg.sub(rep, var)
322                 break
323         return var
324
325     def get_filter_name(self, filter):
326         """Returns the filter name for a filter function or `None` if there
327         is no such filter.
328         """
329         if filter not in _resolved_filters:
330             for library in libraries.values():
331                 for key, value in library.filters.iteritems():
332                     _resolved_filters[value] = key
333         return _resolved_filters.get(filter, None)
334
335     def node(self, node):
336         """Invokes the node handler for a node."""
337         for cls, handler in self.node_handlers.iteritems():
338             if type(node) is cls or type(node).__name__ == cls:
339                 handler(self, node)
340                 break
341         else:
342             self.warn('Untranslatable node %s.%s found' % (
343                 node.__module__,
344                 node.__class__.__name__
345             ), node)
346
347     def body(self, nodes):
348         """Calls node() for every node in the iterable passed."""
349         for node in nodes:
350             self.node(node)
351
352
353 @node(TextNode)
354 def text_node(writer, node):
355     writer.write(node.s)
356
357
358 @node(Variable)
359 def variable(writer, node):
360     if node.translate:
361         writer.warn('i18n system used, make sure to install translations', node)
362         writer.write('_(')
363     if node.literal is not None:
364         writer.literal(node.literal)
365     else:
366         writer.variable(node.var)
367     if node.translate:
368         writer.write(')')
369
370
371 @node(VariableNode)
372 def variable_node(writer, node):
373     writer.start_variable()
374     if node.filter_expression.var.var == 'block.super' \
375        and not node.filter_expression.filters:
376         writer.write('super()')
377     else:
378         writer.node(node.filter_expression)
379     writer.end_variable()
380
381
382 @node(FilterExpression)
383 def filter_expression(writer, node):
384     writer.node(node.var)
385     writer.filters(node.filters)
386
387
388 @node(core_tags.CommentNode)
389 def comment_tag(writer, node):
390     pass
391
392
393 @node(core_tags.DebugNode)
394 def comment_tag(writer, node):
395     writer.warn('Debug tag detected.  Make sure to add a global function '
396                 'called debug to the namespace.', node=node)
397     writer.print_expr('debug()')
398
399
400 @node(core_tags.ForNode)
401 def for_loop(writer, node):
402     writer.start_block()
403     writer.write('for ')
404     for idx, var in enumerate(node.loopvars):
405         if idx:
406             writer.write(', ')
407         writer.variable(var)
408     writer.write(' in ')
409     if node.is_reversed:
410         writer.write('(')
411     writer.node(node.sequence)
412     if node.is_reversed:
413         writer.write(')|reverse')
414     writer.end_block()
415     writer.enter_loop()
416     writer.body(node.nodelist_loop)
417     writer.leave_loop()
418     writer.tag('endfor')
419
420
421 @node(core_tags.IfNode)
422 def if_condition(writer, node):
423     writer.start_block()
424     writer.write('if ')
425     join_with = 'and'
426     if node.link_type == core_tags.IfNode.LinkTypes.or_:
427         join_with = 'or'
428     
429     for idx, (ifnot, expr) in enumerate(node.bool_exprs):
430         if idx:
431             writer.write(' %s ' % join_with)
432         if ifnot:
433             writer.write('not ')
434         writer.node(expr)
435     writer.end_block()
436     writer.body(node.nodelist_true)
437     if node.nodelist_false:
438         writer.tag('else')
439         writer.body(node.nodelist_false)
440     writer.tag('endif')
441
442
443 @node(core_tags.IfEqualNode)
444 def if_equal(writer, node):
445     writer.start_block()
446     writer.write('if ')
447     writer.node(node.var1)
448     if node.negate:
449         writer.write(' != ')
450     else:
451         writer.write(' == ')
452     writer.node(node.var2)
453     writer.end_block()
454     writer.body(node.nodelist_true)
455     if node.nodelist_false:
456         writer.tag('else')
457         writer.body(node.nodelist_false)
458     writer.tag('endif')
459
460
461 @node(loader_tags.BlockNode)
462 def block(writer, node):
463     writer.tag('block ' + node.name.replace('-', '_').rstrip('_'))
464     node = node
465     while node.parent is not None:
466         node = node.parent
467     writer.body(node.nodelist)
468     writer.tag('endblock')
469
470
471 @node(loader_tags.ExtendsNode)
472 def extends(writer, node):
473     writer.start_block()
474     writer.write('extends ')
475     if node.parent_name_expr:
476         writer.node(node.parent_name_expr)
477     else:
478         writer.literal(node.parent_name)
479     writer.end_block()
480     writer.body(node.nodelist)
481
482
483 @node(loader_tags.ConstantIncludeNode)
484 @node(loader_tags.IncludeNode)
485 def include(writer, node):
486     writer.start_block()
487     writer.write('include ')
488     if hasattr(node, 'template'):
489         writer.literal(node.template.name)
490     else:
491         writer.node(node.template_name)
492     writer.end_block()
493
494
495 @node(core_tags.CycleNode)
496 def cycle(writer, node):
497     if not writer.in_loop:
498         writer.warn('Untranslatable free cycle (cycle outside loop)', node=node)
499         return
500     if node.variable_name is not None:
501         writer.start_block()
502         writer.write('set %s = ' % node.variable_name)
503     else:
504         writer.start_variable()
505     writer.write('loop.cycle(')
506     for idx, var in enumerate(node.raw_cycle_vars):
507         if idx:
508             writer.write(', ')
509         writer.node(var)
510     writer.write(')')
511     if node.variable_name is not None:
512         writer.end_block()
513     else:
514         writer.end_variable()
515
516
517 @node(core_tags.FilterNode)
518 def filter(writer, node):
519     writer.start_block()
520     writer.write('filter ')
521     writer.filters(node.filter_expr.filters, True)
522     writer.end_block()
523     writer.body(node.nodelist)
524     writer.tag('endfilter')
525
526
527 @node(core_tags.AutoEscapeControlNode)
528 def autoescape_control(writer, node):
529     original = writer.autoescape
530     writer.autoescape = node.setting
531     writer.body(node.nodelist)
532     writer.autoescape = original
533
534
535 @node(core_tags.SpacelessNode)
536 def spaceless(writer, node):
537     original = writer.spaceless
538     writer.spaceless = True
539     writer.warn('entering spaceless mode with different semantics', node)
540     # do the initial stripping
541     nodelist = list(node.nodelist)
542     if nodelist:
543         if isinstance(nodelist[0], TextNode):
544             nodelist[0] = TextNode(nodelist[0].s.lstrip())
545         if isinstance(nodelist[-1], TextNode):
546             nodelist[-1] = TextNode(nodelist[-1].s.rstrip())
547     writer.body(nodelist)
548     writer.spaceless = original
549
550
551 @node(core_tags.TemplateTagNode)
552 def template_tag(writer, node):
553     tag = {
554         'openblock':            writer.block_start_string,
555         'closeblock':           writer.block_end_string,
556         'openvariable':         writer.variable_start_string,
557         'closevariable':        writer.variable_end_string,
558         'opencomment':          writer.comment_start_string,
559         'closecomment':         writer.comment_end_string,
560         'openbrace':            '{',
561         'closebrace':           '}'
562     }.get(node.tagtype)
563     if tag:
564         writer.start_variable()
565         writer.literal(tag)
566         writer.end_variable()
567
568
569 @node(core_tags.URLNode)
570 def url_tag(writer, node):
571     writer.warn('url node used.  make sure to provide a proper url() '
572                 'function', node)
573     if node.asvar:
574         writer.start_block()
575         writer.write('set %s = ' % node.asvar)
576     else:
577         writer.start_variable()
578     autoescape = writer.autoescape
579     writer.write('url(')
580     writer.literal(node.view_name)
581     for arg in node.args:
582         writer.write(', ')
583         writer.node(arg)
584     for key, arg in node.kwargs.items():
585         writer.write(', %s=' % key)
586         writer.node(arg)
587     writer.write(')')
588     if node.asvar:
589         writer.end_block()
590     else:
591         writer.end_variable()
592
593
594 @node(core_tags.WidthRatioNode)
595 def width_ratio(writer, node):
596     writer.warn('widthratio expanded into formula.  You may want to provide '
597                 'a helper function for this calculation', node)
598     writer.start_variable()
599     writer.write('(')
600     writer.node(node.val_expr)
601     writer.write(' / ')
602     writer.node(node.max_expr)
603     writer.write(' * ')
604     writer.write(str(int(node.max_width)))
605     writer.write(')|round|int')
606     writer.end_variable(always_safe=True)
607
608
609 @node(core_tags.WithNode)
610 def with_block(writer, node):
611     writer.warn('with block expanded into set statement.  This could cause '
612                 'variables following that block to be overriden.', node)
613     writer.start_block()
614     writer.write('set %s = ' % node.name)
615     writer.node(node.var)
616     writer.end_block()
617     writer.body(node.nodelist)
618
619
620 @node(core_tags.RegroupNode)
621 def regroup(writer, node):
622     if node.expression.var.literal:
623         writer.warn('literal in groupby filter used.   Behavior in that '
624                     'situation is undefined and translation is skipped.', node)
625         return
626     elif node.expression.filters:
627         writer.warn('filters in groupby filter used.   Behavior in that '
628                     'situation is undefined which is most likely a bug '
629                     'in your code.  Filters were ignored.', node)
630     writer.start_block()
631     writer.write('set %s = ' % node.var_name)
632     writer.node(node.target)
633     writer.write('|groupby(')
634     writer.literal(node.expression.var.var)
635     writer.write(')')
636     writer.end_block()
637
638
639 @node(core_tags.LoadNode)
640 def warn_load(writer, node):
641     writer.warn('load statement used which was ignored on conversion', node)
642
643
644 @node(i18n_tags.GetAvailableLanguagesNode)
645 def get_available_languages(writer, node):
646     writer.warn('make sure to provide a get_available_languages function', node)
647     writer.tag('set %s = get_available_languages()' %
648                writer.translate_variable_name(node.variable))
649
650
651 @node(i18n_tags.GetCurrentLanguageNode)
652 def get_current_language(writer, node):
653     writer.warn('make sure to provide a get_current_language function', node)
654     writer.tag('set %s = get_current_language()' %
655                writer.translate_variable_name(node.variable))
656
657
658 @node(i18n_tags.GetCurrentLanguageBidiNode)
659 def get_current_language_bidi(writer, node):
660     writer.warn('make sure to provide a get_current_language_bidi function', node)
661     writer.tag('set %s = get_current_language_bidi()' %
662                writer.translate_variable_name(node.variable))
663
664
665 @node(i18n_tags.TranslateNode)
666 def simple_gettext(writer, node):
667     writer.warn('i18n system used, make sure to install translations', node)
668     writer.start_variable()
669     writer.write('_(')
670     writer.node(node.value)
671     writer.write(')')
672     writer.end_variable()
673
674
675 @node(i18n_tags.BlockTranslateNode)
676 def translate_block(writer, node):
677     first_var = []
678     variables = set()
679
680     def touch_var(name):
681         variables.add(name)
682         if not first_var:
683             first_var.append(name)
684
685     def dump_token_list(tokens):
686         for token in tokens:
687             if token.token_type == TOKEN_TEXT:
688                 writer.write(token.contents)
689             elif token.token_type == TOKEN_VAR:
690                 writer.print_expr(token.contents)
691                 touch_var(token.contents)
692
693     writer.warn('i18n system used, make sure to install translations', node)
694     writer.start_block()
695     writer.write('trans')
696     idx = -1
697     for idx, (key, var) in enumerate(node.extra_context.items()):
698         if idx:
699             writer.write(',')
700         writer.write(' %s=' % key)
701         touch_var(key)
702         writer.node(var.filter_expression)
703
704     have_plural = False
705     plural_var = None
706     if node.plural and node.countervar and node.counter:
707         have_plural = True
708         plural_var = node.countervar
709         if plural_var not in variables:
710             if idx > -1:
711                 writer.write(',')
712             touch_var(plural_var)
713             writer.write(' %s=' % plural_var)
714             writer.node(node.counter)
715
716     writer.end_block()
717     dump_token_list(node.singular)
718     if node.plural and node.countervar and node.counter:
719         writer.start_block()
720         writer.write('pluralize')
721         if node.countervar != first_var[0]:
722             writer.write(' ' + node.countervar)
723         writer.end_block()
724         dump_token_list(node.plural)
725     writer.tag('endtrans')
726
727 @node("SimpleNode")
728 def simple_tag(writer, node):
729     """Check if the simple tag exist as a filter in """
730     name = node.tag_name
731     if writer.env and \
732        name not in writer.env.filters and \
733        name not in writer._filters_warned:
734         writer._filters_warned.add(name)
735         writer.warn('Filter %s probably doesn\'t exist in Jinja' %
736                     name)
737         
738     if not node.vars_to_resolve:
739         # No argument, pass the request
740         writer.start_variable()
741         writer.write('request|')
742         writer.write(name)
743         writer.end_variable()
744         return 
745     
746     first_var =  node.vars_to_resolve[0]
747     args = node.vars_to_resolve[1:]
748     writer.start_variable()
749     
750     # Copied from Writer.filters()
751     writer.node(first_var)
752     
753     writer.write('|')
754     writer.write(name)
755     if args:
756         writer.write('(')
757         for idx, var in enumerate(args):
758             if idx:
759                 writer.write(', ')
760             if var.var:
761                 writer.node(var)
762             else:
763                 writer.literal(var.literal)
764         writer.write(')')
765     writer.end_variable()   
766
767 # get rid of node now, it shouldn't be used normally
768 del node