Added support for variable name translations in django2jinja
[jinja2.git] / ext / django2jinja / django2jinja.py
1 # -*- coding: utf-8 -*-
2 """
3     Django to Jinja
4     ~~~~~~~~~~~~~~~
5
6     Helper module that can convert django templates into Jinja2 templates.
7
8     This file is not intended to be used as stand alone application but to
9     be used as library.  To convert templates you basically create your own
10     writer, add extra conversion logic for your custom template tags,
11     configure your django environment and run the `convert_templates`
12     function.
13
14     Here a simple example::
15
16         # configure django (or use settings.configure)
17         import os
18         os.environ['DJANGO_SETTINGS_MODULE'] = 'yourapplication.settings'
19         from yourapplication.foo.templatetags.bar import MyNode
20
21         from django2jinja import Writer, convert_templates
22
23         def write_my_node(writer, node):
24             writer.start_variable()
25             writer.write('myfunc(')
26             for idx, arg in enumerate(node.args):
27                 if idx:
28                     writer.write(', ')
29                 writer.node(arg)
30             writer.write(')')
31             writer.end_variable()
32
33         writer = Writer()
34         writer.node_handlers[MyNode] = write_my_node
35         convert_templates('/path/to/output/folder', writer=writer)
36     
37     Here is an example hos to automatically translate your django
38     variables to jinja2::
39         
40         import re
41         # List of tuple (Match pattern, Replace pattern, Exclusion pattern)
42         
43         var_re  = ((re.compile(r"(u|user)\.is_authenticated"), r"\1.is_authenticated()", None),
44                   (re.compile(r"\.non_field_errors"), r".non_field_errors()", None),
45                   (re.compile(r"\.label_tag"), r".label_tag()", None),
46                   (re.compile(r"\.as_dl"), r".as_dl()", None),
47                   (re.compile(r"\.as_table"), r".as_table()", None),
48                   (re.compile(r"\.as_widget"), r".as_widget()", None),
49                   (re.compile(r"\.as_hidden"), r".as_hidden()", None),
50                   
51                   (re.compile(r"\.get_([0-9_\w]+)_url"), r".get_\1_url()", None),
52                   (re.compile(r"\.url"), r".url()", re.compile(r"(form|calendar).url")),
53                   (re.compile(r"\.get_([0-9_\w]+)_display"), r".get_\1_display()", None),
54                   (re.compile(r"loop\.counter"), r"loop.index", None),
55                   (re.compile(r"loop\.revcounter"), r"loop.revindex", None),
56                   (re.compile(r"request\.GET\.([0-9_\w]+)"), r"request.GET.get('\1', '')", None),
57                   (re.compile(r"request\.get_host"), r"request.get_host()", None),
58                   
59                   (re.compile(r"\.all(?!_)"), r".all()", None),
60                   (re.compile(r"\.all\.0"), r".all()[0]", None),
61                   (re.compile(r"\.([0-9])($|\s+)"), r"[\1]\2", None),
62                   (re.compile(r"\.items"), r".items()", None),
63         )
64         writer = Writer(var_re=var_re)
65         
66     For details about the writing process have a look at the module code.
67
68     :copyright: Copyright 2008 by Armin Ronacher.
69     :license: BSD.
70 """
71 import re
72 import os
73 import sys
74 from jinja2.defaults import *
75 from django.conf import settings
76 from django.template import defaulttags as core_tags, loader, TextNode, \
77      FilterExpression, libraries, Variable, loader_tags, TOKEN_TEXT, \
78      TOKEN_VAR
79 from django.template.debug import DebugVariableNode as VariableNode
80 from django.templatetags import i18n as i18n_tags
81 from StringIO import StringIO
82
83
84 _node_handlers = {}
85 _resolved_filters = {}
86 _newline_re = re.compile(r'(?:\r\n|\r|\n)')
87
88
89 # don't ask....
90 _old_cycle_init = core_tags.CycleNode.__init__
91 def _fixed_cycle_init(self, cyclevars, variable_name=None):
92     self.raw_cycle_vars = map(Variable, cyclevars)
93     _old_cycle_init(self, cyclevars, variable_name)
94 core_tags.CycleNode.__init__ = _fixed_cycle_init
95
96
97 def node(cls):
98     def proxy(f):
99         _node_handlers[cls] = f
100         return f
101     return proxy
102
103
104 def convert_templates(output_dir, extensions=('.html', '.txt'), writer=None,
105                       callback=None):
106     """Iterates over all templates in the template dirs configured and
107     translates them and writes the new templates into the output directory.
108     """
109     if writer is None:
110         writer = Writer()
111
112     def filter_templates(files):
113         for filename in files:
114             ifilename = filename.lower()
115             for extension in extensions:
116                 if ifilename.endswith(extension):
117                     yield filename
118
119     def translate(f, loadname):
120         template = loader.get_template(loadname)
121         original = writer.stream
122         writer.stream = f
123         writer.body(template.nodelist)
124         writer.stream = original
125
126     if callback is None:
127         def callback(template):
128             print template
129
130     for directory in settings.TEMPLATE_DIRS:
131         for dirname, _, files in os.walk(directory):
132             dirname = dirname[len(directory) + 1:]
133             for filename in filter_templates(files):
134                 source = os.path.normpath(os.path.join(dirname, filename))
135                 target = os.path.join(output_dir, dirname, filename)
136                 basetarget = os.path.dirname(target)
137                 if not os.path.exists(basetarget):
138                     os.makedirs(basetarget)
139                 callback(source)
140                 f = file(target, 'w')
141                 try:
142                     translate(f, source)
143                 finally:
144                     f.close()
145
146
147 class Writer(object):
148     """The core writer class."""
149
150     def __init__(self, stream=None, error_stream=None,
151                  block_start_string=BLOCK_START_STRING,
152                  block_end_string=BLOCK_END_STRING,
153                  variable_start_string=VARIABLE_START_STRING,
154                  variable_end_string=VARIABLE_END_STRING,
155                  comment_start_string=COMMENT_START_STRING,
156                  comment_end_string=COMMENT_END_STRING,
157                  initial_autoescape=True,
158                  use_jinja_autoescape=False,
159                  custom_node_handlers=None,
160                  var_re=[],
161                  env=None):
162         if stream is None:
163             stream = sys.stdout
164         if error_stream is None:
165             error_stream = sys.stderr
166         self.stream = stream
167         self.error_stream = error_stream
168         self.block_start_string = block_start_string
169         self.block_end_string = block_end_string
170         self.variable_start_string = variable_start_string
171         self.variable_end_string = variable_end_string
172         self.comment_start_string = comment_start_string
173         self.comment_end_string = comment_end_string
174         self.autoescape = initial_autoescape
175         self.spaceless = False
176         self.use_jinja_autoescape = use_jinja_autoescape
177         self.node_handlers = dict(_node_handlers,
178                                   **(custom_node_handlers or {}))
179         self._loop_depth = 0
180         self._filters_warned = set()
181         self.var_re = var_re
182         self.env = env
183
184     def enter_loop(self):
185         """Increments the loop depth so that write functions know if they
186         are in a loop.
187         """
188         self._loop_depth += 1
189
190     def leave_loop(self):
191         """Reverse of enter_loop."""
192         self._loop_depth -= 1
193
194     @property
195     def in_loop(self):
196         """True if we are in a loop."""
197         return self._loop_depth > 0
198
199     def write(self, s):
200         """Writes stuff to the stream."""
201         self.stream.write(s.encode(settings.FILE_CHARSET))
202
203     def print_expr(self, expr):
204         """Open a variable tag, write to the string to the stream and close."""
205         self.start_variable()
206         self.write(expr)
207         self.end_variable()
208
209     def _post_open(self):
210         if self.spaceless:
211             self.write('- ')
212         else:
213             self.write(' ')
214
215     def _pre_close(self):
216         if self.spaceless:
217             self.write(' -')
218         else:
219             self.write(' ')
220
221     def start_variable(self):
222         """Start a variable."""
223         self.write(self.variable_start_string)
224         self._post_open()
225
226     def end_variable(self, always_safe=False):
227         """End a variable."""
228         if not always_safe and self.autoescape and \
229            not self.use_jinja_autoescape:
230             self.write('|e')
231         self._pre_close()
232         self.write(self.variable_end_string)
233
234     def start_block(self):
235         """Starts a block."""
236         self.write(self.block_start_string)
237         self._post_open()
238
239     def end_block(self):
240         """Ends a block."""
241         self._pre_close()
242         self.write(self.block_end_string)
243
244     def tag(self, name):
245         """Like `print_expr` just for blocks."""
246         self.start_block()
247         self.write(name)
248         self.end_block()
249
250     def variable(self, name):
251         """Prints a variable.  This performs variable name transformation."""
252         self.write(self.translate_variable_name(name))
253
254     def literal(self, value):
255         """Writes a value as literal."""
256         value = repr(value)
257         if value[:2] in ('u"', "u'"):
258             value = value[1:]
259         self.write(value)
260
261     def filters(self, filters, is_block=False):
262         """Dumps a list of filters."""
263         want_pipe = not is_block
264         for filter, args in filters:
265             name = self.get_filter_name(filter)
266             if name is None:
267                 self.warn('Could not find filter %s' % name)
268                 continue
269             if name not in DEFAULT_FILTERS and \
270                name not in self._filters_warned:
271                 self._filters_warned.add(name)
272                 self.warn('Filter %s probably doesn\'t exist in Jinja' %
273                             name)
274             if not want_pipe:
275                 want_pipe = True
276             else:
277                 self.write('|')
278             self.write(name)
279             if args:
280                 self.write('(')
281                 for idx, (is_var, value) in enumerate(args):
282                     if idx:
283                         self.write(', ')
284                     if is_var:
285                         self.node(value)
286                     else:
287                         self.literal(value)
288                 self.write(')')
289
290     def get_location(self, origin, position):
291         """Returns the location for an origin and position tuple as name
292         and lineno.
293         """
294         if hasattr(origin, 'source'):
295             source = origin.source
296             name = '<unknown source>'
297         else:
298             source = origin.loader(origin.loadname, origin.dirs)[0]
299             name = origin.loadname
300         lineno = len(_newline_re.findall(source[:position[0]])) + 1
301         return name, lineno
302
303     def warn(self, message, node=None):
304         """Prints a warning to the error stream."""
305         if node is not None and hasattr(node, 'source'):
306             filename, lineno = self.get_location(*node.source)
307             message = '[%s:%d] %s' % (filename, lineno, message)
308         print >> self.error_stream, message
309
310     def translate_variable_name(self, var):
311         """Performs variable name translation."""
312         if self.in_loop and var == 'forloop' or var.startswith('forloop.'):
313             var = var[3:]
314         
315         for reg, rep, unless in self.var_re:
316             no_unless = unless and unless.search(var) or True
317             if reg.search(var) and no_unless:
318                 var = reg.sub(rep, var)
319                 break
320         return var
321
322     def get_filter_name(self, filter):
323         """Returns the filter name for a filter function or `None` if there
324         is no such filter.
325         """
326         if filter not in _resolved_filters:
327             for library in libraries.values():
328                 for key, value in library.filters.iteritems():
329                     _resolved_filters[value] = key
330         return _resolved_filters.get(filter, None)
331
332     def node(self, node):
333         """Invokes the node handler for a node."""
334         for cls, handler in self.node_handlers.iteritems():
335             if type(node) is cls or type(node).__name__ == cls:
336                 handler(self, node)
337                 break
338         else:
339             self.warn('Untranslatable node %s.%s found' % (
340                 node.__module__,
341                 node.__class__.__name__
342             ), node)
343
344     def body(self, nodes):
345         """Calls node() for every node in the iterable passed."""
346         for node in nodes:
347             self.node(node)
348
349
350 @node(TextNode)
351 def text_node(writer, node):
352     writer.write(node.s)
353
354
355 @node(Variable)
356 def variable(writer, node):
357     if node.translate:
358         writer.warn('i18n system used, make sure to install translations', node)
359         writer.write('_(')
360     if node.literal is not None:
361         writer.literal(node.literal)
362     else:
363         writer.variable(node.var)
364     if node.translate:
365         writer.write(')')
366
367
368 @node(VariableNode)
369 def variable_node(writer, node):
370     writer.start_variable()
371     if node.filter_expression.var.var == 'block.super' \
372        and not node.filter_expression.filters:
373         writer.write('super()')
374     else:
375         writer.node(node.filter_expression)
376     writer.end_variable()
377
378
379 @node(FilterExpression)
380 def filter_expression(writer, node):
381     writer.node(node.var)
382     writer.filters(node.filters)
383
384
385 @node(core_tags.CommentNode)
386 def comment_tag(writer, node):
387     pass
388
389
390 @node(core_tags.DebugNode)
391 def comment_tag(writer, node):
392     writer.warn('Debug tag detected.  Make sure to add a global function '
393                 'called debug to the namespace.', node=node)
394     writer.print_expr('debug()')
395
396
397 @node(core_tags.ForNode)
398 def for_loop(writer, node):
399     writer.start_block()
400     writer.write('for ')
401     for idx, var in enumerate(node.loopvars):
402         if idx:
403             writer.write(', ')
404         writer.variable(var)
405     writer.write(' in ')
406     if node.is_reversed:
407         writer.write('(')
408     writer.node(node.sequence)
409     if node.is_reversed:
410         writer.write(')|reverse')
411     writer.end_block()
412     writer.enter_loop()
413     writer.body(node.nodelist_loop)
414     writer.leave_loop()
415     writer.tag('endfor')
416
417
418 @node(core_tags.IfNode)
419 def if_condition(writer, node):
420     writer.start_block()
421     writer.write('if ')
422     join_with = 'and'
423     if node.link_type == core_tags.IfNode.LinkTypes.or_:
424         join_with = 'or'
425     
426     for idx, (ifnot, expr) in enumerate(node.bool_exprs):
427         if idx:
428             writer.write(' %s ' % join_with)
429         if ifnot:
430             writer.write('not ')
431         writer.node(expr)
432     writer.end_block()
433     writer.body(node.nodelist_true)
434     if node.nodelist_false:
435         writer.tag('else')
436         writer.body(node.nodelist_false)
437     writer.tag('endif')
438
439
440 @node(core_tags.IfEqualNode)
441 def if_equal(writer, node):
442     writer.start_block()
443     writer.write('if ')
444     writer.node(node.var1)
445     if node.negate:
446         writer.write(' != ')
447     else:
448         writer.write(' == ')
449     writer.node(node.var2)
450     writer.end_block()
451     writer.body(node.nodelist_true)
452     if node.nodelist_false:
453         writer.tag('else')
454         writer.body(node.nodelist_false)
455     writer.tag('endif')
456
457
458 @node(loader_tags.BlockNode)
459 def block(writer, node):
460     writer.tag('block ' + node.name.replace('-', '_').rstrip('_'))
461     node = node
462     while node.parent is not None:
463         node = node.parent
464     writer.body(node.nodelist)
465     writer.tag('endblock')
466
467
468 @node(loader_tags.ExtendsNode)
469 def extends(writer, node):
470     writer.start_block()
471     writer.write('extends ')
472     if node.parent_name_expr:
473         writer.node(node.parent_name_expr)
474     else:
475         writer.literal(node.parent_name)
476     writer.end_block()
477     writer.body(node.nodelist)
478
479
480 @node(loader_tags.ConstantIncludeNode)
481 @node(loader_tags.IncludeNode)
482 def include(writer, node):
483     writer.start_block()
484     writer.write('include ')
485     if hasattr(node, 'template'):
486         writer.literal(node.template.name)
487     else:
488         writer.node(node.template_name)
489     writer.end_block()
490
491
492 @node(core_tags.CycleNode)
493 def cycle(writer, node):
494     if not writer.in_loop:
495         writer.warn('Untranslatable free cycle (cycle outside loop)', node=node)
496         return
497     if node.variable_name is not None:
498         writer.start_block()
499         writer.write('set %s = ' % node.variable_name)
500     else:
501         writer.start_variable()
502     writer.write('loop.cycle(')
503     for idx, var in enumerate(node.raw_cycle_vars):
504         if idx:
505             writer.write(', ')
506         writer.node(var)
507     writer.write(')')
508     if node.variable_name is not None:
509         writer.end_block()
510     else:
511         writer.end_variable()
512
513
514 @node(core_tags.FilterNode)
515 def filter(writer, node):
516     writer.start_block()
517     writer.write('filter ')
518     writer.filters(node.filter_expr.filters, True)
519     writer.end_block()
520     writer.body(node.nodelist)
521     writer.tag('endfilter')
522
523
524 @node(core_tags.AutoEscapeControlNode)
525 def autoescape_control(writer, node):
526     original = writer.autoescape
527     writer.autoescape = node.setting
528     writer.body(node.nodelist)
529     writer.autoescape = original
530
531
532 @node(core_tags.SpacelessNode)
533 def spaceless(writer, node):
534     original = writer.spaceless
535     writer.spaceless = True
536     writer.warn('entering spaceless mode with different semantics', node)
537     # do the initial stripping
538     nodelist = list(node.nodelist)
539     if nodelist:
540         if isinstance(nodelist[0], TextNode):
541             nodelist[0] = TextNode(nodelist[0].s.lstrip())
542         if isinstance(nodelist[-1], TextNode):
543             nodelist[-1] = TextNode(nodelist[-1].s.rstrip())
544     writer.body(nodelist)
545     writer.spaceless = original
546
547
548 @node(core_tags.TemplateTagNode)
549 def template_tag(writer, node):
550     tag = {
551         'openblock':            writer.block_start_string,
552         'closeblock':           writer.block_end_string,
553         'openvariable':         writer.variable_start_string,
554         'closevariable':        writer.variable_end_string,
555         'opencomment':          writer.comment_start_string,
556         'closecomment':         writer.comment_end_string,
557         'openbrace':            '{',
558         'closebrace':           '}'
559     }.get(node.tagtype)
560     if tag:
561         writer.start_variable()
562         writer.literal(tag)
563         writer.end_variable()
564
565
566 @node(core_tags.URLNode)
567 def url_tag(writer, node):
568     writer.warn('url node used.  make sure to provide a proper url() '
569                 'function', node)
570     if node.asvar:
571         writer.start_block()
572         writer.write('set %s = ' % node.asvar)
573     else:
574         writer.start_variable()
575     autoescape = writer.autoescape
576     writer.write('url(')
577     writer.literal(node.view_name)
578     for arg in node.args:
579         writer.write(', ')
580         writer.node(arg)
581     for key, arg in node.kwargs.items():
582         writer.write(', %s=' % key)
583         writer.node(arg)
584     writer.write(')')
585     if node.asvar:
586         writer.end_block()
587     else:
588         writer.end_variable()
589
590
591 @node(core_tags.WidthRatioNode)
592 def width_ratio(writer, node):
593     writer.warn('widthratio expanded into formula.  You may want to provide '
594                 'a helper function for this calculation', node)
595     writer.start_variable()
596     writer.write('(')
597     writer.node(node.val_expr)
598     writer.write(' / ')
599     writer.node(node.max_expr)
600     writer.write(' * ')
601     writer.write(str(int(node.max_width)))
602     writer.write(')|round|int')
603     writer.end_variable(always_safe=True)
604
605
606 @node(core_tags.WithNode)
607 def with_block(writer, node):
608     writer.warn('with block expanded into set statement.  This could cause '
609                 'variables following that block to be overriden.', node)
610     writer.start_block()
611     writer.write('set %s = ' % node.name)
612     writer.node(node.var)
613     writer.end_block()
614     writer.body(node.nodelist)
615
616
617 @node(core_tags.RegroupNode)
618 def regroup(writer, node):
619     if node.expression.var.literal:
620         writer.warn('literal in groupby filter used.   Behavior in that '
621                     'situation is undefined and translation is skipped.', node)
622         return
623     elif node.expression.filters:
624         writer.warn('filters in groupby filter used.   Behavior in that '
625                     'situation is undefined which is most likely a bug '
626                     'in your code.  Filters were ignored.', node)
627     writer.start_block()
628     writer.write('set %s = ' % node.var_name)
629     writer.node(node.target)
630     writer.write('|groupby(')
631     writer.literal(node.expression.var.var)
632     writer.write(')')
633     writer.end_block()
634
635
636 @node(core_tags.LoadNode)
637 def warn_load(writer, node):
638     writer.warn('load statement used which was ignored on conversion', node)
639
640
641 @node(i18n_tags.GetAvailableLanguagesNode)
642 def get_available_languages(writer, node):
643     writer.warn('make sure to provide a get_available_languages function', node)
644     writer.tag('set %s = get_available_languages()' %
645                writer.translate_variable_name(node.variable))
646
647
648 @node(i18n_tags.GetCurrentLanguageNode)
649 def get_current_language(writer, node):
650     writer.warn('make sure to provide a get_current_language function', node)
651     writer.tag('set %s = get_current_language()' %
652                writer.translate_variable_name(node.variable))
653
654
655 @node(i18n_tags.GetCurrentLanguageBidiNode)
656 def get_current_language_bidi(writer, node):
657     writer.warn('make sure to provide a get_current_language_bidi function', node)
658     writer.tag('set %s = get_current_language_bidi()' %
659                writer.translate_variable_name(node.variable))
660
661
662 @node(i18n_tags.TranslateNode)
663 def simple_gettext(writer, node):
664     writer.warn('i18n system used, make sure to install translations', node)
665     writer.start_variable()
666     writer.write('_(')
667     writer.node(node.value)
668     writer.write(')')
669     writer.end_variable()
670
671
672 @node(i18n_tags.BlockTranslateNode)
673 def translate_block(writer, node):
674     first_var = []
675     variables = set()
676
677     def touch_var(name):
678         variables.add(name)
679         if not first_var:
680             first_var.append(name)
681
682     def dump_token_list(tokens):
683         for token in tokens:
684             if token.token_type == TOKEN_TEXT:
685                 writer.write(token.contents)
686             elif token.token_type == TOKEN_VAR:
687                 writer.print_expr(token.contents)
688                 touch_var(token.contents)
689
690     writer.warn('i18n system used, make sure to install translations', node)
691     writer.start_block()
692     writer.write('trans')
693     idx = -1
694     for idx, (key, var) in enumerate(node.extra_context.items()):
695         if idx:
696             writer.write(',')
697         writer.write(' %s=' % key)
698         touch_var(key)
699         writer.node(var.filter_expression)
700
701     have_plural = False
702     plural_var = None
703     if node.plural and node.countervar and node.counter:
704         have_plural = True
705         plural_var = node.countervar
706         if plural_var not in variables:
707             if idx > -1:
708                 writer.write(',')
709             touch_var(plural_var)
710             writer.write(' %s=' % plural_var)
711             writer.node(node.counter)
712
713     writer.end_block()
714     dump_token_list(node.singular)
715     if node.plural and node.countervar and node.counter:
716         writer.start_block()
717         writer.write('pluralize')
718         if node.countervar != first_var[0]:
719             writer.write(' ' + node.countervar)
720         writer.end_block()
721         dump_token_list(node.plural)
722     writer.tag('endtrans')
723
724 @node("SimpleNode")
725 def simple_tag(writer, node):
726     """Check if the simple tag exist as a filter in """
727     name = node.tag_name
728     if writer.env and \
729        name not in writer.env.filters and \
730        name not in writer._filters_warned:
731         writer._filters_warned.add(name)
732         writer.warn('Filter %s probably doesn\'t exist in Jinja' %
733                     name)
734         
735     if not node.vars_to_resolve:
736         # No argument, pass the request
737         writer.start_variable()
738         writer.write('request|')
739         writer.write(name)
740         writer.end_variable()
741         return 
742     
743     first_var =  node.vars_to_resolve[0]
744     args = node.vars_to_resolve[1:]
745     writer.start_variable()
746     
747     # Copied from Writer.filters()
748     writer.node(first_var)
749     
750     writer.write('|')
751     writer.write(name)
752     if args:
753         writer.write('(')
754         for idx, var in enumerate(args):
755             if idx:
756                 writer.write(', ')
757             if var.var:
758                 writer.node(var)
759             else:
760                 writer.literal(var.literal)
761         writer.write(')')
762     writer.end_variable()   
763
764 # get rid of node now, it shouldn't be used normally
765 del node