3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
17 from Cython.Compiler.ModuleNode import ModuleNode
18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
20 from Cython.Compiler.StringEncoding import EncodedString
21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
26 class NameNodeCollector(TreeVisitor):
27 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
31 super(NameNodeCollector, self).__init__()
34 def visit_NameNode(self, node):
35 self.name_nodes.append(node)
37 def visit_Node(self, node):
38 self._visitchildren(node, None)
41 class SkipDeclarations(object):
43 Variable and function declarations can often have a deep tree structure,
44 and yet most transformations don't need to descend to this depth.
46 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
47 is no need to use this for transformations after that point.
49 def visit_CTypeDefNode(self, node):
52 def visit_CVarDefNode(self, node):
55 def visit_CDeclaratorNode(self, node):
58 def visit_CBaseTypeNode(self, node):
61 def visit_CEnumDefNode(self, node):
64 def visit_CStructOrUnionDefNode(self, node):
67 class NormalizeTree(CythonTransform):
69 This transform fixes up a few things after parsing
70 in order to make the parse tree more suitable for
73 a) After parsing, blocks with only one statement will
74 be represented by that statement, not by a StatListNode.
75 When doing transforms this is annoying and inconsistent,
76 as one cannot in general remove a statement in a consistent
77 way and so on. This transform wraps any single statements
78 in a StatListNode containing a single statement.
80 b) The PassStatNode is a noop and serves no purpose beyond
81 plugging such one-statement blocks; i.e., once parsed a
82 ` "pass" can just as well be represented using an empty
83 StatListNode. This means less special cases to worry about
84 in subsequent transforms (one always checks to see if a
85 StatListNode has no children to see if the block is empty).
88 def __init__(self, context):
89 super(NormalizeTree, self).__init__(context)
90 self.is_in_statlist = False
91 self.is_in_expr = False
93 def visit_ExprNode(self, node):
94 stacktmp = self.is_in_expr
95 self.is_in_expr = True
96 self.visitchildren(node)
97 self.is_in_expr = stacktmp
100 def visit_StatNode(self, node, is_listcontainer=False):
101 stacktmp = self.is_in_statlist
102 self.is_in_statlist = is_listcontainer
103 self.visitchildren(node)
104 self.is_in_statlist = stacktmp
105 if not self.is_in_statlist and not self.is_in_expr:
106 return Nodes.StatListNode(pos=node.pos, stats=[node])
110 def visit_StatListNode(self, node):
111 self.is_in_statlist = True
112 self.visitchildren(node)
113 self.is_in_statlist = False
116 def visit_ParallelAssignmentNode(self, node):
117 return self.visit_StatNode(node, True)
119 def visit_CEnumDefNode(self, node):
120 return self.visit_StatNode(node, True)
122 def visit_CStructOrUnionDefNode(self, node):
123 return self.visit_StatNode(node, True)
125 # Eliminate PassStatNode
126 def visit_PassStatNode(self, node):
127 if not self.is_in_statlist:
128 return Nodes.StatListNode(pos=node.pos, stats=[])
132 def visit_CDeclaratorNode(self, node):
136 class PostParseError(CompileError): pass
138 # error strings checked by unit tests, so define them
139 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
140 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
141 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
142 class PostParse(ScopeTrackingTransform):
144 Basic interpretation of the parse tree, as well as validity
145 checking that can be done on a very basic level on the parse
146 tree (while still not being a problem with the basic syntax,
150 - Default values to cdef assignments are turned into single
151 assignments following the declaration (everywhere but in class
152 bodies, where they raise a compile error)
154 - Interpret some node structures into Python runtime values.
155 Some nodes take compile-time arguments (currently:
156 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
157 which should be interpreted. This happens in a general way
158 and other steps should be taken to ensure validity.
160 Type arguments cannot be interpreted in this way.
162 - For __cythonbufferdefaults__ the arguments are checked for
165 TemplatedTypeNode has its directives interpreted:
166 Any first positional argument goes into the "dtype" attribute,
167 any "ndim" keyword argument goes into the "ndim" attribute and
168 so on. Also it is checked that the directive combination is valid.
169 - __cythonbufferdefaults__ attributes are parsed and put into the
172 Note: Currently Parsing.py does a lot of interpretation and
173 reorganization that can be refactored into this transform
174 if a more pure Abstract Syntax Tree is wanted.
177 def __init__(self, context):
178 super(PostParse, self).__init__(context)
179 self.specialattribute_handlers = {
180 '__cythonbufferdefaults__' : self.handle_bufferdefaults
183 def visit_ModuleNode(self, node):
184 self.lambda_counter = 1
185 return super(PostParse, self).visit_ModuleNode(node)
187 def visit_LambdaNode(self, node):
188 # unpack a lambda expression into the corresponding DefNode
189 lambda_id = self.lambda_counter
190 self.lambda_counter += 1
191 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
193 body = Nodes.ReturnStatNode(
194 node.result_expr.pos, value = node.result_expr)
195 node.def_node = Nodes.DefNode(
196 node.pos, name=node.name, lambda_name=node.lambda_name,
197 args=node.args, star_arg=node.star_arg,
198 starstar_arg=node.starstar_arg,
200 self.visitchildren(node)
204 def handle_bufferdefaults(self, decl):
205 if not isinstance(decl.default, ExprNodes.DictNode):
206 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
207 self.scope_node.buffer_defaults_node = decl.default
208 self.scope_node.buffer_defaults_pos = decl.pos
210 def visit_CVarDefNode(self, node):
211 # This assumes only plain names and pointers are assignable on
212 # declaration. Also, it makes use of the fact that a cdef decl
213 # must appear before the first use, so we don't have to deal with
214 # "i = 3; cdef int i = i" and can simply move the nodes around.
216 self.visitchildren(node)
219 for decl in node.declarators:
221 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
222 declbase = declbase.base
223 if isinstance(declbase, Nodes.CNameDeclaratorNode):
224 if declbase.default is not None:
225 if self.scope_type in ('cclass', 'pyclass', 'struct'):
226 if isinstance(self.scope_node, Nodes.CClassDefNode):
227 handler = self.specialattribute_handlers.get(decl.name)
229 if decl is not declbase:
230 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
232 continue # Remove declaration
233 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
234 first_assignment = self.scope_type != 'module'
235 stats.append(Nodes.SingleAssignmentNode(node.pos,
236 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
237 rhs=declbase.default, first=first_assignment))
238 declbase.default = None
239 newdecls.append(decl)
240 node.declarators = newdecls
242 except PostParseError, e:
243 # An error in a cdef clause is ok, simply remove the declaration
244 # and try to move on to report more errors
245 self.context.nonfatal_error(e)
248 # Split parallel assignments (a,b = b,a) into separate partial
249 # assignments that are executed rhs-first using temps. This
250 # restructuring must be applied before type analysis so that known
251 # types on rhs and lhs can be matched directly. It is required in
252 # the case that the types cannot be coerced to a Python type in
253 # order to assign from a tuple.
255 def visit_SingleAssignmentNode(self, node):
256 self.visitchildren(node)
257 return self._visit_assignment_node(node, [node.lhs, node.rhs])
259 def visit_CascadedAssignmentNode(self, node):
260 self.visitchildren(node)
261 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
263 def _visit_assignment_node(self, node, expr_list):
264 """Flatten parallel assignments into separate single
265 assignments or cascaded assignments.
267 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
268 # no parallel assignments => nothing to do
272 flatten_parallel_assignments(expr_list, expr_list_list)
274 eliminate_rhs_duplicates(expr_list_list, temp_refs)
277 for expr_list in expr_list_list:
278 lhs_list = expr_list[:-1]
280 if len(lhs_list) == 1:
281 node = Nodes.SingleAssignmentNode(rhs.pos,
282 lhs = lhs_list[0], rhs = rhs)
284 node = Nodes.CascadedAssignmentNode(rhs.pos,
285 lhs_list = lhs_list, rhs = rhs)
289 assign_node = nodes[0]
291 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
294 duplicates_and_temps = [ (temp.expression, temp)
295 for temp in temp_refs ]
296 sort_common_subsequences(duplicates_and_temps)
297 for _, temp_ref in duplicates_and_temps[::-1]:
298 assign_node = LetNode(temp_ref, assign_node)
302 def _flatten_sequence(self, seq, result):
304 if arg.is_sequence_constructor:
305 self._flatten_sequence(arg, result)
310 def visit_DelStatNode(self, node):
311 self.visitchildren(node)
312 node.args = self._flatten_sequence(node, [])
316 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
317 """Replace rhs items by LetRefNodes if they appear more than once.
318 Creates a sequence of LetRefNodes that set up the required temps
319 and appends them to ref_node_sequence. The input list is modified
322 seen_nodes = cython.set()
324 def find_duplicates(node):
325 if node.is_literal or node.is_name:
326 # no need to replace those; can't include attributes here
327 # as their access is not necessarily side-effect free
329 if node in seen_nodes:
330 if node not in ref_nodes:
331 ref_node = LetRefNode(node)
332 ref_nodes[node] = ref_node
333 ref_node_sequence.append(ref_node)
336 if node.is_sequence_constructor:
337 for item in node.args:
338 find_duplicates(item)
340 for expr_list in expr_list_list:
346 def substitute_nodes(node):
347 if node in ref_nodes:
348 return ref_nodes[node]
349 elif node.is_sequence_constructor:
350 node.args = list(map(substitute_nodes, node.args))
353 # replace nodes inside of the common subexpressions
354 for node in ref_nodes:
355 if node.is_sequence_constructor:
356 node.args = list(map(substitute_nodes, node.args))
358 # replace common subexpressions on all rhs items
359 for expr_list in expr_list_list:
360 expr_list[-1] = substitute_nodes(expr_list[-1])
362 def sort_common_subsequences(items):
363 """Sort items/subsequences so that all items and subsequences that
364 an item contains appear before the item itself. This is needed
365 because each rhs item must only be evaluated once, so its value
366 must be evaluated first and then reused when packing sequences
369 This implies a partial order, and the sort must be stable to
370 preserve the original order as much as possible, so we use a
371 simple insertion sort (which is very fast for short sequences, the
372 normal case in practice).
374 def contains(seq, x):
378 elif item.is_sequence_constructor and contains(item.args, x):
382 return b.is_sequence_constructor and contains(b.args, a)
384 for pos, item in enumerate(items):
385 key = item[1] # the ResultRefNode which has already been injected into the sequences
387 for i in xrange(pos-1, -1, -1):
388 if lower_than(key, items[i][0]):
391 for i in xrange(pos, new_pos, -1):
392 items[i] = items[i-1]
393 items[new_pos] = item
395 def flatten_parallel_assignments(input, output):
396 # The input is a list of expression nodes, representing the LHSs
397 # and RHS of one (possibly cascaded) assignment statement. For
398 # sequence constructors, rearranges the matching parts of both
399 # sides into a list of equivalent assignments between the
400 # individual elements. This transformation is applied
401 # recursively, so that nested structures get matched as well.
403 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
407 complete_assignments = []
409 rhs_size = len(rhs.args)
410 lhs_targets = [ [] for _ in xrange(rhs_size) ]
411 starred_assignments = []
412 for lhs in input[:-1]:
413 if not lhs.is_sequence_constructor:
415 error(lhs.pos, "starred assignment target must be in a list or tuple")
416 complete_assignments.append(lhs)
418 lhs_size = len(lhs.args)
419 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
420 if starred_targets > 1:
421 error(lhs.pos, "more than 1 starred expression in assignment")
422 output.append([lhs,rhs])
424 elif lhs_size - starred_targets > rhs_size:
425 error(lhs.pos, "need more than %d value%s to unpack"
426 % (rhs_size, (rhs_size != 1) and 's' or ''))
427 output.append([lhs,rhs])
429 elif starred_targets:
430 map_starred_assignment(lhs_targets, starred_assignments,
432 elif lhs_size < rhs_size:
433 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
434 % (lhs_size, rhs_size))
435 output.append([lhs,rhs])
438 for targets, expr in zip(lhs_targets, lhs.args):
441 if complete_assignments:
442 complete_assignments.append(rhs)
443 output.append(complete_assignments)
445 # recursively flatten partial assignments
446 for cascade, rhs in zip(lhs_targets, rhs.args):
449 flatten_parallel_assignments(cascade, output)
451 # recursively flatten starred assignments
452 for cascade in starred_assignments:
453 if cascade[0].is_sequence_constructor:
454 flatten_parallel_assignments(cascade, output)
456 output.append(cascade)
458 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
459 # Appends the fixed-position LHS targets to the target list that
460 # appear left and right of the starred argument.
462 # The starred_assignments list receives a new tuple
463 # (lhs_target, rhs_values_list) that maps the remaining arguments
464 # (those that match the starred target) to a list.
466 # left side of the starred target
467 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
470 lhs_remaining = len(lhs_args) - i - 1
474 raise InternalError("no starred arg found when splitting starred assignment")
476 # right side of the starred target
477 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
478 lhs_args[starred + 1:])):
481 # the starred target itself, must be assigned a (potentially empty) list
482 target = lhs_args[starred].target # unpack starred node
483 starred_rhs = rhs_args[starred:]
485 starred_rhs = starred_rhs[:-lhs_remaining]
487 pos = starred_rhs[0].pos
490 starred_assignments.append([
491 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
494 class PxdPostParse(CythonTransform, SkipDeclarations):
496 Basic interpretation/validity checking that should only be
499 A lot of this checking currently happens in the parser; but
500 what is listed below happens here.
502 - "def" functions are let through only if they fill the
503 getbuffer/releasebuffer slots
505 - cdef functions are let through only if they are on the
506 top level and are declared "inline"
508 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
509 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
511 def __call__(self, node):
512 self.scope_type = 'pxd'
513 return super(PxdPostParse, self).__call__(node)
515 def visit_CClassDefNode(self, node):
516 old = self.scope_type
517 self.scope_type = 'cclass'
518 self.visitchildren(node)
519 self.scope_type = old
522 def visit_FuncDefNode(self, node):
523 # FuncDefNode always come with an implementation (without
524 # an imp they are CVarDefNodes..)
525 err = self.ERR_INLINE_ONLY
527 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
528 and node.name in ('__getbuffer__', '__releasebuffer__')):
529 err = None # allow these slots
531 if isinstance(node, Nodes.CFuncDefNode):
532 if u'inline' in node.modifiers and self.scope_type == 'pxd':
533 node.inline_in_pxd = True
534 if node.visibility != 'private':
535 err = self.ERR_NOGO_WITH_INLINE % node.visibility
537 err = self.ERR_NOGO_WITH_INLINE % 'api'
539 err = None # allow inline function
541 err = self.ERR_INLINE_ONLY
544 self.context.nonfatal_error(PostParseError(node.pos, err))
549 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
551 After parsing, directives can be stored in a number of places:
552 - #cython-comments at the top of the file (stored in ModuleNode)
553 - Command-line arguments overriding these
554 - @cython.directivename decorators
555 - with cython.directivename: statements
557 This transform is responsible for interpreting these various sources
558 and store the directive in two ways:
559 - Set the directives attribute of the ModuleNode for global directives.
560 - Use a CompilerDirectivesNode to override directives for a subtree.
562 (The first one is primarily to not have to modify with the tree
563 structure, so that ModuleNode stay on top.)
565 The directives are stored in dictionaries from name to value in effect.
566 Each such dictionary is always filled in for all possible directives,
567 using default values where no value is given by the user.
569 The available directives are controlled in Options.py.
571 Note that we have to run this prior to analysis, and so some minor
572 duplication of functionality has to occur: We manually track cimports
573 and which names the "cython" module may have been imported to.
575 unop_method_nodes = {
576 'typeof': ExprNodes.TypeofNode,
578 'operator.address': ExprNodes.AmpersandNode,
579 'operator.dereference': ExprNodes.DereferenceNode,
580 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
581 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
582 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
583 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
585 # For backwards compatability.
586 'address': ExprNodes.AmpersandNode,
589 binop_method_nodes = {
590 'operator.comma' : ExprNodes.c_binop_constructor(','),
593 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
594 'cast', 'pointer', 'compiled', 'NULL'])
595 special_methods.update(unop_method_nodes.keys())
597 def __init__(self, context, compilation_directive_defaults):
598 super(InterpretCompilerDirectives, self).__init__(context)
599 self.compilation_directive_defaults = {}
600 for key, value in compilation_directive_defaults.items():
601 self.compilation_directive_defaults[unicode(key)] = value
602 self.cython_module_names = cython.set()
603 self.directive_names = {}
605 def check_directive_scope(self, pos, directive, scope):
606 legal_scopes = Options.directive_scopes.get(directive, None)
607 if legal_scopes and scope not in legal_scopes:
608 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
609 'is not allowed in %s scope' % (directive, scope)))
614 # Set up processing and handle the cython: comments.
615 def visit_ModuleNode(self, node):
616 for key, value in node.directive_comments.items():
617 if not self.check_directive_scope(node.pos, key, 'module'):
618 self.wrong_scope_error(node.pos, key, 'module')
619 del node.directive_comments[key]
621 directives = copy.copy(Options.directive_defaults)
622 directives.update(self.compilation_directive_defaults)
623 directives.update(node.directive_comments)
624 self.directives = directives
625 node.directives = directives
626 self.visitchildren(node)
627 node.cython_module_names = self.cython_module_names
630 # The following four functions track imports and cimports that
631 # begin with "cython"
632 def is_cython_directive(self, name):
633 return (name in Options.directive_types or
634 name in self.special_methods or
635 PyrexTypes.parse_basic_type(name))
637 def visit_CImportStatNode(self, node):
638 if node.module_name == u"cython":
639 self.cython_module_names.add(node.as_name or u"cython")
640 elif node.module_name.startswith(u"cython."):
642 self.directive_names[node.as_name] = node.module_name[7:]
644 self.cython_module_names.add(u"cython")
645 # if this cimport was a compiler directive, we don't
646 # want to leave the cimport node sitting in the tree
650 def visit_FromCImportStatNode(self, node):
651 if (node.module_name == u"cython") or \
652 node.module_name.startswith(u"cython."):
653 submodule = (node.module_name + u".")[7:]
655 for pos, name, as_name, kind in node.imported_names:
656 full_name = submodule + name
657 if self.is_cython_directive(full_name):
660 self.directive_names[as_name] = full_name
662 self.context.nonfatal_error(PostParseError(pos,
663 "Compiler directive imports must be plain imports"))
665 newimp.append((pos, name, as_name, kind))
668 node.imported_names = newimp
671 def visit_FromImportStatNode(self, node):
672 if (node.module.module_name.value == u"cython") or \
673 node.module.module_name.value.startswith(u"cython."):
674 submodule = (node.module.module_name.value + u".")[7:]
676 for name, name_node in node.items:
677 full_name = submodule + name
678 if self.is_cython_directive(full_name):
679 self.directive_names[name_node.name] = full_name
681 newimp.append((name, name_node))
687 def visit_SingleAssignmentNode(self, node):
688 if (isinstance(node.rhs, ExprNodes.ImportNode) and
689 node.rhs.module_name.value == u'cython'):
690 node = Nodes.CImportStatNode(node.pos,
691 module_name = u'cython',
692 as_name = node.lhs.name)
693 self.visit_CImportStatNode(node)
695 self.visitchildren(node)
698 def visit_NameNode(self, node):
699 if node.name in self.cython_module_names:
700 node.is_cython_module = True
702 node.cython_attribute = self.directive_names.get(node.name)
705 def try_to_parse_directives(self, node):
706 # If node is the contents of an directive (in a with statement or
707 # decorator), returns a list of (directivename, value) pairs.
708 # Otherwise, returns None
709 if isinstance(node, ExprNodes.CallNode):
710 self.visit(node.function)
711 optname = node.function.as_cython_attribute()
713 directivetype = Options.directive_types.get(optname)
715 args, kwds = node.explicit_args_kwds()
718 if kwds is not None and directivetype is not dict:
719 for keyvalue in kwds.key_value_pairs:
720 key, value = keyvalue
721 sub_optname = "%s.%s" % (optname, key.value)
722 if Options.directive_types.get(sub_optname):
723 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
725 key_value_pairs.append(keyvalue)
726 if not key_value_pairs:
729 kwds.key_value_pairs = key_value_pairs
730 if directives and not kwds and not args:
732 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
734 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
736 optname = node.as_cython_attribute()
738 directivetype = Options.directive_types.get(optname)
739 if directivetype is bool:
740 return [(optname, True)]
741 elif directivetype is None:
742 return [(optname, None)]
744 raise PostParseError(
745 node.pos, "The '%s' directive should be used as a function call." % optname)
748 def try_to_parse_directive(self, optname, args, kwds, pos):
749 directivetype = Options.directive_types.get(optname)
750 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
751 return optname, Options.directive_defaults[optname]
752 elif directivetype is bool:
753 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
754 raise PostParseError(pos,
755 'The %s directive takes one compile-time boolean argument' % optname)
756 return (optname, args[0].value)
757 elif directivetype is str:
758 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
759 ExprNodes.UnicodeNode)):
760 raise PostParseError(pos,
761 'The %s directive takes one compile-time string argument' % optname)
762 return (optname, str(args[0].value))
763 elif directivetype is dict:
765 raise PostParseError(pos,
766 'The %s directive takes no prepositional arguments' % optname)
767 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
768 elif directivetype is list:
769 if kwds and len(kwds) != 0:
770 raise PostParseError(pos,
771 'The %s directive takes no keyword arguments' % optname)
772 return optname, [ str(arg.value) for arg in args ]
776 def visit_with_directives(self, body, directives):
777 olddirectives = self.directives
778 newdirectives = copy.copy(olddirectives)
779 newdirectives.update(directives)
780 self.directives = newdirectives
781 assert isinstance(body, Nodes.StatListNode), body
782 retbody = self.visit_Node(body)
783 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
784 directives=newdirectives)
785 self.directives = olddirectives
789 def visit_FuncDefNode(self, node):
790 directives = self._extract_directives(node, 'function')
792 return self.visit_Node(node)
793 body = Nodes.StatListNode(node.pos, stats=[node])
794 return self.visit_with_directives(body, directives)
796 def visit_CVarDefNode(self, node):
797 if not node.decorators:
799 for dec in node.decorators:
800 for directive in self.try_to_parse_directives(dec.decorator) or ():
801 if directive is not None and directive[0] == u'locals':
802 node.directive_locals = directive[1]
804 self.context.nonfatal_error(PostParseError(dec.pos,
805 "Cdef functions can only take cython.locals() decorator."))
808 def visit_CClassDefNode(self, node):
809 directives = self._extract_directives(node, 'cclass')
811 return self.visit_Node(node)
812 body = Nodes.StatListNode(node.pos, stats=[node])
813 return self.visit_with_directives(body, directives)
815 def visit_PyClassDefNode(self, node):
816 directives = self._extract_directives(node, 'class')
818 return self.visit_Node(node)
819 body = Nodes.StatListNode(node.pos, stats=[node])
820 return self.visit_with_directives(body, directives)
822 def _extract_directives(self, node, scope_name):
823 if not node.decorators:
825 # Split the decorators into two lists -- real decorators and directives
828 for dec in node.decorators:
829 new_directives = self.try_to_parse_directives(dec.decorator)
830 if new_directives is not None:
831 for directive in new_directives:
832 if self.check_directive_scope(node.pos, directive[0], scope_name):
833 directives.append(directive)
836 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
837 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
839 node.decorators = realdecs
840 # merge or override repeated directives
842 directives.reverse() # Decorators coming first take precedence
843 for directive in directives:
844 name, value = directive
846 old_value = optdict[name]
847 # keywords and arg lists can be merged, everything
848 # else overrides completely
849 if isinstance(old_value, dict):
850 old_value.update(value)
851 elif isinstance(old_value, list):
852 old_value.extend(value)
854 optdict[name] = value
856 optdict[name] = value
859 # Handle with statements
860 def visit_WithStatNode(self, node):
862 for directive in self.try_to_parse_directives(node.manager) or []:
863 if directive is not None:
864 if node.target is not None:
865 self.context.nonfatal_error(
866 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
868 name, value = directive
870 # special case: in pure mode, "with nogil" spells "with cython.nogil"
871 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
872 return self.visit_Node(node)
873 if self.check_directive_scope(node.pos, name, 'with statement'):
874 directive_dict[name] = value
876 return self.visit_with_directives(node.body, directive_dict)
877 return self.visit_Node(node)
879 class WithTransform(CythonTransform, SkipDeclarations):
881 # EXCINFO is manually set to a variable that contains
882 # the exc_info() tuple that can be generated by the enclosing except
884 template_without_target = TreeFragment(u"""
895 if not EXIT(*EXCINFO):
899 EXIT(None, None, None)
900 """, temps=[u'MGR', u'EXC', u"EXIT"],
901 pipeline=[NormalizeTree(None)])
903 template_with_target = TreeFragment(u"""
906 VALUE = MGR.__enter__()
915 if not EXIT(*EXCINFO):
919 EXIT(None, None, None)
920 MGR = EXIT = VALUE = EXC = None
922 """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
923 pipeline=[NormalizeTree(None)])
925 def visit_WithStatNode(self, node):
926 # TODO: Cleanup badly needed
927 TemplateTransform.temp_name_counter += 1
928 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
930 self.visitchildren(node, ['body'])
931 excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
932 if node.target is not None:
933 result = self.template_with_target.substitute({
934 u'EXPR' : node.manager,
936 u'TARGET' : node.target,
937 u'EXCINFO' : excinfo_temp
940 result = self.template_without_target.substitute({
941 u'EXPR' : node.manager,
943 u'EXCINFO' : excinfo_temp
946 # Set except excinfo target to EXCINFO
947 try_except = result.stats[-1].body.stats[-1]
948 try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
949 # excinfo_temp.ref(node.pos))
951 # result.stats[-1].body.stats[-1] = TempsBlockNode(
952 # node.pos, temps=[excinfo_temp], body=try_except)
956 def visit_ExprNode(self, node):
957 # With statements are never inside expressions.
961 class DecoratorTransform(CythonTransform, SkipDeclarations):
963 def visit_DefNode(self, func_node):
964 self.visitchildren(func_node)
965 if not func_node.decorators:
967 return self._handle_decorators(
968 func_node, func_node.name)
970 def visit_CClassDefNode(self, class_node):
971 # This doesn't currently work, so it's disabled.
973 # Problem: assignments to cdef class names do not work. They
974 # would require an additional check anyway, as the extension
975 # type must not change its C type, so decorators cannot
976 # replace an extension type, just alter it and return it.
978 self.visitchildren(class_node)
979 if not class_node.decorators:
981 error(class_node.pos,
982 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
984 #return self._handle_decorators(
985 # class_node, class_node.class_name)
987 def visit_ClassDefNode(self, class_node):
988 self.visitchildren(class_node)
989 if not class_node.decorators:
991 return self._handle_decorators(
992 class_node, class_node.name)
994 def _handle_decorators(self, node, name):
995 decorator_result = ExprNodes.NameNode(node.pos, name = name)
996 for decorator in node.decorators[::-1]:
997 decorator_result = ExprNodes.SimpleCallNode(
999 function = decorator.decorator,
1000 args = [decorator_result])
1002 name_node = ExprNodes.NameNode(node.pos, name = name)
1003 reassignment = Nodes.SingleAssignmentNode(
1006 rhs = decorator_result)
1007 return [node, reassignment]
1010 class AnalyseDeclarationsTransform(CythonTransform):
1012 basic_property = TreeFragment(u"""
1016 def __set__(self, value):
1018 """, level='c_class')
1019 basic_pyobject_property = TreeFragment(u"""
1023 def __set__(self, value):
1027 """, level='c_class')
1028 basic_property_ro = TreeFragment(u"""
1032 """, level='c_class')
1034 struct_or_union_wrapper = TreeFragment(u"""
1037 def __init__(self, MEMBER=None):
1041 if IS_UNION and count > 1:
1042 raise ValueError, "At most one union member should be specified."
1044 return STR_FORMAT % MEMBER_TUPLE
1046 return REPR_FORMAT % MEMBER_TUPLE
1049 init_assignment = TreeFragment(u"""
1050 if VALUE is not None:
1055 def __call__(self, root):
1056 self.env_stack = [root.scope]
1057 # needed to determine if a cdef var is declared after it's used.
1058 self.seen_vars_stack = []
1059 return super(AnalyseDeclarationsTransform, self).__call__(root)
1061 def visit_NameNode(self, node):
1062 self.seen_vars_stack[-1].add(node.name)
1065 def visit_ModuleNode(self, node):
1066 self.seen_vars_stack.append(cython.set())
1067 node.analyse_declarations(self.env_stack[-1])
1068 self.visitchildren(node)
1069 self.seen_vars_stack.pop()
1072 def visit_LambdaNode(self, node):
1073 node.analyse_declarations(self.env_stack[-1])
1074 self.visitchildren(node)
1077 def visit_ClassDefNode(self, node):
1078 self.env_stack.append(node.scope)
1079 self.visitchildren(node)
1080 self.env_stack.pop()
1083 def visit_CClassDefNode(self, node):
1084 node = self.visit_ClassDefNode(node)
1085 if node.scope and node.scope.implemented:
1087 for entry in node.scope.var_entries:
1088 if entry.needs_property:
1089 property = self.create_Property(entry)
1090 property.analyse_declarations(node.scope)
1091 self.visit(property)
1092 stats.append(property)
1094 node.body.stats += stats
1097 def visit_FuncDefNode(self, node):
1098 self.seen_vars_stack.append(cython.set())
1099 lenv = node.local_scope
1100 node.body.analyse_control_flow(lenv) # this will be totally refactored
1101 node.declare_arguments(lenv)
1102 for var, type_node in node.directive_locals.items():
1103 if not lenv.lookup_here(var): # don't redeclare args
1104 type = type_node.analyse_as_type(lenv)
1106 lenv.declare_var(var, type, type_node.pos)
1108 error(type_node.pos, "Not a type")
1109 node.body.analyse_declarations(lenv)
1110 self.env_stack.append(lenv)
1111 self.visitchildren(node)
1112 self.env_stack.pop()
1113 self.seen_vars_stack.pop()
1116 def visit_ScopedExprNode(self, node):
1117 env = self.env_stack[-1]
1118 node.analyse_declarations(env)
1119 # the node may or may not have a local scope
1120 if node.has_local_scope:
1121 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1122 self.env_stack.append(node.expr_scope)
1123 node.analyse_scoped_declarations(node.expr_scope)
1124 self.visitchildren(node)
1125 self.env_stack.pop()
1126 self.seen_vars_stack.pop()
1128 node.analyse_scoped_declarations(env)
1129 self.visitchildren(node)
1132 def visit_TempResultFromStatNode(self, node):
1133 self.visitchildren(node)
1134 node.analyse_declarations(self.env_stack[-1])
1137 def visit_CStructOrUnionDefNode(self, node):
1138 # Create a wrapper node if needed.
1139 # We want to use the struct type information (so it can't happen
1140 # before this phase) but also create new objects to be declared
1141 # (so it can't happen later).
1142 # Note that we don't return the original node, as it is
1143 # never used after this phase.
1144 if True: # private (default)
1147 self_value = ExprNodes.AttributeNode(
1149 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1150 attribute = EncodedString(u"value"))
1151 var_entries = node.entry.type.scope.var_entries
1153 for entry in var_entries:
1154 attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1156 attribute = entry.name))
1157 # __init__ assignments
1158 init_assignments = []
1159 for entry, attr in zip(var_entries, attributes):
1160 # TODO: branch on visibility
1161 init_assignments.append(self.init_assignment.substitute({
1162 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1164 }, pos = entry.pos))
1167 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1168 wrapper_class = self.struct_or_union_wrapper.substitute({
1169 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1170 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1171 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1172 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1173 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1174 }, pos = node.pos).stats[0]
1175 wrapper_class.class_name = node.name
1176 wrapper_class.shadow = True
1177 class_body = wrapper_class.body.stats
1180 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1181 class_body[0].base_type.name = node.name
1183 # fix __init__ arguments
1184 init_method = class_body[1]
1185 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1186 arg_template = init_method.args[1]
1187 if not node.entry.type.is_struct:
1188 arg_template.kw_only = True
1189 del init_method.args[1]
1190 for entry, attr in zip(var_entries, attributes):
1191 arg = copy.deepcopy(arg_template)
1192 arg.declarator.name = entry.name
1193 init_method.args.append(arg)
1196 for entry, attr in zip(var_entries, attributes):
1197 # TODO: branch on visibility
1198 if entry.type.is_pyobject:
1199 template = self.basic_pyobject_property
1201 template = self.basic_property
1202 property = template.substitute({
1204 }, pos = entry.pos).stats[0]
1205 property.name = entry.name
1206 wrapper_class.body.stats.append(property)
1208 wrapper_class.analyse_declarations(self.env_stack[-1])
1209 return self.visit_CClassDefNode(wrapper_class)
1211 # Some nodes are no longer needed after declaration
1212 # analysis and can be dropped. The analysis was performed
1213 # on these nodes in a seperate recursive process from the
1214 # enclosing function or module, so we can simply drop them.
1215 def visit_CDeclaratorNode(self, node):
1216 # necessary to ensure that all CNameDeclaratorNodes are visited.
1217 self.visitchildren(node)
1220 def visit_CTypeDefNode(self, node):
1223 def visit_CBaseTypeNode(self, node):
1226 def visit_CEnumDefNode(self, node):
1227 if node.visibility == 'public':
1232 def visit_CNameDeclaratorNode(self, node):
1233 if node.name in self.seen_vars_stack[-1]:
1234 entry = self.env_stack[-1].lookup(node.name)
1235 if entry is None or entry.visibility != 'extern':
1236 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1237 self.visitchildren(node)
1240 def visit_CVarDefNode(self, node):
1241 # to ensure all CNameDeclaratorNodes are visited.
1242 self.visitchildren(node)
1245 def create_Property(self, entry):
1246 if entry.visibility == 'public':
1247 if entry.type.is_pyobject:
1248 template = self.basic_pyobject_property
1250 template = self.basic_property
1251 elif entry.visibility == 'readonly':
1252 template = self.basic_property_ro
1253 property = template.substitute({
1254 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1255 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1256 attribute=entry.name),
1257 }, pos=entry.pos).stats[0]
1258 property.name = entry.name
1259 # ---------------------------------------
1260 # XXX This should go to AutoDocTransforms
1261 # ---------------------------------------
1262 if (Options.docstrings and
1263 self.current_directives['embedsignature']):
1264 attr_name = entry.name
1265 type_name = entry.type.declaration_code("", for_display=1)
1267 if not entry.type.is_pyobject:
1268 type_name = "'%s'" % type_name
1269 elif entry.type.is_extension_type:
1270 type_name = entry.type.module_name + '.' + type_name
1271 if entry.init is not None:
1272 default_value = ' = ' + entry.init
1273 elif entry.init_to_none:
1274 default_value = ' = ' + repr(None)
1275 docstring = attr_name + ': ' + type_name + default_value
1276 property.doc = EncodedString(docstring)
1277 # ---------------------------------------
1280 class AnalyseExpressionsTransform(CythonTransform):
1282 def visit_ModuleNode(self, node):
1283 node.scope.infer_types()
1284 node.body.analyse_expressions(node.scope)
1285 self.visitchildren(node)
1288 def visit_FuncDefNode(self, node):
1289 node.local_scope.infer_types()
1290 node.body.analyse_expressions(node.local_scope)
1291 self.visitchildren(node)
1294 def visit_ScopedExprNode(self, node):
1295 if node.has_local_scope:
1296 node.expr_scope.infer_types()
1297 node.analyse_scoped_expressions(node.expr_scope)
1298 self.visitchildren(node)
1301 class ExpandInplaceOperators(EnvTransform):
1303 def visit_InPlaceAssignmentNode(self, node):
1306 if lhs.type.is_cpp_class:
1307 # No getting around this exact operator here.
1309 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1310 # There is code to handle this case.
1313 env = self.current_env()
1314 def side_effect_free_reference(node, setting=False):
1315 if isinstance(node, ExprNodes.NameNode):
1317 elif node.type.is_pyobject and not setting:
1318 node = LetRefNode(node)
1320 elif isinstance(node, ExprNodes.IndexNode):
1321 if node.is_buffer_access:
1322 raise ValueError, "Buffer access"
1323 base, temps = side_effect_free_reference(node.base)
1324 index = LetRefNode(node.index)
1325 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1326 elif isinstance(node, ExprNodes.AttributeNode):
1327 obj, temps = side_effect_free_reference(node.obj)
1328 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1330 node = LetRefNode(node)
1333 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1336 dup = lhs.__class__(**lhs.__dict__)
1337 binop = ExprNodes.binop_node(node.pos,
1338 operator = node.operator,
1342 # Manually analyse types for new node.
1343 lhs.analyse_target_types(env)
1344 dup.analyse_types(env)
1345 binop.analyse_operation(env)
1346 node = Nodes.SingleAssignmentNode(
1349 rhs=binop.coerce_to(lhs.type, env))
1350 # Use LetRefNode to avoid side effects.
1351 let_ref_nodes.reverse()
1352 for t in let_ref_nodes:
1353 node = LetNode(t, node)
1356 def visit_ExprNode(self, node):
1357 # In-place assignments can't happen within an expression.
1361 class AlignFunctionDefinitions(CythonTransform):
1363 This class takes the signatures from a .pxd file and applies them to
1364 the def methods in a .py file.
1367 def visit_ModuleNode(self, node):
1368 self.scope = node.scope
1369 self.directives = node.directives
1370 self.visitchildren(node)
1373 def visit_PyClassDefNode(self, node):
1374 pxd_def = self.scope.lookup(node.name)
1376 if pxd_def.is_cclass:
1377 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1379 error(node.pos, "'%s' redeclared" % node.name)
1380 error(pxd_def.pos, "previous declaration here")
1385 def visit_CClassDefNode(self, node, pxd_def=None):
1387 pxd_def = self.scope.lookup(node.class_name)
1389 outer_scope = self.scope
1390 self.scope = pxd_def.type.scope
1391 self.visitchildren(node)
1393 self.scope = outer_scope
1396 def visit_DefNode(self, node):
1397 pxd_def = self.scope.lookup(node.name)
1399 if not pxd_def.is_cfunction:
1400 error(node.pos, "'%s' redeclared" % node.name)
1401 error(pxd_def.pos, "previous declaration here")
1403 node = node.as_cfunction(pxd_def)
1404 elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1405 node = node.as_cfunction(scope=self.scope)
1406 # Enable this when internal def functions are allowed.
1407 # self.visitchildren(node)
1411 class MarkClosureVisitor(CythonTransform):
1413 def visit_ModuleNode(self, node):
1414 self.needs_closure = False
1415 self.visitchildren(node)
1418 def visit_FuncDefNode(self, node):
1419 self.needs_closure = False
1420 self.visitchildren(node)
1421 node.needs_closure = self.needs_closure
1422 self.needs_closure = True
1425 def visit_CFuncDefNode(self, node):
1426 self.visit_FuncDefNode(node)
1427 if node.needs_closure:
1428 error(node.pos, "closures inside cdef functions not yet supported")
1431 def visit_LambdaNode(self, node):
1432 self.needs_closure = False
1433 self.visitchildren(node)
1434 node.needs_closure = self.needs_closure
1435 self.needs_closure = True
1438 def visit_ClassDefNode(self, node):
1439 self.visitchildren(node)
1440 self.needs_closure = True
1444 class CreateClosureClasses(CythonTransform):
1445 # Output closure classes in module scope for all functions
1446 # that really need it.
1448 def __init__(self, context):
1449 super(CreateClosureClasses, self).__init__(context)
1451 self.in_lambda = False
1453 def visit_ModuleNode(self, node):
1454 self.module_scope = node.scope
1455 self.visitchildren(node)
1458 def get_scope_use(self, node):
1461 for name, entry in node.local_scope.entries.items():
1462 if entry.from_closure:
1463 from_closure.append((name, entry))
1464 elif entry.in_closure and not entry.from_closure:
1465 in_closure.append((name, entry))
1466 return from_closure, in_closure
1468 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1469 from_closure, in_closure = self.get_scope_use(node)
1472 # Now from the begining
1473 node.needs_closure = False
1474 node.needs_outer_scope = False
1476 func_scope = node.local_scope
1477 cscope = node.entry.scope
1478 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1479 cscope = cscope.outer_scope
1481 if not from_closure and (self.path or inner_node):
1484 raise InternalError, "DefNode does not have assignment node"
1485 inner_node = node.assmt.rhs
1486 inner_node.needs_self_code = False
1487 node.needs_outer_scope = False
1489 if not in_closure and not from_closure:
1491 elif not in_closure:
1492 func_scope.is_passthrough = True
1493 func_scope.scope_class = cscope.scope_class
1494 node.needs_outer_scope = True
1497 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1499 entry = target_module_scope.declare_c_class(name = as_name,
1500 pos = node.pos, defining = True, implementing = True)
1501 func_scope.scope_class = entry
1502 class_scope = entry.type.scope
1503 class_scope.is_internal = True
1504 class_scope.directives = {'final': True}
1507 assert cscope.is_closure_scope
1508 class_scope.declare_var(pos=node.pos,
1509 name=Naming.outer_scope_cname,
1510 cname=Naming.outer_scope_cname,
1511 type=cscope.scope_class.type,
1513 node.needs_outer_scope = True
1514 for name, entry in in_closure:
1515 class_scope.declare_var(pos=entry.pos,
1520 node.needs_closure = True
1521 # Do it here because other classes are already checked
1522 target_module_scope.check_c_class(func_scope.scope_class)
1524 def visit_LambdaNode(self, node):
1525 was_in_lambda = self.in_lambda
1526 self.in_lambda = True
1527 self.create_class_from_scope(node.def_node, self.module_scope, node)
1528 self.visitchildren(node)
1529 self.in_lambda = was_in_lambda
1532 def visit_FuncDefNode(self, node):
1534 self.visitchildren(node)
1536 if node.needs_closure or self.path:
1537 self.create_class_from_scope(node, self.module_scope)
1538 self.path.append(node)
1539 self.visitchildren(node)
1544 class GilCheck(VisitorTransform):
1546 Call `node.gil_check(env)` on each node to make sure we hold the
1547 GIL when we need it. Raise an error when on Python operations
1548 inside a `nogil` environment.
1550 def __call__(self, root):
1551 self.env_stack = [root.scope]
1553 return super(GilCheck, self).__call__(root)
1555 def visit_FuncDefNode(self, node):
1556 self.env_stack.append(node.local_scope)
1557 was_nogil = self.nogil
1558 self.nogil = node.local_scope.nogil
1559 if self.nogil and node.nogil_check:
1560 node.nogil_check(node.local_scope)
1561 self.visitchildren(node)
1562 self.env_stack.pop()
1563 self.nogil = was_nogil
1566 def visit_GILStatNode(self, node):
1567 env = self.env_stack[-1]
1568 if self.nogil and node.nogil_check: node.nogil_check()
1569 was_nogil = self.nogil
1570 self.nogil = (node.state == 'nogil')
1571 self.visitchildren(node)
1572 self.nogil = was_nogil
1575 def visit_Node(self, node):
1576 if self.env_stack and self.nogil and node.nogil_check:
1577 node.nogil_check(self.env_stack[-1])
1578 self.visitchildren(node)
1582 class TransformBuiltinMethods(EnvTransform):
1584 def visit_SingleAssignmentNode(self, node):
1585 if node.declaration_only:
1588 self.visitchildren(node)
1591 def visit_AttributeNode(self, node):
1592 self.visitchildren(node)
1593 return self.visit_cython_attribute(node)
1595 def visit_NameNode(self, node):
1596 return self.visit_cython_attribute(node)
1598 def visit_cython_attribute(self, node):
1599 attribute = node.as_cython_attribute()
1601 if attribute == u'compiled':
1602 node = ExprNodes.BoolNode(node.pos, value=True)
1603 elif attribute == u'NULL':
1604 node = ExprNodes.NullNode(node.pos)
1605 elif attribute in (u'set', u'frozenset'):
1606 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1607 entry=self.current_env().builtin_scope().lookup_here(attribute))
1608 elif not PyrexTypes.parse_basic_type(attribute):
1609 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1612 def visit_SimpleCallNode(self, node):
1615 if isinstance(node.function, ExprNodes.NameNode):
1616 if node.function.name == 'locals':
1617 lenv = self.current_env()
1618 entry = lenv.lookup_here('locals')
1620 # not the builtin 'locals'
1622 if len(node.args) > 0:
1623 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1626 items = [ ExprNodes.DictItemNode(pos,
1627 key=ExprNodes.StringNode(pos, value=var),
1628 value=ExprNodes.NameNode(pos, name=var))
1629 for var in lenv.entries ]
1630 return ExprNodes.DictNode(pos, key_value_pairs=items)
1633 function = node.function.as_cython_attribute()
1635 if function in InterpretCompilerDirectives.unop_method_nodes:
1636 if len(node.args) != 1:
1637 error(node.function.pos, u"%s() takes exactly one argument" % function)
1639 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1640 elif function in InterpretCompilerDirectives.binop_method_nodes:
1641 if len(node.args) != 2:
1642 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1644 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1645 elif function == u'cast':
1646 if len(node.args) != 2:
1647 error(node.function.pos, u"cast() takes exactly two arguments")
1649 type = node.args[0].analyse_as_type(self.current_env())
1651 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1653 error(node.args[0].pos, "Not a type")
1654 elif function == u'sizeof':
1655 if len(node.args) != 1:
1656 error(node.function.pos, u"sizeof() takes exactly one argument")
1658 type = node.args[0].analyse_as_type(self.current_env())
1660 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1662 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1663 elif function == 'cmod':
1664 if len(node.args) != 2:
1665 error(node.function.pos, u"cmod() takes exactly two arguments")
1667 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1668 node.cdivision = True
1669 elif function == 'cdiv':
1670 if len(node.args) != 2:
1671 error(node.function.pos, u"cdiv() takes exactly two arguments")
1673 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1674 node.cdivision = True
1675 elif function == u'set':
1676 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1678 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1680 self.visitchildren(node)
1684 class DebugTransform(CythonTransform):
1686 Create debug information and all functions' visibility to extern in order
1687 to enable debugging.
1690 def __init__(self, context, options, result):
1691 super(DebugTransform, self).__init__(context)
1692 self.visited = cython.set()
1693 # our treebuilder and debug output writer
1694 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1695 self.tb = self.context.gdb_debug_outputwriter
1696 #self.c_output_file = options.output_file
1697 self.c_output_file = result.c_file
1699 # Closure support, basically treat nested functions as if the AST were
1701 self.nested_funcdefs = []
1703 # tells visit_NameNode whether it should register step-into functions
1704 self.register_stepinto = False
1706 def visit_ModuleNode(self, node):
1707 self.tb.module_name = node.full_module_name
1709 module_name=node.full_module_name,
1710 filename=node.pos[0].filename,
1711 c_filename=self.c_output_file)
1713 self.tb.start('Module', attrs)
1715 # serialize functions
1716 self.tb.start('Functions')
1717 # First, serialize functions normally...
1718 self.visitchildren(node)
1720 # ... then, serialize nested functions
1721 for nested_funcdef in self.nested_funcdefs:
1722 self.visit_FuncDefNode(nested_funcdef)
1724 self.register_stepinto = True
1725 self.serialize_modulenode_as_function(node)
1726 self.register_stepinto = False
1727 self.tb.end('Functions')
1729 # 2.3 compatibility. Serialize global variables
1730 self.tb.start('Globals')
1733 for k, v in node.scope.entries.iteritems():
1734 if (v.qualified_name not in self.visited and not
1735 v.name.startswith('__pyx_') and not
1736 v.type.is_cfunction and not
1737 v.type.is_extension_type):
1740 self.serialize_local_variables(entries)
1741 self.tb.end('Globals')
1742 # self.tb.end('Module') # end Module after the line number mapping in
1743 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1746 def visit_FuncDefNode(self, node):
1747 self.visited.add(node.local_scope.qualified_name)
1749 if getattr(node, 'is_wrapper', False):
1752 if self.register_stepinto:
1753 self.nested_funcdefs.append(node)
1756 # node.entry.visibility = 'extern'
1757 if node.py_func is None:
1760 pf_cname = node.py_func.entry.func_cname
1763 name=node.entry.name,
1764 cname=node.entry.func_cname,
1766 qualified_name=node.local_scope.qualified_name,
1767 lineno=str(node.pos[1]))
1769 self.tb.start('Function', attrs=attrs)
1771 self.tb.start('Locals')
1772 self.serialize_local_variables(node.local_scope.entries)
1773 self.tb.end('Locals')
1775 self.tb.start('Arguments')
1776 for arg in node.local_scope.arg_entries:
1777 self.tb.start(arg.name)
1778 self.tb.end(arg.name)
1779 self.tb.end('Arguments')
1781 self.tb.start('StepIntoFunctions')
1782 self.register_stepinto = True
1783 self.visitchildren(node)
1784 self.register_stepinto = False
1785 self.tb.end('StepIntoFunctions')
1786 self.tb.end('Function')
1790 def visit_NameNode(self, node):
1791 if (self.register_stepinto and
1792 node.type.is_cfunction and
1793 getattr(node, 'is_called', False) and
1794 node.entry.func_cname is not None):
1795 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1796 # declared functions are not 'in_cinclude'.
1797 # This means we will list called 'cdef' functions as
1798 # "step into functions", but this is not an issue as they will be
1799 # recognized as Cython functions anyway.
1800 attrs = dict(name=node.entry.func_cname)
1801 self.tb.start('StepIntoFunction', attrs=attrs)
1802 self.tb.end('StepIntoFunction')
1804 self.visitchildren(node)
1807 def serialize_modulenode_as_function(self, node):
1809 Serialize the module-level code as a function so the debugger will know
1810 it's a "relevant frame" and it will know where to set the breakpoint
1811 for 'break modulename'.
1813 name = node.full_module_name.rpartition('.')[-1]
1815 cname_py2 = 'init' + name
1816 cname_py3 = 'PyInit_' + name
1822 # Ignore the qualified_name, breakpoints should be set using
1823 # `cy break modulename:lineno` for module-level breakpoints.
1826 is_initmodule_function="True",
1829 py3_attrs = dict(py2_attrs, cname=cname_py3)
1831 self._serialize_modulenode_as_function(node, py2_attrs)
1832 self._serialize_modulenode_as_function(node, py3_attrs)
1834 def _serialize_modulenode_as_function(self, node, attrs):
1835 self.tb.start('Function', attrs=attrs)
1837 self.tb.start('Locals')
1838 self.serialize_local_variables(node.scope.entries)
1839 self.tb.end('Locals')
1841 self.tb.start('Arguments')
1842 self.tb.end('Arguments')
1844 self.tb.start('StepIntoFunctions')
1845 self.register_stepinto = True
1846 self.visitchildren(node)
1847 self.register_stepinto = False
1848 self.tb.end('StepIntoFunctions')
1850 self.tb.end('Function')
1852 def serialize_local_variables(self, entries):
1853 for entry in entries.values():
1854 if entry.type.is_pyobject:
1855 vartype = 'PythonObject'
1859 if entry.from_closure:
1860 # We're dealing with a closure where a variable from an outer
1861 # scope is accessed, get it from the scope object.
1862 cname = '%s->%s' % (Naming.cur_scope_cname,
1863 entry.outer_entry.cname)
1865 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1868 elif entry.in_closure:
1869 cname = '%s->%s' % (Naming.cur_scope_cname,
1871 qname = entry.qualified_name
1874 qname = entry.qualified_name
1877 # this happens for variables that are not in the user's code,
1878 # e.g. for the global __builtins__, __doc__, etc. We can just
1879 # set the lineno to 0 for those.
1882 lineno = str(entry.pos[1])
1887 qualified_name=qname,
1891 self.tb.start('LocalVar', attrs)
1892 self.tb.end('LocalVar')