3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
17 from Cython.Compiler.ModuleNode import ModuleNode
18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
20 from Cython.Compiler.StringEncoding import EncodedString
21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
26 class NameNodeCollector(TreeVisitor):
27 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
31 super(NameNodeCollector, self).__init__()
34 def visit_NameNode(self, node):
35 self.name_nodes.append(node)
37 def visit_Node(self, node):
38 self._visitchildren(node, None)
41 class SkipDeclarations(object):
43 Variable and function declarations can often have a deep tree structure,
44 and yet most transformations don't need to descend to this depth.
46 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
47 is no need to use this for transformations after that point.
49 def visit_CTypeDefNode(self, node):
52 def visit_CVarDefNode(self, node):
55 def visit_CDeclaratorNode(self, node):
58 def visit_CBaseTypeNode(self, node):
61 def visit_CEnumDefNode(self, node):
64 def visit_CStructOrUnionDefNode(self, node):
67 class NormalizeTree(CythonTransform):
69 This transform fixes up a few things after parsing
70 in order to make the parse tree more suitable for
73 a) After parsing, blocks with only one statement will
74 be represented by that statement, not by a StatListNode.
75 When doing transforms this is annoying and inconsistent,
76 as one cannot in general remove a statement in a consistent
77 way and so on. This transform wraps any single statements
78 in a StatListNode containing a single statement.
80 b) The PassStatNode is a noop and serves no purpose beyond
81 plugging such one-statement blocks; i.e., once parsed a
82 ` "pass" can just as well be represented using an empty
83 StatListNode. This means less special cases to worry about
84 in subsequent transforms (one always checks to see if a
85 StatListNode has no children to see if the block is empty).
88 def __init__(self, context):
89 super(NormalizeTree, self).__init__(context)
90 self.is_in_statlist = False
91 self.is_in_expr = False
93 def visit_ExprNode(self, node):
94 stacktmp = self.is_in_expr
95 self.is_in_expr = True
96 self.visitchildren(node)
97 self.is_in_expr = stacktmp
100 def visit_StatNode(self, node, is_listcontainer=False):
101 stacktmp = self.is_in_statlist
102 self.is_in_statlist = is_listcontainer
103 self.visitchildren(node)
104 self.is_in_statlist = stacktmp
105 if not self.is_in_statlist and not self.is_in_expr:
106 return Nodes.StatListNode(pos=node.pos, stats=[node])
110 def visit_StatListNode(self, node):
111 self.is_in_statlist = True
112 self.visitchildren(node)
113 self.is_in_statlist = False
116 def visit_ParallelAssignmentNode(self, node):
117 return self.visit_StatNode(node, True)
119 def visit_CEnumDefNode(self, node):
120 return self.visit_StatNode(node, True)
122 def visit_CStructOrUnionDefNode(self, node):
123 return self.visit_StatNode(node, True)
125 # Eliminate PassStatNode
126 def visit_PassStatNode(self, node):
127 if not self.is_in_statlist:
128 return Nodes.StatListNode(pos=node.pos, stats=[])
132 def visit_CDeclaratorNode(self, node):
136 class PostParseError(CompileError): pass
138 # error strings checked by unit tests, so define them
139 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
140 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
141 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
142 class PostParse(ScopeTrackingTransform):
144 Basic interpretation of the parse tree, as well as validity
145 checking that can be done on a very basic level on the parse
146 tree (while still not being a problem with the basic syntax,
150 - Default values to cdef assignments are turned into single
151 assignments following the declaration (everywhere but in class
152 bodies, where they raise a compile error)
154 - Interpret some node structures into Python runtime values.
155 Some nodes take compile-time arguments (currently:
156 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
157 which should be interpreted. This happens in a general way
158 and other steps should be taken to ensure validity.
160 Type arguments cannot be interpreted in this way.
162 - For __cythonbufferdefaults__ the arguments are checked for
165 TemplatedTypeNode has its directives interpreted:
166 Any first positional argument goes into the "dtype" attribute,
167 any "ndim" keyword argument goes into the "ndim" attribute and
168 so on. Also it is checked that the directive combination is valid.
169 - __cythonbufferdefaults__ attributes are parsed and put into the
172 Note: Currently Parsing.py does a lot of interpretation and
173 reorganization that can be refactored into this transform
174 if a more pure Abstract Syntax Tree is wanted.
177 def __init__(self, context):
178 super(PostParse, self).__init__(context)
179 self.specialattribute_handlers = {
180 '__cythonbufferdefaults__' : self.handle_bufferdefaults
183 def visit_ModuleNode(self, node):
184 self.lambda_counter = 1
185 self.genexpr_counter = 1
186 return super(PostParse, self).visit_ModuleNode(node)
188 def visit_LambdaNode(self, node):
189 # unpack a lambda expression into the corresponding DefNode
190 lambda_id = self.lambda_counter
191 self.lambda_counter += 1
192 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
193 collector = YieldNodeCollector()
194 collector.visitchildren(node.result_expr)
195 if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
196 body = ExprNodes.YieldExprNode(
197 node.result_expr.pos, arg=node.result_expr)
198 body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
200 body = Nodes.ReturnStatNode(
201 node.result_expr.pos, value=node.result_expr)
202 node.def_node = Nodes.DefNode(
203 node.pos, name=node.name, lambda_name=node.lambda_name,
204 args=node.args, star_arg=node.star_arg,
205 starstar_arg=node.starstar_arg,
207 self.visitchildren(node)
210 def visit_GeneratorExpressionNode(self, node):
211 # unpack a generator expression into the corresponding DefNode
212 genexpr_id = self.genexpr_counter
213 self.genexpr_counter += 1
214 node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
216 node.def_node = Nodes.DefNode(node.pos, name=node.name,
218 args=[], star_arg=None,
221 self.visitchildren(node)
225 def handle_bufferdefaults(self, decl):
226 if not isinstance(decl.default, ExprNodes.DictNode):
227 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
228 self.scope_node.buffer_defaults_node = decl.default
229 self.scope_node.buffer_defaults_pos = decl.pos
231 def visit_CVarDefNode(self, node):
232 # This assumes only plain names and pointers are assignable on
233 # declaration. Also, it makes use of the fact that a cdef decl
234 # must appear before the first use, so we don't have to deal with
235 # "i = 3; cdef int i = i" and can simply move the nodes around.
237 self.visitchildren(node)
240 for decl in node.declarators:
242 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
243 declbase = declbase.base
244 if isinstance(declbase, Nodes.CNameDeclaratorNode):
245 if declbase.default is not None:
246 if self.scope_type in ('cclass', 'pyclass', 'struct'):
247 if isinstance(self.scope_node, Nodes.CClassDefNode):
248 handler = self.specialattribute_handlers.get(decl.name)
250 if decl is not declbase:
251 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
253 continue # Remove declaration
254 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
255 first_assignment = self.scope_type != 'module'
256 stats.append(Nodes.SingleAssignmentNode(node.pos,
257 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
258 rhs=declbase.default, first=first_assignment))
259 declbase.default = None
260 newdecls.append(decl)
261 node.declarators = newdecls
263 except PostParseError, e:
264 # An error in a cdef clause is ok, simply remove the declaration
265 # and try to move on to report more errors
266 self.context.nonfatal_error(e)
269 # Split parallel assignments (a,b = b,a) into separate partial
270 # assignments that are executed rhs-first using temps. This
271 # restructuring must be applied before type analysis so that known
272 # types on rhs and lhs can be matched directly. It is required in
273 # the case that the types cannot be coerced to a Python type in
274 # order to assign from a tuple.
276 def visit_SingleAssignmentNode(self, node):
277 self.visitchildren(node)
278 return self._visit_assignment_node(node, [node.lhs, node.rhs])
280 def visit_CascadedAssignmentNode(self, node):
281 self.visitchildren(node)
282 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
284 def _visit_assignment_node(self, node, expr_list):
285 """Flatten parallel assignments into separate single
286 assignments or cascaded assignments.
288 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
289 # no parallel assignments => nothing to do
293 flatten_parallel_assignments(expr_list, expr_list_list)
295 eliminate_rhs_duplicates(expr_list_list, temp_refs)
298 for expr_list in expr_list_list:
299 lhs_list = expr_list[:-1]
301 if len(lhs_list) == 1:
302 node = Nodes.SingleAssignmentNode(rhs.pos,
303 lhs = lhs_list[0], rhs = rhs)
305 node = Nodes.CascadedAssignmentNode(rhs.pos,
306 lhs_list = lhs_list, rhs = rhs)
310 assign_node = nodes[0]
312 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
315 duplicates_and_temps = [ (temp.expression, temp)
316 for temp in temp_refs ]
317 sort_common_subsequences(duplicates_and_temps)
318 for _, temp_ref in duplicates_and_temps[::-1]:
319 assign_node = LetNode(temp_ref, assign_node)
323 def _flatten_sequence(self, seq, result):
325 if arg.is_sequence_constructor:
326 self._flatten_sequence(arg, result)
331 def visit_DelStatNode(self, node):
332 self.visitchildren(node)
333 node.args = self._flatten_sequence(node, [])
337 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
338 """Replace rhs items by LetRefNodes if they appear more than once.
339 Creates a sequence of LetRefNodes that set up the required temps
340 and appends them to ref_node_sequence. The input list is modified
343 seen_nodes = cython.set()
345 def find_duplicates(node):
346 if node.is_literal or node.is_name:
347 # no need to replace those; can't include attributes here
348 # as their access is not necessarily side-effect free
350 if node in seen_nodes:
351 if node not in ref_nodes:
352 ref_node = LetRefNode(node)
353 ref_nodes[node] = ref_node
354 ref_node_sequence.append(ref_node)
357 if node.is_sequence_constructor:
358 for item in node.args:
359 find_duplicates(item)
361 for expr_list in expr_list_list:
367 def substitute_nodes(node):
368 if node in ref_nodes:
369 return ref_nodes[node]
370 elif node.is_sequence_constructor:
371 node.args = list(map(substitute_nodes, node.args))
374 # replace nodes inside of the common subexpressions
375 for node in ref_nodes:
376 if node.is_sequence_constructor:
377 node.args = list(map(substitute_nodes, node.args))
379 # replace common subexpressions on all rhs items
380 for expr_list in expr_list_list:
381 expr_list[-1] = substitute_nodes(expr_list[-1])
383 def sort_common_subsequences(items):
384 """Sort items/subsequences so that all items and subsequences that
385 an item contains appear before the item itself. This is needed
386 because each rhs item must only be evaluated once, so its value
387 must be evaluated first and then reused when packing sequences
390 This implies a partial order, and the sort must be stable to
391 preserve the original order as much as possible, so we use a
392 simple insertion sort (which is very fast for short sequences, the
393 normal case in practice).
395 def contains(seq, x):
399 elif item.is_sequence_constructor and contains(item.args, x):
403 return b.is_sequence_constructor and contains(b.args, a)
405 for pos, item in enumerate(items):
406 key = item[1] # the ResultRefNode which has already been injected into the sequences
408 for i in xrange(pos-1, -1, -1):
409 if lower_than(key, items[i][0]):
412 for i in xrange(pos, new_pos, -1):
413 items[i] = items[i-1]
414 items[new_pos] = item
416 def flatten_parallel_assignments(input, output):
417 # The input is a list of expression nodes, representing the LHSs
418 # and RHS of one (possibly cascaded) assignment statement. For
419 # sequence constructors, rearranges the matching parts of both
420 # sides into a list of equivalent assignments between the
421 # individual elements. This transformation is applied
422 # recursively, so that nested structures get matched as well.
424 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
428 complete_assignments = []
430 rhs_size = len(rhs.args)
431 lhs_targets = [ [] for _ in xrange(rhs_size) ]
432 starred_assignments = []
433 for lhs in input[:-1]:
434 if not lhs.is_sequence_constructor:
436 error(lhs.pos, "starred assignment target must be in a list or tuple")
437 complete_assignments.append(lhs)
439 lhs_size = len(lhs.args)
440 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
441 if starred_targets > 1:
442 error(lhs.pos, "more than 1 starred expression in assignment")
443 output.append([lhs,rhs])
445 elif lhs_size - starred_targets > rhs_size:
446 error(lhs.pos, "need more than %d value%s to unpack"
447 % (rhs_size, (rhs_size != 1) and 's' or ''))
448 output.append([lhs,rhs])
450 elif starred_targets:
451 map_starred_assignment(lhs_targets, starred_assignments,
453 elif lhs_size < rhs_size:
454 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
455 % (lhs_size, rhs_size))
456 output.append([lhs,rhs])
459 for targets, expr in zip(lhs_targets, lhs.args):
462 if complete_assignments:
463 complete_assignments.append(rhs)
464 output.append(complete_assignments)
466 # recursively flatten partial assignments
467 for cascade, rhs in zip(lhs_targets, rhs.args):
470 flatten_parallel_assignments(cascade, output)
472 # recursively flatten starred assignments
473 for cascade in starred_assignments:
474 if cascade[0].is_sequence_constructor:
475 flatten_parallel_assignments(cascade, output)
477 output.append(cascade)
479 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
480 # Appends the fixed-position LHS targets to the target list that
481 # appear left and right of the starred argument.
483 # The starred_assignments list receives a new tuple
484 # (lhs_target, rhs_values_list) that maps the remaining arguments
485 # (those that match the starred target) to a list.
487 # left side of the starred target
488 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
491 lhs_remaining = len(lhs_args) - i - 1
495 raise InternalError("no starred arg found when splitting starred assignment")
497 # right side of the starred target
498 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
499 lhs_args[starred + 1:])):
502 # the starred target itself, must be assigned a (potentially empty) list
503 target = lhs_args[starred].target # unpack starred node
504 starred_rhs = rhs_args[starred:]
506 starred_rhs = starred_rhs[:-lhs_remaining]
508 pos = starred_rhs[0].pos
511 starred_assignments.append([
512 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
515 class PxdPostParse(CythonTransform, SkipDeclarations):
517 Basic interpretation/validity checking that should only be
520 A lot of this checking currently happens in the parser; but
521 what is listed below happens here.
523 - "def" functions are let through only if they fill the
524 getbuffer/releasebuffer slots
526 - cdef functions are let through only if they are on the
527 top level and are declared "inline"
529 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
530 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
532 def __call__(self, node):
533 self.scope_type = 'pxd'
534 return super(PxdPostParse, self).__call__(node)
536 def visit_CClassDefNode(self, node):
537 old = self.scope_type
538 self.scope_type = 'cclass'
539 self.visitchildren(node)
540 self.scope_type = old
543 def visit_FuncDefNode(self, node):
544 # FuncDefNode always come with an implementation (without
545 # an imp they are CVarDefNodes..)
546 err = self.ERR_INLINE_ONLY
548 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
549 and node.name in ('__getbuffer__', '__releasebuffer__')):
550 err = None # allow these slots
552 if isinstance(node, Nodes.CFuncDefNode):
553 if u'inline' in node.modifiers and self.scope_type == 'pxd':
554 node.inline_in_pxd = True
555 if node.visibility != 'private':
556 err = self.ERR_NOGO_WITH_INLINE % node.visibility
558 err = self.ERR_NOGO_WITH_INLINE % 'api'
560 err = None # allow inline function
562 err = self.ERR_INLINE_ONLY
565 self.context.nonfatal_error(PostParseError(node.pos, err))
570 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
572 After parsing, directives can be stored in a number of places:
573 - #cython-comments at the top of the file (stored in ModuleNode)
574 - Command-line arguments overriding these
575 - @cython.directivename decorators
576 - with cython.directivename: statements
578 This transform is responsible for interpreting these various sources
579 and store the directive in two ways:
580 - Set the directives attribute of the ModuleNode for global directives.
581 - Use a CompilerDirectivesNode to override directives for a subtree.
583 (The first one is primarily to not have to modify with the tree
584 structure, so that ModuleNode stay on top.)
586 The directives are stored in dictionaries from name to value in effect.
587 Each such dictionary is always filled in for all possible directives,
588 using default values where no value is given by the user.
590 The available directives are controlled in Options.py.
592 Note that we have to run this prior to analysis, and so some minor
593 duplication of functionality has to occur: We manually track cimports
594 and which names the "cython" module may have been imported to.
596 unop_method_nodes = {
597 'typeof': ExprNodes.TypeofNode,
599 'operator.address': ExprNodes.AmpersandNode,
600 'operator.dereference': ExprNodes.DereferenceNode,
601 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
602 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
603 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
604 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
606 # For backwards compatability.
607 'address': ExprNodes.AmpersandNode,
610 binop_method_nodes = {
611 'operator.comma' : ExprNodes.c_binop_constructor(','),
614 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
615 'cast', 'pointer', 'compiled', 'NULL'])
616 special_methods.update(unop_method_nodes.keys())
618 def __init__(self, context, compilation_directive_defaults):
619 super(InterpretCompilerDirectives, self).__init__(context)
620 self.compilation_directive_defaults = {}
621 for key, value in compilation_directive_defaults.items():
622 self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
623 self.cython_module_names = cython.set()
624 self.directive_names = {}
626 def check_directive_scope(self, pos, directive, scope):
627 legal_scopes = Options.directive_scopes.get(directive, None)
628 if legal_scopes and scope not in legal_scopes:
629 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
630 'is not allowed in %s scope' % (directive, scope)))
635 # Set up processing and handle the cython: comments.
636 def visit_ModuleNode(self, node):
637 for key, value in node.directive_comments.items():
638 if not self.check_directive_scope(node.pos, key, 'module'):
639 self.wrong_scope_error(node.pos, key, 'module')
640 del node.directive_comments[key]
642 directives = copy.deepcopy(Options.directive_defaults)
643 directives.update(copy.deepcopy(self.compilation_directive_defaults))
644 directives.update(node.directive_comments)
645 self.directives = directives
646 node.directives = directives
647 self.visitchildren(node)
648 node.cython_module_names = self.cython_module_names
651 # The following four functions track imports and cimports that
652 # begin with "cython"
653 def is_cython_directive(self, name):
654 return (name in Options.directive_types or
655 name in self.special_methods or
656 PyrexTypes.parse_basic_type(name))
658 def visit_CImportStatNode(self, node):
659 if node.module_name == u"cython":
660 self.cython_module_names.add(node.as_name or u"cython")
661 elif node.module_name.startswith(u"cython."):
663 self.directive_names[node.as_name] = node.module_name[7:]
665 self.cython_module_names.add(u"cython")
666 # if this cimport was a compiler directive, we don't
667 # want to leave the cimport node sitting in the tree
671 def visit_FromCImportStatNode(self, node):
672 if (node.module_name == u"cython") or \
673 node.module_name.startswith(u"cython."):
674 submodule = (node.module_name + u".")[7:]
676 for pos, name, as_name, kind in node.imported_names:
677 full_name = submodule + name
678 if self.is_cython_directive(full_name):
681 self.directive_names[as_name] = full_name
683 self.context.nonfatal_error(PostParseError(pos,
684 "Compiler directive imports must be plain imports"))
686 newimp.append((pos, name, as_name, kind))
689 node.imported_names = newimp
692 def visit_FromImportStatNode(self, node):
693 if (node.module.module_name.value == u"cython") or \
694 node.module.module_name.value.startswith(u"cython."):
695 submodule = (node.module.module_name.value + u".")[7:]
697 for name, name_node in node.items:
698 full_name = submodule + name
699 if self.is_cython_directive(full_name):
700 self.directive_names[name_node.name] = full_name
702 newimp.append((name, name_node))
708 def visit_SingleAssignmentNode(self, node):
709 if (isinstance(node.rhs, ExprNodes.ImportNode) and
710 node.rhs.module_name.value == u'cython'):
711 node = Nodes.CImportStatNode(node.pos,
712 module_name = u'cython',
713 as_name = node.lhs.name)
714 self.visit_CImportStatNode(node)
716 self.visitchildren(node)
719 def visit_NameNode(self, node):
720 if node.name in self.cython_module_names:
721 node.is_cython_module = True
723 node.cython_attribute = self.directive_names.get(node.name)
726 def try_to_parse_directives(self, node):
727 # If node is the contents of an directive (in a with statement or
728 # decorator), returns a list of (directivename, value) pairs.
729 # Otherwise, returns None
730 if isinstance(node, ExprNodes.CallNode):
731 self.visit(node.function)
732 optname = node.function.as_cython_attribute()
734 directivetype = Options.directive_types.get(optname)
736 args, kwds = node.explicit_args_kwds()
739 if kwds is not None and directivetype is not dict:
740 for keyvalue in kwds.key_value_pairs:
741 key, value = keyvalue
742 sub_optname = "%s.%s" % (optname, key.value)
743 if Options.directive_types.get(sub_optname):
744 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
746 key_value_pairs.append(keyvalue)
747 if not key_value_pairs:
750 kwds.key_value_pairs = key_value_pairs
751 if directives and not kwds and not args:
753 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
755 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
757 optname = node.as_cython_attribute()
759 directivetype = Options.directive_types.get(optname)
760 if directivetype is bool:
761 return [(optname, True)]
762 elif directivetype is None:
763 return [(optname, None)]
765 raise PostParseError(
766 node.pos, "The '%s' directive should be used as a function call." % optname)
769 def try_to_parse_directive(self, optname, args, kwds, pos):
770 directivetype = Options.directive_types.get(optname)
771 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
772 return optname, Options.directive_defaults[optname]
773 elif directivetype is bool:
774 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
775 raise PostParseError(pos,
776 'The %s directive takes one compile-time boolean argument' % optname)
777 return (optname, args[0].value)
778 elif directivetype is str:
779 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
780 ExprNodes.UnicodeNode)):
781 raise PostParseError(pos,
782 'The %s directive takes one compile-time string argument' % optname)
783 return (optname, str(args[0].value))
784 elif directivetype is dict:
786 raise PostParseError(pos,
787 'The %s directive takes no prepositional arguments' % optname)
788 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
789 elif directivetype is list:
790 if kwds and len(kwds) != 0:
791 raise PostParseError(pos,
792 'The %s directive takes no keyword arguments' % optname)
793 return optname, [ str(arg.value) for arg in args ]
797 def visit_with_directives(self, body, directives):
798 olddirectives = self.directives
799 newdirectives = copy.copy(olddirectives)
800 newdirectives.update(directives)
801 self.directives = newdirectives
802 assert isinstance(body, Nodes.StatListNode), body
803 retbody = self.visit_Node(body)
804 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
805 directives=newdirectives)
806 self.directives = olddirectives
810 def visit_FuncDefNode(self, node):
811 directives = self._extract_directives(node, 'function')
813 return self.visit_Node(node)
814 body = Nodes.StatListNode(node.pos, stats=[node])
815 return self.visit_with_directives(body, directives)
817 def visit_CVarDefNode(self, node):
818 if not node.decorators:
820 for dec in node.decorators:
821 for directive in self.try_to_parse_directives(dec.decorator) or ():
822 if directive is not None and directive[0] == u'locals':
823 node.directive_locals = directive[1]
825 self.context.nonfatal_error(PostParseError(dec.pos,
826 "Cdef functions can only take cython.locals() decorator."))
829 def visit_CClassDefNode(self, node):
830 directives = self._extract_directives(node, 'cclass')
832 return self.visit_Node(node)
833 body = Nodes.StatListNode(node.pos, stats=[node])
834 return self.visit_with_directives(body, directives)
836 def visit_PyClassDefNode(self, node):
837 directives = self._extract_directives(node, 'class')
839 return self.visit_Node(node)
840 body = Nodes.StatListNode(node.pos, stats=[node])
841 return self.visit_with_directives(body, directives)
843 def _extract_directives(self, node, scope_name):
844 if not node.decorators:
846 # Split the decorators into two lists -- real decorators and directives
849 for dec in node.decorators:
850 new_directives = self.try_to_parse_directives(dec.decorator)
851 if new_directives is not None:
852 for directive in new_directives:
853 if self.check_directive_scope(node.pos, directive[0], scope_name):
854 directives.append(directive)
857 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
858 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
860 node.decorators = realdecs
861 # merge or override repeated directives
863 directives.reverse() # Decorators coming first take precedence
864 for directive in directives:
865 name, value = directive
867 old_value = optdict[name]
868 # keywords and arg lists can be merged, everything
869 # else overrides completely
870 if isinstance(old_value, dict):
871 old_value.update(value)
872 elif isinstance(old_value, list):
873 old_value.extend(value)
875 optdict[name] = value
877 optdict[name] = value
880 # Handle with statements
881 def visit_WithStatNode(self, node):
883 for directive in self.try_to_parse_directives(node.manager) or []:
884 if directive is not None:
885 if node.target is not None:
886 self.context.nonfatal_error(
887 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
889 name, value = directive
891 # special case: in pure mode, "with nogil" spells "with cython.nogil"
892 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
893 return self.visit_Node(node)
894 if self.check_directive_scope(node.pos, name, 'with statement'):
895 directive_dict[name] = value
897 return self.visit_with_directives(node.body, directive_dict)
898 return self.visit_Node(node)
900 class WithTransform(CythonTransform, SkipDeclarations):
902 # EXCINFO is manually set to a variable that contains
903 # the exc_info() tuple that can be generated by the enclosing except
905 template_without_target = TreeFragment(u"""
916 if not EXIT(*EXCINFO):
920 EXIT(None, None, None)
921 """, temps=[u'MGR', u'EXC', u"EXIT"],
922 pipeline=[NormalizeTree(None)])
924 template_with_target = TreeFragment(u"""
927 VALUE = MGR.__enter__()
936 if not EXIT(*EXCINFO):
940 EXIT(None, None, None)
941 MGR = EXIT = VALUE = EXC = None
943 """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
944 pipeline=[NormalizeTree(None)])
946 def visit_WithStatNode(self, node):
947 # TODO: Cleanup badly needed
948 TemplateTransform.temp_name_counter += 1
949 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
951 self.visitchildren(node, ['body'])
952 excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
953 if node.target is not None:
954 result = self.template_with_target.substitute({
955 u'EXPR' : node.manager,
957 u'TARGET' : node.target,
958 u'EXCINFO' : excinfo_temp
961 result = self.template_without_target.substitute({
962 u'EXPR' : node.manager,
964 u'EXCINFO' : excinfo_temp
967 # Set except excinfo target to EXCINFO
968 try_except = result.stats[-1].body.stats[-1]
969 try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
970 # excinfo_temp.ref(node.pos))
972 # result.stats[-1].body.stats[-1] = TempsBlockNode(
973 # node.pos, temps=[excinfo_temp], body=try_except)
977 def visit_ExprNode(self, node):
978 # With statements are never inside expressions.
982 class DecoratorTransform(CythonTransform, SkipDeclarations):
984 def visit_DefNode(self, func_node):
985 self.visitchildren(func_node)
986 if not func_node.decorators:
988 return self._handle_decorators(
989 func_node, func_node.name)
991 def visit_CClassDefNode(self, class_node):
992 # This doesn't currently work, so it's disabled.
994 # Problem: assignments to cdef class names do not work. They
995 # would require an additional check anyway, as the extension
996 # type must not change its C type, so decorators cannot
997 # replace an extension type, just alter it and return it.
999 self.visitchildren(class_node)
1000 if not class_node.decorators:
1002 error(class_node.pos,
1003 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
1005 #return self._handle_decorators(
1006 # class_node, class_node.class_name)
1008 def visit_ClassDefNode(self, class_node):
1009 self.visitchildren(class_node)
1010 if not class_node.decorators:
1012 return self._handle_decorators(
1013 class_node, class_node.name)
1015 def _handle_decorators(self, node, name):
1016 decorator_result = ExprNodes.NameNode(node.pos, name = name)
1017 for decorator in node.decorators[::-1]:
1018 decorator_result = ExprNodes.SimpleCallNode(
1020 function = decorator.decorator,
1021 args = [decorator_result])
1023 name_node = ExprNodes.NameNode(node.pos, name = name)
1024 reassignment = Nodes.SingleAssignmentNode(
1027 rhs = decorator_result)
1028 return [node, reassignment]
1031 class AnalyseDeclarationsTransform(CythonTransform):
1033 basic_property = TreeFragment(u"""
1037 def __set__(self, value):
1039 """, level='c_class')
1040 basic_pyobject_property = TreeFragment(u"""
1044 def __set__(self, value):
1048 """, level='c_class')
1049 basic_property_ro = TreeFragment(u"""
1053 """, level='c_class')
1055 struct_or_union_wrapper = TreeFragment(u"""
1058 def __init__(self, MEMBER=None):
1062 if IS_UNION and count > 1:
1063 raise ValueError, "At most one union member should be specified."
1065 return STR_FORMAT % MEMBER_TUPLE
1067 return REPR_FORMAT % MEMBER_TUPLE
1070 init_assignment = TreeFragment(u"""
1071 if VALUE is not None:
1076 def __call__(self, root):
1077 self.env_stack = [root.scope]
1078 # needed to determine if a cdef var is declared after it's used.
1079 self.seen_vars_stack = []
1080 return super(AnalyseDeclarationsTransform, self).__call__(root)
1082 def visit_NameNode(self, node):
1083 self.seen_vars_stack[-1].add(node.name)
1086 def visit_ModuleNode(self, node):
1087 self.seen_vars_stack.append(cython.set())
1088 node.analyse_declarations(self.env_stack[-1])
1089 self.visitchildren(node)
1090 self.seen_vars_stack.pop()
1093 def visit_LambdaNode(self, node):
1094 node.analyse_declarations(self.env_stack[-1])
1095 self.visitchildren(node)
1098 def visit_ClassDefNode(self, node):
1099 self.env_stack.append(node.scope)
1100 self.visitchildren(node)
1101 self.env_stack.pop()
1104 def visit_CClassDefNode(self, node):
1105 node = self.visit_ClassDefNode(node)
1106 if node.scope and node.scope.implemented:
1108 for entry in node.scope.var_entries:
1109 if entry.needs_property:
1110 property = self.create_Property(entry)
1111 property.analyse_declarations(node.scope)
1112 self.visit(property)
1113 stats.append(property)
1115 node.body.stats += stats
1118 def visit_FuncDefNode(self, node):
1119 self.seen_vars_stack.append(cython.set())
1120 lenv = node.local_scope
1121 node.body.analyse_control_flow(lenv) # this will be totally refactored
1122 node.declare_arguments(lenv)
1123 for var, type_node in node.directive_locals.items():
1124 if not lenv.lookup_here(var): # don't redeclare args
1125 type = type_node.analyse_as_type(lenv)
1127 lenv.declare_var(var, type, type_node.pos)
1129 error(type_node.pos, "Not a type")
1130 node.body.analyse_declarations(lenv)
1131 self.env_stack.append(lenv)
1132 self.visitchildren(node)
1133 self.env_stack.pop()
1134 self.seen_vars_stack.pop()
1137 def visit_ScopedExprNode(self, node):
1138 env = self.env_stack[-1]
1139 node.analyse_declarations(env)
1140 # the node may or may not have a local scope
1141 if node.has_local_scope:
1142 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1143 self.env_stack.append(node.expr_scope)
1144 node.analyse_scoped_declarations(node.expr_scope)
1145 self.visitchildren(node)
1146 self.env_stack.pop()
1147 self.seen_vars_stack.pop()
1149 node.analyse_scoped_declarations(env)
1150 self.visitchildren(node)
1153 def visit_TempResultFromStatNode(self, node):
1154 self.visitchildren(node)
1155 node.analyse_declarations(self.env_stack[-1])
1158 def visit_CStructOrUnionDefNode(self, node):
1159 # Create a wrapper node if needed.
1160 # We want to use the struct type information (so it can't happen
1161 # before this phase) but also create new objects to be declared
1162 # (so it can't happen later).
1163 # Note that we don't return the original node, as it is
1164 # never used after this phase.
1165 if True: # private (default)
1168 self_value = ExprNodes.AttributeNode(
1170 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1171 attribute = EncodedString(u"value"))
1172 var_entries = node.entry.type.scope.var_entries
1174 for entry in var_entries:
1175 attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1177 attribute = entry.name))
1178 # __init__ assignments
1179 init_assignments = []
1180 for entry, attr in zip(var_entries, attributes):
1181 # TODO: branch on visibility
1182 init_assignments.append(self.init_assignment.substitute({
1183 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1185 }, pos = entry.pos))
1188 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1189 wrapper_class = self.struct_or_union_wrapper.substitute({
1190 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1191 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1192 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1193 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1194 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1195 }, pos = node.pos).stats[0]
1196 wrapper_class.class_name = node.name
1197 wrapper_class.shadow = True
1198 class_body = wrapper_class.body.stats
1201 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1202 class_body[0].base_type.name = node.name
1204 # fix __init__ arguments
1205 init_method = class_body[1]
1206 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1207 arg_template = init_method.args[1]
1208 if not node.entry.type.is_struct:
1209 arg_template.kw_only = True
1210 del init_method.args[1]
1211 for entry, attr in zip(var_entries, attributes):
1212 arg = copy.deepcopy(arg_template)
1213 arg.declarator.name = entry.name
1214 init_method.args.append(arg)
1217 for entry, attr in zip(var_entries, attributes):
1218 # TODO: branch on visibility
1219 if entry.type.is_pyobject:
1220 template = self.basic_pyobject_property
1222 template = self.basic_property
1223 property = template.substitute({
1225 }, pos = entry.pos).stats[0]
1226 property.name = entry.name
1227 wrapper_class.body.stats.append(property)
1229 wrapper_class.analyse_declarations(self.env_stack[-1])
1230 return self.visit_CClassDefNode(wrapper_class)
1232 # Some nodes are no longer needed after declaration
1233 # analysis and can be dropped. The analysis was performed
1234 # on these nodes in a seperate recursive process from the
1235 # enclosing function or module, so we can simply drop them.
1236 def visit_CDeclaratorNode(self, node):
1237 # necessary to ensure that all CNameDeclaratorNodes are visited.
1238 self.visitchildren(node)
1241 def visit_CTypeDefNode(self, node):
1244 def visit_CBaseTypeNode(self, node):
1247 def visit_CEnumDefNode(self, node):
1248 if node.visibility == 'public':
1253 def visit_CNameDeclaratorNode(self, node):
1254 if node.name in self.seen_vars_stack[-1]:
1255 entry = self.env_stack[-1].lookup(node.name)
1256 if entry is None or entry.visibility != 'extern':
1257 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1258 self.visitchildren(node)
1261 def visit_CVarDefNode(self, node):
1262 # to ensure all CNameDeclaratorNodes are visited.
1263 self.visitchildren(node)
1266 def create_Property(self, entry):
1267 if entry.visibility == 'public':
1268 if entry.type.is_pyobject:
1269 template = self.basic_pyobject_property
1271 template = self.basic_property
1272 elif entry.visibility == 'readonly':
1273 template = self.basic_property_ro
1274 property = template.substitute({
1275 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1276 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1277 attribute=entry.name),
1278 }, pos=entry.pos).stats[0]
1279 property.name = entry.name
1280 # ---------------------------------------
1281 # XXX This should go to AutoDocTransforms
1282 # ---------------------------------------
1283 if (Options.docstrings and
1284 self.current_directives['embedsignature']):
1285 attr_name = entry.name
1286 type_name = entry.type.declaration_code("", for_display=1)
1288 if not entry.type.is_pyobject:
1289 type_name = "'%s'" % type_name
1290 elif entry.type.is_extension_type:
1291 type_name = entry.type.module_name + '.' + type_name
1292 if entry.init is not None:
1293 default_value = ' = ' + entry.init
1294 elif entry.init_to_none:
1295 default_value = ' = ' + repr(None)
1296 docstring = attr_name + ': ' + type_name + default_value
1297 property.doc = EncodedString(docstring)
1298 # ---------------------------------------
1301 class AnalyseExpressionsTransform(CythonTransform):
1303 def visit_ModuleNode(self, node):
1304 node.scope.infer_types()
1305 node.body.analyse_expressions(node.scope)
1306 self.visitchildren(node)
1309 def visit_FuncDefNode(self, node):
1310 node.local_scope.infer_types()
1311 node.body.analyse_expressions(node.local_scope)
1312 self.visitchildren(node)
1315 def visit_ScopedExprNode(self, node):
1316 if node.has_local_scope:
1317 node.expr_scope.infer_types()
1318 node.analyse_scoped_expressions(node.expr_scope)
1319 self.visitchildren(node)
1322 class ExpandInplaceOperators(EnvTransform):
1324 def visit_InPlaceAssignmentNode(self, node):
1327 if lhs.type.is_cpp_class:
1328 # No getting around this exact operator here.
1330 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1331 # There is code to handle this case.
1334 env = self.current_env()
1335 def side_effect_free_reference(node, setting=False):
1336 if isinstance(node, ExprNodes.NameNode):
1338 elif node.type.is_pyobject and not setting:
1339 node = LetRefNode(node)
1341 elif isinstance(node, ExprNodes.IndexNode):
1342 if node.is_buffer_access:
1343 raise ValueError, "Buffer access"
1344 base, temps = side_effect_free_reference(node.base)
1345 index = LetRefNode(node.index)
1346 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1347 elif isinstance(node, ExprNodes.AttributeNode):
1348 obj, temps = side_effect_free_reference(node.obj)
1349 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1351 node = LetRefNode(node)
1354 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1357 dup = lhs.__class__(**lhs.__dict__)
1358 binop = ExprNodes.binop_node(node.pos,
1359 operator = node.operator,
1363 # Manually analyse types for new node.
1364 lhs.analyse_target_types(env)
1365 dup.analyse_types(env)
1366 binop.analyse_operation(env)
1367 node = Nodes.SingleAssignmentNode(
1370 rhs=binop.coerce_to(lhs.type, env))
1371 # Use LetRefNode to avoid side effects.
1372 let_ref_nodes.reverse()
1373 for t in let_ref_nodes:
1374 node = LetNode(t, node)
1377 def visit_ExprNode(self, node):
1378 # In-place assignments can't happen within an expression.
1382 class AlignFunctionDefinitions(CythonTransform):
1384 This class takes the signatures from a .pxd file and applies them to
1385 the def methods in a .py file.
1388 def visit_ModuleNode(self, node):
1389 self.scope = node.scope
1390 self.directives = node.directives
1391 self.visitchildren(node)
1394 def visit_PyClassDefNode(self, node):
1395 pxd_def = self.scope.lookup(node.name)
1397 if pxd_def.is_cclass:
1398 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1400 error(node.pos, "'%s' redeclared" % node.name)
1401 error(pxd_def.pos, "previous declaration here")
1406 def visit_CClassDefNode(self, node, pxd_def=None):
1408 pxd_def = self.scope.lookup(node.class_name)
1410 outer_scope = self.scope
1411 self.scope = pxd_def.type.scope
1412 self.visitchildren(node)
1414 self.scope = outer_scope
1417 def visit_DefNode(self, node):
1418 pxd_def = self.scope.lookup(node.name)
1420 if not pxd_def.is_cfunction:
1421 error(node.pos, "'%s' redeclared" % node.name)
1422 error(pxd_def.pos, "previous declaration here")
1424 node = node.as_cfunction(pxd_def)
1425 elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1426 node = node.as_cfunction(scope=self.scope)
1427 # Enable this when internal def functions are allowed.
1428 # self.visitchildren(node)
1432 class YieldNodeCollector(TreeVisitor):
1435 super(YieldNodeCollector, self).__init__()
1438 self.has_return_value = False
1440 def visit_Node(self, node):
1441 return self.visitchildren(node)
1443 def visit_YieldExprNode(self, node):
1444 if self.has_return_value:
1445 error(node.pos, "'yield' outside function")
1446 self.yields.append(node)
1447 self.visitchildren(node)
1449 def visit_ReturnStatNode(self, node):
1451 self.has_return_value = True
1453 error(node.pos, "'return' with argument inside generator")
1454 self.returns.append(node)
1456 def visit_ClassDefNode(self, node):
1459 def visit_DefNode(self, node):
1462 def visit_LambdaNode(self, node):
1465 def visit_GeneratorExpressionNode(self, node):
1468 class MarkClosureVisitor(CythonTransform):
1470 def visit_ModuleNode(self, node):
1471 self.needs_closure = False
1472 self.visitchildren(node)
1475 def visit_FuncDefNode(self, node):
1476 self.needs_closure = False
1477 self.visitchildren(node)
1478 node.needs_closure = self.needs_closure
1479 self.needs_closure = True
1481 collector = YieldNodeCollector()
1482 collector.visitchildren(node)
1484 if collector.yields:
1485 for i, yield_expr in enumerate(collector.yields):
1486 yield_expr.label_num = i + 1
1488 gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1491 generator = Nodes.GeneratorDefNode(pos=node.pos,
1494 star_arg=node.star_arg,
1495 starstar_arg=node.starstar_arg,
1497 decorators=node.decorators,
1499 lambda_name=node.lambda_name)
1503 def visit_CFuncDefNode(self, node):
1504 self.visit_FuncDefNode(node)
1505 if node.needs_closure:
1506 error(node.pos, "closures inside cdef functions not yet supported")
1509 def visit_LambdaNode(self, node):
1510 self.needs_closure = False
1511 self.visitchildren(node)
1512 node.needs_closure = self.needs_closure
1513 self.needs_closure = True
1516 def visit_ClassDefNode(self, node):
1517 self.visitchildren(node)
1518 self.needs_closure = True
1521 class CreateClosureClasses(CythonTransform):
1522 # Output closure classes in module scope for all functions
1523 # that really need it.
1525 def __init__(self, context):
1526 super(CreateClosureClasses, self).__init__(context)
1528 self.in_lambda = False
1529 self.generator_class = None
1531 def visit_ModuleNode(self, node):
1532 self.module_scope = node.scope
1533 self.visitchildren(node)
1536 def create_generator_class(self, target_module_scope, pos):
1537 if self.generator_class:
1538 return self.generator_class
1539 # XXX: make generator class creation cleaner
1540 entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1541 objstruct_cname='__pyx_Generator_object',
1542 typeobj_cname='__pyx_Generator_type',
1543 pos=pos, defining=True, implementing=True)
1544 klass = entry.type.scope
1545 klass.is_internal = True
1546 klass.directives = {'final': True}
1548 body_type = PyrexTypes.create_typedef_type('generator_body',
1549 PyrexTypes.c_void_ptr_type,
1550 '__pyx_generator_body_t')
1551 klass.declare_var(pos=pos, name='body', cname='body',
1552 type=body_type, is_cdef=True)
1553 klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1555 klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1559 e = klass.declare_pyfunction('send', pos)
1560 e.func_cname = '__Pyx_Generator_Send'
1561 e.signature = TypeSlots.binaryfunc
1563 e = klass.declare_pyfunction('close', pos)
1564 e.func_cname = '__Pyx_Generator_Close'
1565 e.signature = TypeSlots.unaryfunc
1567 e = klass.declare_pyfunction('throw', pos)
1568 e.func_cname = '__Pyx_Generator_Throw'
1569 e.signature = TypeSlots.pyfunction_signature
1571 e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1572 e.func_cname = 'PyObject_SelfIter'
1574 e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1575 e.func_cname = '__Pyx_Generator_Next'
1577 self.generator_class = entry.type
1578 return self.generator_class
1580 def find_entries_used_in_closures(self, node):
1583 for name, entry in node.local_scope.entries.items():
1584 if entry.from_closure:
1585 from_closure.append((name, entry))
1586 elif entry.in_closure:
1587 in_closure.append((name, entry))
1588 return from_closure, in_closure
1590 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1591 # skip generator body
1592 if node.is_generator_body:
1594 # move local variables into closure
1595 if node.is_generator:
1596 for entry in node.local_scope.entries.values():
1597 if not entry.from_closure:
1598 entry.in_closure = True
1600 from_closure, in_closure = self.find_entries_used_in_closures(node)
1603 # Now from the begining
1604 node.needs_closure = False
1605 node.needs_outer_scope = False
1607 func_scope = node.local_scope
1608 cscope = node.entry.scope
1609 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1610 cscope = cscope.outer_scope
1612 if not from_closure and (self.path or inner_node):
1615 raise InternalError, "DefNode does not have assignment node"
1616 inner_node = node.assmt.rhs
1617 inner_node.needs_self_code = False
1618 node.needs_outer_scope = False
1621 if node.is_generator:
1622 base_type = self.create_generator_class(target_module_scope, node.pos)
1623 elif not in_closure and not from_closure:
1625 elif not in_closure:
1626 func_scope.is_passthrough = True
1627 func_scope.scope_class = cscope.scope_class
1628 node.needs_outer_scope = True
1631 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1633 entry = target_module_scope.declare_c_class(
1634 name=as_name, pos=node.pos, defining=True,
1635 implementing=True, base_type=base_type)
1637 func_scope.scope_class = entry
1638 class_scope = entry.type.scope
1639 class_scope.is_internal = True
1640 class_scope.directives = {'final': True}
1643 assert cscope.is_closure_scope
1644 class_scope.declare_var(pos=node.pos,
1645 name=Naming.outer_scope_cname,
1646 cname=Naming.outer_scope_cname,
1647 type=cscope.scope_class.type,
1649 node.needs_outer_scope = True
1650 for name, entry in in_closure:
1651 closure_entry = class_scope.declare_var(pos=entry.pos,
1656 if entry.is_declared_generic:
1657 closure_entry.is_declared_generic = 1
1658 node.needs_closure = True
1659 # Do it here because other classes are already checked
1660 target_module_scope.check_c_class(func_scope.scope_class)
1662 def visit_LambdaNode(self, node):
1663 was_in_lambda = self.in_lambda
1664 self.in_lambda = True
1665 self.create_class_from_scope(node.def_node, self.module_scope, node)
1666 self.visitchildren(node)
1667 self.in_lambda = was_in_lambda
1670 def visit_FuncDefNode(self, node):
1672 self.visitchildren(node)
1674 if node.needs_closure or self.path:
1675 self.create_class_from_scope(node, self.module_scope)
1676 self.path.append(node)
1677 self.visitchildren(node)
1682 class GilCheck(VisitorTransform):
1684 Call `node.gil_check(env)` on each node to make sure we hold the
1685 GIL when we need it. Raise an error when on Python operations
1686 inside a `nogil` environment.
1688 def __call__(self, root):
1689 self.env_stack = [root.scope]
1691 return super(GilCheck, self).__call__(root)
1693 def visit_FuncDefNode(self, node):
1694 self.env_stack.append(node.local_scope)
1695 was_nogil = self.nogil
1696 self.nogil = node.local_scope.nogil
1697 if self.nogil and node.nogil_check:
1698 node.nogil_check(node.local_scope)
1699 self.visitchildren(node)
1700 self.env_stack.pop()
1701 self.nogil = was_nogil
1704 def visit_GILStatNode(self, node):
1705 env = self.env_stack[-1]
1706 if self.nogil and node.nogil_check: node.nogil_check()
1707 was_nogil = self.nogil
1708 self.nogil = (node.state == 'nogil')
1709 self.visitchildren(node)
1710 self.nogil = was_nogil
1713 def visit_Node(self, node):
1714 if self.env_stack and self.nogil and node.nogil_check:
1715 node.nogil_check(self.env_stack[-1])
1716 self.visitchildren(node)
1720 class TransformBuiltinMethods(EnvTransform):
1722 def visit_SingleAssignmentNode(self, node):
1723 if node.declaration_only:
1726 self.visitchildren(node)
1729 def visit_AttributeNode(self, node):
1730 self.visitchildren(node)
1731 return self.visit_cython_attribute(node)
1733 def visit_NameNode(self, node):
1734 return self.visit_cython_attribute(node)
1736 def visit_cython_attribute(self, node):
1737 attribute = node.as_cython_attribute()
1739 if attribute == u'compiled':
1740 node = ExprNodes.BoolNode(node.pos, value=True)
1741 elif attribute == u'NULL':
1742 node = ExprNodes.NullNode(node.pos)
1743 elif attribute in (u'set', u'frozenset'):
1744 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1745 entry=self.current_env().builtin_scope().lookup_here(attribute))
1746 elif not PyrexTypes.parse_basic_type(attribute):
1747 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1750 def visit_SimpleCallNode(self, node):
1753 if isinstance(node.function, ExprNodes.NameNode):
1754 if node.function.name == 'locals':
1755 lenv = self.current_env()
1756 entry = lenv.lookup_here('locals')
1758 # not the builtin 'locals'
1760 if len(node.args) > 0:
1761 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1764 items = [ ExprNodes.DictItemNode(pos,
1765 key=ExprNodes.StringNode(pos, value=var),
1766 value=ExprNodes.NameNode(pos, name=var))
1767 for var in lenv.entries ]
1768 return ExprNodes.DictNode(pos, key_value_pairs=items)
1771 function = node.function.as_cython_attribute()
1773 if function in InterpretCompilerDirectives.unop_method_nodes:
1774 if len(node.args) != 1:
1775 error(node.function.pos, u"%s() takes exactly one argument" % function)
1777 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1778 elif function in InterpretCompilerDirectives.binop_method_nodes:
1779 if len(node.args) != 2:
1780 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1782 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1783 elif function == u'cast':
1784 if len(node.args) != 2:
1785 error(node.function.pos, u"cast() takes exactly two arguments")
1787 type = node.args[0].analyse_as_type(self.current_env())
1789 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1791 error(node.args[0].pos, "Not a type")
1792 elif function == u'sizeof':
1793 if len(node.args) != 1:
1794 error(node.function.pos, u"sizeof() takes exactly one argument")
1796 type = node.args[0].analyse_as_type(self.current_env())
1798 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1800 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1801 elif function == 'cmod':
1802 if len(node.args) != 2:
1803 error(node.function.pos, u"cmod() takes exactly two arguments")
1805 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1806 node.cdivision = True
1807 elif function == 'cdiv':
1808 if len(node.args) != 2:
1809 error(node.function.pos, u"cdiv() takes exactly two arguments")
1811 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1812 node.cdivision = True
1813 elif function == u'set':
1814 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1816 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1818 self.visitchildren(node)
1822 class DebugTransform(CythonTransform):
1824 Create debug information and all functions' visibility to extern in order
1825 to enable debugging.
1828 def __init__(self, context, options, result):
1829 super(DebugTransform, self).__init__(context)
1830 self.visited = cython.set()
1831 # our treebuilder and debug output writer
1832 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1833 self.tb = self.context.gdb_debug_outputwriter
1834 #self.c_output_file = options.output_file
1835 self.c_output_file = result.c_file
1837 # Closure support, basically treat nested functions as if the AST were
1839 self.nested_funcdefs = []
1841 # tells visit_NameNode whether it should register step-into functions
1842 self.register_stepinto = False
1844 def visit_ModuleNode(self, node):
1845 self.tb.module_name = node.full_module_name
1847 module_name=node.full_module_name,
1848 filename=node.pos[0].filename,
1849 c_filename=self.c_output_file)
1851 self.tb.start('Module', attrs)
1853 # serialize functions
1854 self.tb.start('Functions')
1855 # First, serialize functions normally...
1856 self.visitchildren(node)
1858 # ... then, serialize nested functions
1859 for nested_funcdef in self.nested_funcdefs:
1860 self.visit_FuncDefNode(nested_funcdef)
1862 self.register_stepinto = True
1863 self.serialize_modulenode_as_function(node)
1864 self.register_stepinto = False
1865 self.tb.end('Functions')
1867 # 2.3 compatibility. Serialize global variables
1868 self.tb.start('Globals')
1871 for k, v in node.scope.entries.iteritems():
1872 if (v.qualified_name not in self.visited and not
1873 v.name.startswith('__pyx_') and not
1874 v.type.is_cfunction and not
1875 v.type.is_extension_type):
1878 self.serialize_local_variables(entries)
1879 self.tb.end('Globals')
1880 # self.tb.end('Module') # end Module after the line number mapping in
1881 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1884 def visit_FuncDefNode(self, node):
1885 self.visited.add(node.local_scope.qualified_name)
1887 if getattr(node, 'is_wrapper', False):
1890 if self.register_stepinto:
1891 self.nested_funcdefs.append(node)
1894 # node.entry.visibility = 'extern'
1895 if node.py_func is None:
1898 pf_cname = node.py_func.entry.func_cname
1901 name=node.entry.name,
1902 cname=node.entry.func_cname,
1904 qualified_name=node.local_scope.qualified_name,
1905 lineno=str(node.pos[1]))
1907 self.tb.start('Function', attrs=attrs)
1909 self.tb.start('Locals')
1910 self.serialize_local_variables(node.local_scope.entries)
1911 self.tb.end('Locals')
1913 self.tb.start('Arguments')
1914 for arg in node.local_scope.arg_entries:
1915 self.tb.start(arg.name)
1916 self.tb.end(arg.name)
1917 self.tb.end('Arguments')
1919 self.tb.start('StepIntoFunctions')
1920 self.register_stepinto = True
1921 self.visitchildren(node)
1922 self.register_stepinto = False
1923 self.tb.end('StepIntoFunctions')
1924 self.tb.end('Function')
1928 def visit_NameNode(self, node):
1929 if (self.register_stepinto and
1930 node.type.is_cfunction and
1931 getattr(node, 'is_called', False) and
1932 node.entry.func_cname is not None):
1933 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1934 # declared functions are not 'in_cinclude'.
1935 # This means we will list called 'cdef' functions as
1936 # "step into functions", but this is not an issue as they will be
1937 # recognized as Cython functions anyway.
1938 attrs = dict(name=node.entry.func_cname)
1939 self.tb.start('StepIntoFunction', attrs=attrs)
1940 self.tb.end('StepIntoFunction')
1942 self.visitchildren(node)
1945 def serialize_modulenode_as_function(self, node):
1947 Serialize the module-level code as a function so the debugger will know
1948 it's a "relevant frame" and it will know where to set the breakpoint
1949 for 'break modulename'.
1951 name = node.full_module_name.rpartition('.')[-1]
1953 cname_py2 = 'init' + name
1954 cname_py3 = 'PyInit_' + name
1960 # Ignore the qualified_name, breakpoints should be set using
1961 # `cy break modulename:lineno` for module-level breakpoints.
1964 is_initmodule_function="True",
1967 py3_attrs = dict(py2_attrs, cname=cname_py3)
1969 self._serialize_modulenode_as_function(node, py2_attrs)
1970 self._serialize_modulenode_as_function(node, py3_attrs)
1972 def _serialize_modulenode_as_function(self, node, attrs):
1973 self.tb.start('Function', attrs=attrs)
1975 self.tb.start('Locals')
1976 self.serialize_local_variables(node.scope.entries)
1977 self.tb.end('Locals')
1979 self.tb.start('Arguments')
1980 self.tb.end('Arguments')
1982 self.tb.start('StepIntoFunctions')
1983 self.register_stepinto = True
1984 self.visitchildren(node)
1985 self.register_stepinto = False
1986 self.tb.end('StepIntoFunctions')
1988 self.tb.end('Function')
1990 def serialize_local_variables(self, entries):
1991 for entry in entries.values():
1992 if entry.type.is_pyobject:
1993 vartype = 'PythonObject'
1997 if entry.from_closure:
1998 # We're dealing with a closure where a variable from an outer
1999 # scope is accessed, get it from the scope object.
2000 cname = '%s->%s' % (Naming.cur_scope_cname,
2001 entry.outer_entry.cname)
2003 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2006 elif entry.in_closure:
2007 cname = '%s->%s' % (Naming.cur_scope_cname,
2009 qname = entry.qualified_name
2012 qname = entry.qualified_name
2015 # this happens for variables that are not in the user's code,
2016 # e.g. for the global __builtins__, __doc__, etc. We can just
2017 # set the lineno to 0 for those.
2020 lineno = str(entry.pos[1])
2025 qualified_name=qname,
2029 self.tb.start('LocalVar', attrs)
2030 self.tb.end('LocalVar')