3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
17 from Cython.Compiler.ModuleNode import ModuleNode
18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
20 from Cython.Compiler.StringEncoding import EncodedString
21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
26 class NameNodeCollector(TreeVisitor):
27 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
31 super(NameNodeCollector, self).__init__()
34 def visit_NameNode(self, node):
35 self.name_nodes.append(node)
37 def visit_Node(self, node):
38 self._visitchildren(node, None)
41 class SkipDeclarations(object):
43 Variable and function declarations can often have a deep tree structure,
44 and yet most transformations don't need to descend to this depth.
46 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
47 is no need to use this for transformations after that point.
49 def visit_CTypeDefNode(self, node):
52 def visit_CVarDefNode(self, node):
55 def visit_CDeclaratorNode(self, node):
58 def visit_CBaseTypeNode(self, node):
61 def visit_CEnumDefNode(self, node):
64 def visit_CStructOrUnionDefNode(self, node):
67 class NormalizeTree(CythonTransform):
69 This transform fixes up a few things after parsing
70 in order to make the parse tree more suitable for
73 a) After parsing, blocks with only one statement will
74 be represented by that statement, not by a StatListNode.
75 When doing transforms this is annoying and inconsistent,
76 as one cannot in general remove a statement in a consistent
77 way and so on. This transform wraps any single statements
78 in a StatListNode containing a single statement.
80 b) The PassStatNode is a noop and serves no purpose beyond
81 plugging such one-statement blocks; i.e., once parsed a
82 ` "pass" can just as well be represented using an empty
83 StatListNode. This means less special cases to worry about
84 in subsequent transforms (one always checks to see if a
85 StatListNode has no children to see if the block is empty).
88 def __init__(self, context):
89 super(NormalizeTree, self).__init__(context)
90 self.is_in_statlist = False
91 self.is_in_expr = False
93 def visit_ExprNode(self, node):
94 stacktmp = self.is_in_expr
95 self.is_in_expr = True
96 self.visitchildren(node)
97 self.is_in_expr = stacktmp
100 def visit_StatNode(self, node, is_listcontainer=False):
101 stacktmp = self.is_in_statlist
102 self.is_in_statlist = is_listcontainer
103 self.visitchildren(node)
104 self.is_in_statlist = stacktmp
105 if not self.is_in_statlist and not self.is_in_expr:
106 return Nodes.StatListNode(pos=node.pos, stats=[node])
110 def visit_StatListNode(self, node):
111 self.is_in_statlist = True
112 self.visitchildren(node)
113 self.is_in_statlist = False
116 def visit_ParallelAssignmentNode(self, node):
117 return self.visit_StatNode(node, True)
119 def visit_CEnumDefNode(self, node):
120 return self.visit_StatNode(node, True)
122 def visit_CStructOrUnionDefNode(self, node):
123 return self.visit_StatNode(node, True)
125 # Eliminate PassStatNode
126 def visit_PassStatNode(self, node):
127 if not self.is_in_statlist:
128 return Nodes.StatListNode(pos=node.pos, stats=[])
132 def visit_CDeclaratorNode(self, node):
136 class PostParseError(CompileError): pass
138 # error strings checked by unit tests, so define them
139 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
140 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
141 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
142 class PostParse(ScopeTrackingTransform):
144 Basic interpretation of the parse tree, as well as validity
145 checking that can be done on a very basic level on the parse
146 tree (while still not being a problem with the basic syntax,
150 - Default values to cdef assignments are turned into single
151 assignments following the declaration (everywhere but in class
152 bodies, where they raise a compile error)
154 - Interpret some node structures into Python runtime values.
155 Some nodes take compile-time arguments (currently:
156 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
157 which should be interpreted. This happens in a general way
158 and other steps should be taken to ensure validity.
160 Type arguments cannot be interpreted in this way.
162 - For __cythonbufferdefaults__ the arguments are checked for
165 TemplatedTypeNode has its directives interpreted:
166 Any first positional argument goes into the "dtype" attribute,
167 any "ndim" keyword argument goes into the "ndim" attribute and
168 so on. Also it is checked that the directive combination is valid.
169 - __cythonbufferdefaults__ attributes are parsed and put into the
172 Note: Currently Parsing.py does a lot of interpretation and
173 reorganization that can be refactored into this transform
174 if a more pure Abstract Syntax Tree is wanted.
177 def __init__(self, context):
178 super(PostParse, self).__init__(context)
179 self.specialattribute_handlers = {
180 '__cythonbufferdefaults__' : self.handle_bufferdefaults
183 def visit_ModuleNode(self, node):
184 self.lambda_counter = 1
185 self.genexpr_counter = 1
186 return super(PostParse, self).visit_ModuleNode(node)
188 def visit_LambdaNode(self, node):
189 # unpack a lambda expression into the corresponding DefNode
190 lambda_id = self.lambda_counter
191 self.lambda_counter += 1
192 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
193 collector = YieldNodeCollector()
194 collector.visitchildren(node.result_expr)
195 if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
196 body = ExprNodes.YieldExprNode(
197 node.result_expr.pos, arg=node.result_expr)
198 body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
200 body = Nodes.ReturnStatNode(
201 node.result_expr.pos, value=node.result_expr)
202 node.def_node = Nodes.DefNode(
203 node.pos, name=node.name, lambda_name=node.lambda_name,
204 args=node.args, star_arg=node.star_arg,
205 starstar_arg=node.starstar_arg,
207 self.visitchildren(node)
210 def visit_GeneratorExpressionNode(self, node):
211 # unpack a generator expression into the corresponding DefNode
212 genexpr_id = self.genexpr_counter
213 self.genexpr_counter += 1
214 node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
216 node.def_node = Nodes.DefNode(node.pos, name=node.name,
218 args=[], star_arg=None,
221 self.visitchildren(node)
225 def handle_bufferdefaults(self, decl):
226 if not isinstance(decl.default, ExprNodes.DictNode):
227 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
228 self.scope_node.buffer_defaults_node = decl.default
229 self.scope_node.buffer_defaults_pos = decl.pos
231 def visit_CVarDefNode(self, node):
232 # This assumes only plain names and pointers are assignable on
233 # declaration. Also, it makes use of the fact that a cdef decl
234 # must appear before the first use, so we don't have to deal with
235 # "i = 3; cdef int i = i" and can simply move the nodes around.
237 self.visitchildren(node)
240 for decl in node.declarators:
242 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
243 declbase = declbase.base
244 if isinstance(declbase, Nodes.CNameDeclaratorNode):
245 if declbase.default is not None:
246 if self.scope_type in ('cclass', 'pyclass', 'struct'):
247 if isinstance(self.scope_node, Nodes.CClassDefNode):
248 handler = self.specialattribute_handlers.get(decl.name)
250 if decl is not declbase:
251 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
253 continue # Remove declaration
254 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
255 first_assignment = self.scope_type != 'module'
256 stats.append(Nodes.SingleAssignmentNode(node.pos,
257 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
258 rhs=declbase.default, first=first_assignment))
259 declbase.default = None
260 newdecls.append(decl)
261 node.declarators = newdecls
263 except PostParseError, e:
264 # An error in a cdef clause is ok, simply remove the declaration
265 # and try to move on to report more errors
266 self.context.nonfatal_error(e)
269 # Split parallel assignments (a,b = b,a) into separate partial
270 # assignments that are executed rhs-first using temps. This
271 # restructuring must be applied before type analysis so that known
272 # types on rhs and lhs can be matched directly. It is required in
273 # the case that the types cannot be coerced to a Python type in
274 # order to assign from a tuple.
276 def visit_SingleAssignmentNode(self, node):
277 self.visitchildren(node)
278 return self._visit_assignment_node(node, [node.lhs, node.rhs])
280 def visit_CascadedAssignmentNode(self, node):
281 self.visitchildren(node)
282 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
284 def _visit_assignment_node(self, node, expr_list):
285 """Flatten parallel assignments into separate single
286 assignments or cascaded assignments.
288 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
289 # no parallel assignments => nothing to do
293 flatten_parallel_assignments(expr_list, expr_list_list)
295 eliminate_rhs_duplicates(expr_list_list, temp_refs)
298 for expr_list in expr_list_list:
299 lhs_list = expr_list[:-1]
301 if len(lhs_list) == 1:
302 node = Nodes.SingleAssignmentNode(rhs.pos,
303 lhs = lhs_list[0], rhs = rhs)
305 node = Nodes.CascadedAssignmentNode(rhs.pos,
306 lhs_list = lhs_list, rhs = rhs)
310 assign_node = nodes[0]
312 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
315 duplicates_and_temps = [ (temp.expression, temp)
316 for temp in temp_refs ]
317 sort_common_subsequences(duplicates_and_temps)
318 for _, temp_ref in duplicates_and_temps[::-1]:
319 assign_node = LetNode(temp_ref, assign_node)
323 def _flatten_sequence(self, seq, result):
325 if arg.is_sequence_constructor:
326 self._flatten_sequence(arg, result)
331 def visit_DelStatNode(self, node):
332 self.visitchildren(node)
333 node.args = self._flatten_sequence(node, [])
337 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
338 """Replace rhs items by LetRefNodes if they appear more than once.
339 Creates a sequence of LetRefNodes that set up the required temps
340 and appends them to ref_node_sequence. The input list is modified
343 seen_nodes = cython.set()
345 def find_duplicates(node):
346 if node.is_literal or node.is_name:
347 # no need to replace those; can't include attributes here
348 # as their access is not necessarily side-effect free
350 if node in seen_nodes:
351 if node not in ref_nodes:
352 ref_node = LetRefNode(node)
353 ref_nodes[node] = ref_node
354 ref_node_sequence.append(ref_node)
357 if node.is_sequence_constructor:
358 for item in node.args:
359 find_duplicates(item)
361 for expr_list in expr_list_list:
367 def substitute_nodes(node):
368 if node in ref_nodes:
369 return ref_nodes[node]
370 elif node.is_sequence_constructor:
371 node.args = list(map(substitute_nodes, node.args))
374 # replace nodes inside of the common subexpressions
375 for node in ref_nodes:
376 if node.is_sequence_constructor:
377 node.args = list(map(substitute_nodes, node.args))
379 # replace common subexpressions on all rhs items
380 for expr_list in expr_list_list:
381 expr_list[-1] = substitute_nodes(expr_list[-1])
383 def sort_common_subsequences(items):
384 """Sort items/subsequences so that all items and subsequences that
385 an item contains appear before the item itself. This is needed
386 because each rhs item must only be evaluated once, so its value
387 must be evaluated first and then reused when packing sequences
390 This implies a partial order, and the sort must be stable to
391 preserve the original order as much as possible, so we use a
392 simple insertion sort (which is very fast for short sequences, the
393 normal case in practice).
395 def contains(seq, x):
399 elif item.is_sequence_constructor and contains(item.args, x):
403 return b.is_sequence_constructor and contains(b.args, a)
405 for pos, item in enumerate(items):
406 key = item[1] # the ResultRefNode which has already been injected into the sequences
408 for i in xrange(pos-1, -1, -1):
409 if lower_than(key, items[i][0]):
412 for i in xrange(pos, new_pos, -1):
413 items[i] = items[i-1]
414 items[new_pos] = item
416 def flatten_parallel_assignments(input, output):
417 # The input is a list of expression nodes, representing the LHSs
418 # and RHS of one (possibly cascaded) assignment statement. For
419 # sequence constructors, rearranges the matching parts of both
420 # sides into a list of equivalent assignments between the
421 # individual elements. This transformation is applied
422 # recursively, so that nested structures get matched as well.
424 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
428 complete_assignments = []
430 rhs_size = len(rhs.args)
431 lhs_targets = [ [] for _ in xrange(rhs_size) ]
432 starred_assignments = []
433 for lhs in input[:-1]:
434 if not lhs.is_sequence_constructor:
436 error(lhs.pos, "starred assignment target must be in a list or tuple")
437 complete_assignments.append(lhs)
439 lhs_size = len(lhs.args)
440 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
441 if starred_targets > 1:
442 error(lhs.pos, "more than 1 starred expression in assignment")
443 output.append([lhs,rhs])
445 elif lhs_size - starred_targets > rhs_size:
446 error(lhs.pos, "need more than %d value%s to unpack"
447 % (rhs_size, (rhs_size != 1) and 's' or ''))
448 output.append([lhs,rhs])
450 elif starred_targets:
451 map_starred_assignment(lhs_targets, starred_assignments,
453 elif lhs_size < rhs_size:
454 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
455 % (lhs_size, rhs_size))
456 output.append([lhs,rhs])
459 for targets, expr in zip(lhs_targets, lhs.args):
462 if complete_assignments:
463 complete_assignments.append(rhs)
464 output.append(complete_assignments)
466 # recursively flatten partial assignments
467 for cascade, rhs in zip(lhs_targets, rhs.args):
470 flatten_parallel_assignments(cascade, output)
472 # recursively flatten starred assignments
473 for cascade in starred_assignments:
474 if cascade[0].is_sequence_constructor:
475 flatten_parallel_assignments(cascade, output)
477 output.append(cascade)
479 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
480 # Appends the fixed-position LHS targets to the target list that
481 # appear left and right of the starred argument.
483 # The starred_assignments list receives a new tuple
484 # (lhs_target, rhs_values_list) that maps the remaining arguments
485 # (those that match the starred target) to a list.
487 # left side of the starred target
488 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
491 lhs_remaining = len(lhs_args) - i - 1
495 raise InternalError("no starred arg found when splitting starred assignment")
497 # right side of the starred target
498 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
499 lhs_args[starred + 1:])):
502 # the starred target itself, must be assigned a (potentially empty) list
503 target = lhs_args[starred].target # unpack starred node
504 starred_rhs = rhs_args[starred:]
506 starred_rhs = starred_rhs[:-lhs_remaining]
508 pos = starred_rhs[0].pos
511 starred_assignments.append([
512 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
515 class PxdPostParse(CythonTransform, SkipDeclarations):
517 Basic interpretation/validity checking that should only be
520 A lot of this checking currently happens in the parser; but
521 what is listed below happens here.
523 - "def" functions are let through only if they fill the
524 getbuffer/releasebuffer slots
526 - cdef functions are let through only if they are on the
527 top level and are declared "inline"
529 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
530 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
532 def __call__(self, node):
533 self.scope_type = 'pxd'
534 return super(PxdPostParse, self).__call__(node)
536 def visit_CClassDefNode(self, node):
537 old = self.scope_type
538 self.scope_type = 'cclass'
539 self.visitchildren(node)
540 self.scope_type = old
543 def visit_FuncDefNode(self, node):
544 # FuncDefNode always come with an implementation (without
545 # an imp they are CVarDefNodes..)
546 err = self.ERR_INLINE_ONLY
548 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
549 and node.name in ('__getbuffer__', '__releasebuffer__')):
550 err = None # allow these slots
552 if isinstance(node, Nodes.CFuncDefNode):
553 if u'inline' in node.modifiers and self.scope_type == 'pxd':
554 node.inline_in_pxd = True
555 if node.visibility != 'private':
556 err = self.ERR_NOGO_WITH_INLINE % node.visibility
558 err = self.ERR_NOGO_WITH_INLINE % 'api'
560 err = None # allow inline function
562 err = self.ERR_INLINE_ONLY
565 self.context.nonfatal_error(PostParseError(node.pos, err))
570 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
572 After parsing, directives can be stored in a number of places:
573 - #cython-comments at the top of the file (stored in ModuleNode)
574 - Command-line arguments overriding these
575 - @cython.directivename decorators
576 - with cython.directivename: statements
578 This transform is responsible for interpreting these various sources
579 and store the directive in two ways:
580 - Set the directives attribute of the ModuleNode for global directives.
581 - Use a CompilerDirectivesNode to override directives for a subtree.
583 (The first one is primarily to not have to modify with the tree
584 structure, so that ModuleNode stay on top.)
586 The directives are stored in dictionaries from name to value in effect.
587 Each such dictionary is always filled in for all possible directives,
588 using default values where no value is given by the user.
590 The available directives are controlled in Options.py.
592 Note that we have to run this prior to analysis, and so some minor
593 duplication of functionality has to occur: We manually track cimports
594 and which names the "cython" module may have been imported to.
596 unop_method_nodes = {
597 'typeof': ExprNodes.TypeofNode,
599 'operator.address': ExprNodes.AmpersandNode,
600 'operator.dereference': ExprNodes.DereferenceNode,
601 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
602 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
603 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
604 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
606 # For backwards compatability.
607 'address': ExprNodes.AmpersandNode,
610 binop_method_nodes = {
611 'operator.comma' : ExprNodes.c_binop_constructor(','),
614 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
615 'cast', 'pointer', 'compiled', 'NULL'])
616 special_methods.update(unop_method_nodes.keys())
618 def __init__(self, context, compilation_directive_defaults):
619 super(InterpretCompilerDirectives, self).__init__(context)
620 self.compilation_directive_defaults = {}
621 for key, value in compilation_directive_defaults.items():
622 self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
623 self.cython_module_names = cython.set()
624 self.directive_names = {}
626 def check_directive_scope(self, pos, directive, scope):
627 legal_scopes = Options.directive_scopes.get(directive, None)
628 if legal_scopes and scope not in legal_scopes:
629 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
630 'is not allowed in %s scope' % (directive, scope)))
635 # Set up processing and handle the cython: comments.
636 def visit_ModuleNode(self, node):
637 for key, value in node.directive_comments.items():
638 if not self.check_directive_scope(node.pos, key, 'module'):
639 self.wrong_scope_error(node.pos, key, 'module')
640 del node.directive_comments[key]
642 directives = copy.deepcopy(Options.directive_defaults)
643 directives.update(copy.deepcopy(self.compilation_directive_defaults))
644 directives.update(node.directive_comments)
645 self.directives = directives
646 node.directives = directives
647 self.visitchildren(node)
648 node.cython_module_names = self.cython_module_names
651 # The following four functions track imports and cimports that
652 # begin with "cython"
653 def is_cython_directive(self, name):
654 return (name in Options.directive_types or
655 name in self.special_methods or
656 PyrexTypes.parse_basic_type(name))
658 def visit_CImportStatNode(self, node):
659 if node.module_name == u"cython":
660 self.cython_module_names.add(node.as_name or u"cython")
661 elif node.module_name.startswith(u"cython."):
663 self.directive_names[node.as_name] = node.module_name[7:]
665 self.cython_module_names.add(u"cython")
666 # if this cimport was a compiler directive, we don't
667 # want to leave the cimport node sitting in the tree
671 def visit_FromCImportStatNode(self, node):
672 if (node.module_name == u"cython") or \
673 node.module_name.startswith(u"cython."):
674 submodule = (node.module_name + u".")[7:]
676 for pos, name, as_name, kind in node.imported_names:
677 full_name = submodule + name
678 if self.is_cython_directive(full_name):
681 self.directive_names[as_name] = full_name
683 self.context.nonfatal_error(PostParseError(pos,
684 "Compiler directive imports must be plain imports"))
686 newimp.append((pos, name, as_name, kind))
689 node.imported_names = newimp
692 def visit_FromImportStatNode(self, node):
693 if (node.module.module_name.value == u"cython") or \
694 node.module.module_name.value.startswith(u"cython."):
695 submodule = (node.module.module_name.value + u".")[7:]
697 for name, name_node in node.items:
698 full_name = submodule + name
699 if self.is_cython_directive(full_name):
700 self.directive_names[name_node.name] = full_name
702 newimp.append((name, name_node))
708 def visit_SingleAssignmentNode(self, node):
709 if (isinstance(node.rhs, ExprNodes.ImportNode) and
710 node.rhs.module_name.value == u'cython'):
711 node = Nodes.CImportStatNode(node.pos,
712 module_name = u'cython',
713 as_name = node.lhs.name)
714 self.visit_CImportStatNode(node)
716 self.visitchildren(node)
719 def visit_NameNode(self, node):
720 if node.name in self.cython_module_names:
721 node.is_cython_module = True
723 node.cython_attribute = self.directive_names.get(node.name)
726 def try_to_parse_directives(self, node):
727 # If node is the contents of an directive (in a with statement or
728 # decorator), returns a list of (directivename, value) pairs.
729 # Otherwise, returns None
730 if isinstance(node, ExprNodes.CallNode):
731 self.visit(node.function)
732 optname = node.function.as_cython_attribute()
734 directivetype = Options.directive_types.get(optname)
736 args, kwds = node.explicit_args_kwds()
739 if kwds is not None and directivetype is not dict:
740 for keyvalue in kwds.key_value_pairs:
741 key, value = keyvalue
742 sub_optname = "%s.%s" % (optname, key.value)
743 if Options.directive_types.get(sub_optname):
744 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
746 key_value_pairs.append(keyvalue)
747 if not key_value_pairs:
750 kwds.key_value_pairs = key_value_pairs
751 if directives and not kwds and not args:
753 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
755 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
757 optname = node.as_cython_attribute()
759 directivetype = Options.directive_types.get(optname)
760 if directivetype is bool:
761 return [(optname, True)]
762 elif directivetype is None:
763 return [(optname, None)]
765 raise PostParseError(
766 node.pos, "The '%s' directive should be used as a function call." % optname)
769 def try_to_parse_directive(self, optname, args, kwds, pos):
770 directivetype = Options.directive_types.get(optname)
771 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
772 return optname, Options.directive_defaults[optname]
773 elif directivetype is bool:
774 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
775 raise PostParseError(pos,
776 'The %s directive takes one compile-time boolean argument' % optname)
777 return (optname, args[0].value)
778 elif directivetype is str:
779 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
780 ExprNodes.UnicodeNode)):
781 raise PostParseError(pos,
782 'The %s directive takes one compile-time string argument' % optname)
783 return (optname, str(args[0].value))
784 elif directivetype is dict:
786 raise PostParseError(pos,
787 'The %s directive takes no prepositional arguments' % optname)
788 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
789 elif directivetype is list:
790 if kwds and len(kwds) != 0:
791 raise PostParseError(pos,
792 'The %s directive takes no keyword arguments' % optname)
793 return optname, [ str(arg.value) for arg in args ]
797 def visit_with_directives(self, body, directives):
798 olddirectives = self.directives
799 newdirectives = copy.copy(olddirectives)
800 newdirectives.update(directives)
801 self.directives = newdirectives
802 assert isinstance(body, Nodes.StatListNode), body
803 retbody = self.visit_Node(body)
804 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
805 directives=newdirectives)
806 self.directives = olddirectives
810 def visit_FuncDefNode(self, node):
811 directives = self._extract_directives(node, 'function')
813 return self.visit_Node(node)
814 body = Nodes.StatListNode(node.pos, stats=[node])
815 return self.visit_with_directives(body, directives)
817 def visit_CVarDefNode(self, node):
818 if not node.decorators:
820 for dec in node.decorators:
821 for directive in self.try_to_parse_directives(dec.decorator) or ():
822 if directive is not None and directive[0] == u'locals':
823 node.directive_locals = directive[1]
825 self.context.nonfatal_error(PostParseError(dec.pos,
826 "Cdef functions can only take cython.locals() decorator."))
829 def visit_CClassDefNode(self, node):
830 directives = self._extract_directives(node, 'cclass')
832 return self.visit_Node(node)
833 body = Nodes.StatListNode(node.pos, stats=[node])
834 return self.visit_with_directives(body, directives)
836 def visit_PyClassDefNode(self, node):
837 directives = self._extract_directives(node, 'class')
839 return self.visit_Node(node)
840 body = Nodes.StatListNode(node.pos, stats=[node])
841 return self.visit_with_directives(body, directives)
843 def _extract_directives(self, node, scope_name):
844 if not node.decorators:
846 # Split the decorators into two lists -- real decorators and directives
849 for dec in node.decorators:
850 new_directives = self.try_to_parse_directives(dec.decorator)
851 if new_directives is not None:
852 for directive in new_directives:
853 if self.check_directive_scope(node.pos, directive[0], scope_name):
854 directives.append(directive)
857 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
858 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
860 node.decorators = realdecs
861 # merge or override repeated directives
863 directives.reverse() # Decorators coming first take precedence
864 for directive in directives:
865 name, value = directive
867 old_value = optdict[name]
868 # keywords and arg lists can be merged, everything
869 # else overrides completely
870 if isinstance(old_value, dict):
871 old_value.update(value)
872 elif isinstance(old_value, list):
873 old_value.extend(value)
875 optdict[name] = value
877 optdict[name] = value
880 # Handle with statements
881 def visit_WithStatNode(self, node):
883 for directive in self.try_to_parse_directives(node.manager) or []:
884 if directive is not None:
885 if node.target is not None:
886 self.context.nonfatal_error(
887 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
889 name, value = directive
891 # special case: in pure mode, "with nogil" spells "with cython.nogil"
892 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
893 return self.visit_Node(node)
894 if self.check_directive_scope(node.pos, name, 'with statement'):
895 directive_dict[name] = value
897 return self.visit_with_directives(node.body, directive_dict)
898 return self.visit_Node(node)
900 class WithTransform(CythonTransform, SkipDeclarations):
902 # EXCINFO is manually set to a variable that contains
903 # the exc_info() tuple that can be generated by the enclosing except
905 template_without_target = TreeFragment(u"""
916 if not EXIT(*EXCINFO):
920 EXIT(None, None, None)
921 """, temps=[u'MGR', u'EXC', u"EXIT"],
922 pipeline=[NormalizeTree(None)])
924 template_with_target = TreeFragment(u"""
927 VALUE = MGR.__enter__()
936 if not EXIT(*EXCINFO):
940 EXIT(None, None, None)
941 MGR = EXIT = VALUE = EXC = None
943 """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
944 pipeline=[NormalizeTree(None)])
946 def visit_WithStatNode(self, node):
947 # TODO: Cleanup badly needed
948 TemplateTransform.temp_name_counter += 1
949 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
951 self.visitchildren(node, ['body'])
952 excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
953 if node.target is not None:
954 result = self.template_with_target.substitute({
955 u'EXPR' : node.manager,
957 u'TARGET' : node.target,
958 u'EXCINFO' : excinfo_temp
961 result = self.template_without_target.substitute({
962 u'EXPR' : node.manager,
964 u'EXCINFO' : excinfo_temp
967 # Set except excinfo target to EXCINFO
968 try_except = result.stats[-1].body.stats[-1]
969 try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
970 # excinfo_temp.ref(node.pos))
972 # result.stats[-1].body.stats[-1] = TempsBlockNode(
973 # node.pos, temps=[excinfo_temp], body=try_except)
977 def visit_ExprNode(self, node):
978 # With statements are never inside expressions.
982 class DecoratorTransform(CythonTransform, SkipDeclarations):
984 def visit_DefNode(self, func_node):
985 self.visitchildren(func_node)
986 if not func_node.decorators:
988 return self._handle_decorators(
989 func_node, func_node.name)
991 def visit_CClassDefNode(self, class_node):
992 # This doesn't currently work, so it's disabled.
994 # Problem: assignments to cdef class names do not work. They
995 # would require an additional check anyway, as the extension
996 # type must not change its C type, so decorators cannot
997 # replace an extension type, just alter it and return it.
999 self.visitchildren(class_node)
1000 if not class_node.decorators:
1002 error(class_node.pos,
1003 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
1005 #return self._handle_decorators(
1006 # class_node, class_node.class_name)
1008 def visit_ClassDefNode(self, class_node):
1009 self.visitchildren(class_node)
1010 if not class_node.decorators:
1012 return self._handle_decorators(
1013 class_node, class_node.name)
1015 def _handle_decorators(self, node, name):
1016 decorator_result = ExprNodes.NameNode(node.pos, name = name)
1017 for decorator in node.decorators[::-1]:
1018 decorator_result = ExprNodes.SimpleCallNode(
1020 function = decorator.decorator,
1021 args = [decorator_result])
1023 name_node = ExprNodes.NameNode(node.pos, name = name)
1024 reassignment = Nodes.SingleAssignmentNode(
1027 rhs = decorator_result)
1028 return [node, reassignment]
1031 class AnalyseDeclarationsTransform(CythonTransform):
1033 basic_property = TreeFragment(u"""
1037 def __set__(self, value):
1039 """, level='c_class')
1040 basic_pyobject_property = TreeFragment(u"""
1044 def __set__(self, value):
1048 """, level='c_class')
1049 basic_property_ro = TreeFragment(u"""
1053 """, level='c_class')
1055 struct_or_union_wrapper = TreeFragment(u"""
1058 def __init__(self, MEMBER=None):
1062 if IS_UNION and count > 1:
1063 raise ValueError, "At most one union member should be specified."
1065 return STR_FORMAT % MEMBER_TUPLE
1067 return REPR_FORMAT % MEMBER_TUPLE
1070 init_assignment = TreeFragment(u"""
1071 if VALUE is not None:
1076 def __call__(self, root):
1077 self.env_stack = [root.scope]
1078 # needed to determine if a cdef var is declared after it's used.
1079 self.seen_vars_stack = []
1080 return super(AnalyseDeclarationsTransform, self).__call__(root)
1082 def visit_NameNode(self, node):
1083 self.seen_vars_stack[-1].add(node.name)
1086 def visit_ModuleNode(self, node):
1087 self.seen_vars_stack.append(cython.set())
1088 node.analyse_declarations(self.env_stack[-1])
1089 self.visitchildren(node)
1090 self.seen_vars_stack.pop()
1093 def visit_LambdaNode(self, node):
1094 node.analyse_declarations(self.env_stack[-1])
1095 self.visitchildren(node)
1098 def visit_ClassDefNode(self, node):
1099 self.env_stack.append(node.scope)
1100 self.visitchildren(node)
1101 self.env_stack.pop()
1104 def visit_CClassDefNode(self, node):
1105 node = self.visit_ClassDefNode(node)
1106 if node.scope and node.scope.implemented:
1108 for entry in node.scope.var_entries:
1109 if entry.needs_property:
1110 property = self.create_Property(entry)
1111 property.analyse_declarations(node.scope)
1112 self.visit(property)
1113 stats.append(property)
1115 node.body.stats += stats
1118 def visit_FuncDefNode(self, node):
1119 self.seen_vars_stack.append(cython.set())
1120 lenv = node.local_scope
1121 node.body.analyse_control_flow(lenv) # this will be totally refactored
1122 node.declare_arguments(lenv)
1123 for var, type_node in node.directive_locals.items():
1124 if not lenv.lookup_here(var): # don't redeclare args
1125 type = type_node.analyse_as_type(lenv)
1127 lenv.declare_var(var, type, type_node.pos)
1129 error(type_node.pos, "Not a type")
1130 node.body.analyse_declarations(lenv)
1131 self.env_stack.append(lenv)
1132 self.visitchildren(node)
1133 self.env_stack.pop()
1134 self.seen_vars_stack.pop()
1137 def visit_ScopedExprNode(self, node):
1138 env = self.env_stack[-1]
1139 node.analyse_declarations(env)
1140 # the node may or may not have a local scope
1141 if node.has_local_scope:
1142 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1143 self.env_stack.append(node.expr_scope)
1144 node.analyse_scoped_declarations(node.expr_scope)
1145 self.visitchildren(node)
1146 self.env_stack.pop()
1147 self.seen_vars_stack.pop()
1149 node.analyse_scoped_declarations(env)
1150 self.visitchildren(node)
1153 def visit_TempResultFromStatNode(self, node):
1154 self.visitchildren(node)
1155 node.analyse_declarations(self.env_stack[-1])
1158 def visit_CStructOrUnionDefNode(self, node):
1159 # Create a wrapper node if needed.
1160 # We want to use the struct type information (so it can't happen
1161 # before this phase) but also create new objects to be declared
1162 # (so it can't happen later).
1163 # Note that we don't return the original node, as it is
1164 # never used after this phase.
1165 if True: # private (default)
1168 self_value = ExprNodes.AttributeNode(
1170 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1171 attribute = EncodedString(u"value"))
1172 var_entries = node.entry.type.scope.var_entries
1174 for entry in var_entries:
1175 attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1177 attribute = entry.name))
1178 # __init__ assignments
1179 init_assignments = []
1180 for entry, attr in zip(var_entries, attributes):
1181 # TODO: branch on visibility
1182 init_assignments.append(self.init_assignment.substitute({
1183 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1185 }, pos = entry.pos))
1188 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1189 wrapper_class = self.struct_or_union_wrapper.substitute({
1190 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1191 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1192 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1193 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1194 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1195 }, pos = node.pos).stats[0]
1196 wrapper_class.class_name = node.name
1197 wrapper_class.shadow = True
1198 class_body = wrapper_class.body.stats
1201 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1202 class_body[0].base_type.name = node.name
1204 # fix __init__ arguments
1205 init_method = class_body[1]
1206 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1207 arg_template = init_method.args[1]
1208 if not node.entry.type.is_struct:
1209 arg_template.kw_only = True
1210 del init_method.args[1]
1211 for entry, attr in zip(var_entries, attributes):
1212 arg = copy.deepcopy(arg_template)
1213 arg.declarator.name = entry.name
1214 init_method.args.append(arg)
1217 for entry, attr in zip(var_entries, attributes):
1218 # TODO: branch on visibility
1219 if entry.type.is_pyobject:
1220 template = self.basic_pyobject_property
1222 template = self.basic_property
1223 property = template.substitute({
1225 }, pos = entry.pos).stats[0]
1226 property.name = entry.name
1227 wrapper_class.body.stats.append(property)
1229 wrapper_class.analyse_declarations(self.env_stack[-1])
1230 return self.visit_CClassDefNode(wrapper_class)
1232 # Some nodes are no longer needed after declaration
1233 # analysis and can be dropped. The analysis was performed
1234 # on these nodes in a seperate recursive process from the
1235 # enclosing function or module, so we can simply drop them.
1236 def visit_CDeclaratorNode(self, node):
1237 # necessary to ensure that all CNameDeclaratorNodes are visited.
1238 self.visitchildren(node)
1241 def visit_CTypeDefNode(self, node):
1244 def visit_CBaseTypeNode(self, node):
1247 def visit_CEnumDefNode(self, node):
1248 if node.visibility == 'public':
1253 def visit_CNameDeclaratorNode(self, node):
1254 if node.name in self.seen_vars_stack[-1]:
1255 entry = self.env_stack[-1].lookup(node.name)
1256 if (entry is None or entry.visibility != 'extern'
1257 and not entry.scope.is_c_class_scope):
1258 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1259 self.visitchildren(node)
1262 def visit_CVarDefNode(self, node):
1263 # to ensure all CNameDeclaratorNodes are visited.
1264 self.visitchildren(node)
1267 def create_Property(self, entry):
1268 if entry.visibility == 'public':
1269 if entry.type.is_pyobject:
1270 template = self.basic_pyobject_property
1272 template = self.basic_property
1273 elif entry.visibility == 'readonly':
1274 template = self.basic_property_ro
1275 property = template.substitute({
1276 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1277 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1278 attribute=entry.name),
1279 }, pos=entry.pos).stats[0]
1280 property.name = entry.name
1281 # ---------------------------------------
1282 # XXX This should go to AutoDocTransforms
1283 # ---------------------------------------
1284 if (Options.docstrings and
1285 self.current_directives['embedsignature']):
1286 attr_name = entry.name
1287 type_name = entry.type.declaration_code("", for_display=1)
1289 if not entry.type.is_pyobject:
1290 type_name = "'%s'" % type_name
1291 elif entry.type.is_extension_type:
1292 type_name = entry.type.module_name + '.' + type_name
1293 if entry.init is not None:
1294 default_value = ' = ' + entry.init
1295 elif entry.init_to_none:
1296 default_value = ' = ' + repr(None)
1297 docstring = attr_name + ': ' + type_name + default_value
1298 property.doc = EncodedString(docstring)
1299 # ---------------------------------------
1302 class AnalyseExpressionsTransform(CythonTransform):
1304 def visit_ModuleNode(self, node):
1305 node.scope.infer_types()
1306 node.body.analyse_expressions(node.scope)
1307 self.visitchildren(node)
1310 def visit_FuncDefNode(self, node):
1311 node.local_scope.infer_types()
1312 node.body.analyse_expressions(node.local_scope)
1313 self.visitchildren(node)
1316 def visit_ScopedExprNode(self, node):
1317 if node.has_local_scope:
1318 node.expr_scope.infer_types()
1319 node.analyse_scoped_expressions(node.expr_scope)
1320 self.visitchildren(node)
1323 class ExpandInplaceOperators(EnvTransform):
1325 def visit_InPlaceAssignmentNode(self, node):
1328 if lhs.type.is_cpp_class:
1329 # No getting around this exact operator here.
1331 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1332 # There is code to handle this case.
1335 env = self.current_env()
1336 def side_effect_free_reference(node, setting=False):
1337 if isinstance(node, ExprNodes.NameNode):
1339 elif node.type.is_pyobject and not setting:
1340 node = LetRefNode(node)
1342 elif isinstance(node, ExprNodes.IndexNode):
1343 if node.is_buffer_access:
1344 raise ValueError, "Buffer access"
1345 base, temps = side_effect_free_reference(node.base)
1346 index = LetRefNode(node.index)
1347 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1348 elif isinstance(node, ExprNodes.AttributeNode):
1349 obj, temps = side_effect_free_reference(node.obj)
1350 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1352 node = LetRefNode(node)
1355 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1358 dup = lhs.__class__(**lhs.__dict__)
1359 binop = ExprNodes.binop_node(node.pos,
1360 operator = node.operator,
1364 # Manually analyse types for new node.
1365 lhs.analyse_target_types(env)
1366 dup.analyse_types(env)
1367 binop.analyse_operation(env)
1368 node = Nodes.SingleAssignmentNode(
1371 rhs=binop.coerce_to(lhs.type, env))
1372 # Use LetRefNode to avoid side effects.
1373 let_ref_nodes.reverse()
1374 for t in let_ref_nodes:
1375 node = LetNode(t, node)
1378 def visit_ExprNode(self, node):
1379 # In-place assignments can't happen within an expression.
1383 class AlignFunctionDefinitions(CythonTransform):
1385 This class takes the signatures from a .pxd file and applies them to
1386 the def methods in a .py file.
1389 def visit_ModuleNode(self, node):
1390 self.scope = node.scope
1391 self.directives = node.directives
1392 self.visitchildren(node)
1395 def visit_PyClassDefNode(self, node):
1396 pxd_def = self.scope.lookup(node.name)
1398 if pxd_def.is_cclass:
1399 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1401 error(node.pos, "'%s' redeclared" % node.name)
1403 error(pxd_def.pos, "previous declaration here")
1408 def visit_CClassDefNode(self, node, pxd_def=None):
1410 pxd_def = self.scope.lookup(node.class_name)
1412 outer_scope = self.scope
1413 self.scope = pxd_def.type.scope
1414 self.visitchildren(node)
1416 self.scope = outer_scope
1419 def visit_DefNode(self, node):
1420 pxd_def = self.scope.lookup(node.name)
1422 if not pxd_def.is_cfunction:
1423 error(node.pos, "'%s' redeclared" % node.name)
1425 error(pxd_def.pos, "previous declaration here")
1427 node = node.as_cfunction(pxd_def)
1428 elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1429 node = node.as_cfunction(scope=self.scope)
1430 # Enable this when internal def functions are allowed.
1431 # self.visitchildren(node)
1435 class YieldNodeCollector(TreeVisitor):
1438 super(YieldNodeCollector, self).__init__()
1441 self.has_return_value = False
1443 def visit_Node(self, node):
1444 return self.visitchildren(node)
1446 def visit_YieldExprNode(self, node):
1447 if self.has_return_value:
1448 error(node.pos, "'yield' outside function")
1449 self.yields.append(node)
1450 self.visitchildren(node)
1452 def visit_ReturnStatNode(self, node):
1454 self.has_return_value = True
1456 error(node.pos, "'return' with argument inside generator")
1457 self.returns.append(node)
1459 def visit_ClassDefNode(self, node):
1462 def visit_DefNode(self, node):
1465 def visit_LambdaNode(self, node):
1468 def visit_GeneratorExpressionNode(self, node):
1471 class MarkClosureVisitor(CythonTransform):
1473 def visit_ModuleNode(self, node):
1474 self.needs_closure = False
1475 self.visitchildren(node)
1478 def visit_FuncDefNode(self, node):
1479 self.needs_closure = False
1480 self.visitchildren(node)
1481 node.needs_closure = self.needs_closure
1482 self.needs_closure = True
1484 collector = YieldNodeCollector()
1485 collector.visitchildren(node)
1487 if collector.yields:
1488 for i, yield_expr in enumerate(collector.yields):
1489 yield_expr.label_num = i + 1
1491 gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1494 generator = Nodes.GeneratorDefNode(pos=node.pos,
1497 star_arg=node.star_arg,
1498 starstar_arg=node.starstar_arg,
1500 decorators=node.decorators,
1502 lambda_name=node.lambda_name)
1506 def visit_CFuncDefNode(self, node):
1507 self.visit_FuncDefNode(node)
1508 if node.needs_closure:
1509 error(node.pos, "closures inside cdef functions not yet supported")
1512 def visit_LambdaNode(self, node):
1513 self.needs_closure = False
1514 self.visitchildren(node)
1515 node.needs_closure = self.needs_closure
1516 self.needs_closure = True
1519 def visit_ClassDefNode(self, node):
1520 self.visitchildren(node)
1521 self.needs_closure = True
1524 class CreateClosureClasses(CythonTransform):
1525 # Output closure classes in module scope for all functions
1526 # that really need it.
1528 def __init__(self, context):
1529 super(CreateClosureClasses, self).__init__(context)
1531 self.in_lambda = False
1532 self.generator_class = None
1534 def visit_ModuleNode(self, node):
1535 self.module_scope = node.scope
1536 self.visitchildren(node)
1539 def create_generator_class(self, target_module_scope, pos):
1540 if self.generator_class:
1541 return self.generator_class
1542 # XXX: make generator class creation cleaner
1543 entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1544 objstruct_cname='__pyx_Generator_object',
1545 typeobj_cname='__pyx_Generator_type',
1546 pos=pos, defining=True, implementing=True)
1547 klass = entry.type.scope
1548 klass.is_internal = True
1549 klass.directives = {'final': True}
1551 body_type = PyrexTypes.create_typedef_type('generator_body',
1552 PyrexTypes.c_void_ptr_type,
1553 '__pyx_generator_body_t')
1554 klass.declare_var(pos=pos, name='body', cname='body',
1555 type=body_type, is_cdef=True)
1556 klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1558 klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1562 e = klass.declare_pyfunction('send', pos)
1563 e.func_cname = '__Pyx_Generator_Send'
1564 e.signature = TypeSlots.binaryfunc
1566 e = klass.declare_pyfunction('close', pos)
1567 e.func_cname = '__Pyx_Generator_Close'
1568 e.signature = TypeSlots.unaryfunc
1570 e = klass.declare_pyfunction('throw', pos)
1571 e.func_cname = '__Pyx_Generator_Throw'
1572 e.signature = TypeSlots.pyfunction_signature
1574 e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1575 e.func_cname = 'PyObject_SelfIter'
1577 e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1578 e.func_cname = '__Pyx_Generator_Next'
1580 self.generator_class = entry.type
1581 return self.generator_class
1583 def find_entries_used_in_closures(self, node):
1586 for name, entry in node.local_scope.entries.items():
1587 if entry.from_closure:
1588 from_closure.append((name, entry))
1589 elif entry.in_closure:
1590 in_closure.append((name, entry))
1591 return from_closure, in_closure
1593 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1594 # skip generator body
1595 if node.is_generator_body:
1597 # move local variables into closure
1598 if node.is_generator:
1599 for entry in node.local_scope.entries.values():
1600 if not entry.from_closure:
1601 entry.in_closure = True
1603 from_closure, in_closure = self.find_entries_used_in_closures(node)
1606 # Now from the begining
1607 node.needs_closure = False
1608 node.needs_outer_scope = False
1610 func_scope = node.local_scope
1611 cscope = node.entry.scope
1612 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1613 cscope = cscope.outer_scope
1615 if not from_closure and (self.path or inner_node):
1618 raise InternalError, "DefNode does not have assignment node"
1619 inner_node = node.assmt.rhs
1620 inner_node.needs_self_code = False
1621 node.needs_outer_scope = False
1624 if node.is_generator:
1625 base_type = self.create_generator_class(target_module_scope, node.pos)
1626 elif not in_closure and not from_closure:
1628 elif not in_closure:
1629 func_scope.is_passthrough = True
1630 func_scope.scope_class = cscope.scope_class
1631 node.needs_outer_scope = True
1634 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1636 entry = target_module_scope.declare_c_class(
1637 name=as_name, pos=node.pos, defining=True,
1638 implementing=True, base_type=base_type)
1640 func_scope.scope_class = entry
1641 class_scope = entry.type.scope
1642 class_scope.is_internal = True
1643 class_scope.directives = {'final': True}
1646 assert cscope.is_closure_scope
1647 class_scope.declare_var(pos=node.pos,
1648 name=Naming.outer_scope_cname,
1649 cname=Naming.outer_scope_cname,
1650 type=cscope.scope_class.type,
1652 node.needs_outer_scope = True
1653 for name, entry in in_closure:
1654 closure_entry = class_scope.declare_var(pos=entry.pos,
1659 if entry.is_declared_generic:
1660 closure_entry.is_declared_generic = 1
1661 node.needs_closure = True
1662 # Do it here because other classes are already checked
1663 target_module_scope.check_c_class(func_scope.scope_class)
1665 def visit_LambdaNode(self, node):
1666 was_in_lambda = self.in_lambda
1667 self.in_lambda = True
1668 self.create_class_from_scope(node.def_node, self.module_scope, node)
1669 self.visitchildren(node)
1670 self.in_lambda = was_in_lambda
1673 def visit_FuncDefNode(self, node):
1675 self.visitchildren(node)
1677 if node.needs_closure or self.path:
1678 self.create_class_from_scope(node, self.module_scope)
1679 self.path.append(node)
1680 self.visitchildren(node)
1685 class GilCheck(VisitorTransform):
1687 Call `node.gil_check(env)` on each node to make sure we hold the
1688 GIL when we need it. Raise an error when on Python operations
1689 inside a `nogil` environment.
1691 def __call__(self, root):
1692 self.env_stack = [root.scope]
1694 return super(GilCheck, self).__call__(root)
1696 def visit_FuncDefNode(self, node):
1697 self.env_stack.append(node.local_scope)
1698 was_nogil = self.nogil
1699 self.nogil = node.local_scope.nogil
1700 if self.nogil and node.nogil_check:
1701 node.nogil_check(node.local_scope)
1702 self.visitchildren(node)
1703 self.env_stack.pop()
1704 self.nogil = was_nogil
1707 def visit_GILStatNode(self, node):
1708 env = self.env_stack[-1]
1709 if self.nogil and node.nogil_check: node.nogil_check()
1710 was_nogil = self.nogil
1711 self.nogil = (node.state == 'nogil')
1712 self.visitchildren(node)
1713 self.nogil = was_nogil
1716 def visit_Node(self, node):
1717 if self.env_stack and self.nogil and node.nogil_check:
1718 node.nogil_check(self.env_stack[-1])
1719 self.visitchildren(node)
1723 class TransformBuiltinMethods(EnvTransform):
1725 def visit_SingleAssignmentNode(self, node):
1726 if node.declaration_only:
1729 self.visitchildren(node)
1732 def visit_AttributeNode(self, node):
1733 self.visitchildren(node)
1734 return self.visit_cython_attribute(node)
1736 def visit_NameNode(self, node):
1737 return self.visit_cython_attribute(node)
1739 def visit_cython_attribute(self, node):
1740 attribute = node.as_cython_attribute()
1742 if attribute == u'compiled':
1743 node = ExprNodes.BoolNode(node.pos, value=True)
1744 elif attribute == u'NULL':
1745 node = ExprNodes.NullNode(node.pos)
1746 elif attribute in (u'set', u'frozenset'):
1747 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1748 entry=self.current_env().builtin_scope().lookup_here(attribute))
1749 elif not PyrexTypes.parse_basic_type(attribute):
1750 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1753 def visit_SimpleCallNode(self, node):
1756 if isinstance(node.function, ExprNodes.NameNode):
1757 if node.function.name == 'locals':
1758 lenv = self.current_env()
1759 entry = lenv.lookup_here('locals')
1761 # not the builtin 'locals'
1763 if len(node.args) > 0:
1764 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1767 items = [ ExprNodes.DictItemNode(pos,
1768 key=ExprNodes.StringNode(pos, value=var),
1769 value=ExprNodes.NameNode(pos, name=var))
1770 for var in lenv.entries ]
1771 return ExprNodes.DictNode(pos, key_value_pairs=items)
1774 function = node.function.as_cython_attribute()
1776 if function in InterpretCompilerDirectives.unop_method_nodes:
1777 if len(node.args) != 1:
1778 error(node.function.pos, u"%s() takes exactly one argument" % function)
1780 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1781 elif function in InterpretCompilerDirectives.binop_method_nodes:
1782 if len(node.args) != 2:
1783 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1785 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1786 elif function == u'cast':
1787 if len(node.args) != 2:
1788 error(node.function.pos, u"cast() takes exactly two arguments")
1790 type = node.args[0].analyse_as_type(self.current_env())
1792 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1794 error(node.args[0].pos, "Not a type")
1795 elif function == u'sizeof':
1796 if len(node.args) != 1:
1797 error(node.function.pos, u"sizeof() takes exactly one argument")
1799 type = node.args[0].analyse_as_type(self.current_env())
1801 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1803 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1804 elif function == 'cmod':
1805 if len(node.args) != 2:
1806 error(node.function.pos, u"cmod() takes exactly two arguments")
1808 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1809 node.cdivision = True
1810 elif function == 'cdiv':
1811 if len(node.args) != 2:
1812 error(node.function.pos, u"cdiv() takes exactly two arguments")
1814 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1815 node.cdivision = True
1816 elif function == u'set':
1817 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1819 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1821 self.visitchildren(node)
1825 class DebugTransform(CythonTransform):
1827 Create debug information and all functions' visibility to extern in order
1828 to enable debugging.
1831 def __init__(self, context, options, result):
1832 super(DebugTransform, self).__init__(context)
1833 self.visited = cython.set()
1834 # our treebuilder and debug output writer
1835 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1836 self.tb = self.context.gdb_debug_outputwriter
1837 #self.c_output_file = options.output_file
1838 self.c_output_file = result.c_file
1840 # Closure support, basically treat nested functions as if the AST were
1842 self.nested_funcdefs = []
1844 # tells visit_NameNode whether it should register step-into functions
1845 self.register_stepinto = False
1847 def visit_ModuleNode(self, node):
1848 self.tb.module_name = node.full_module_name
1850 module_name=node.full_module_name,
1851 filename=node.pos[0].filename,
1852 c_filename=self.c_output_file)
1854 self.tb.start('Module', attrs)
1856 # serialize functions
1857 self.tb.start('Functions')
1858 # First, serialize functions normally...
1859 self.visitchildren(node)
1861 # ... then, serialize nested functions
1862 for nested_funcdef in self.nested_funcdefs:
1863 self.visit_FuncDefNode(nested_funcdef)
1865 self.register_stepinto = True
1866 self.serialize_modulenode_as_function(node)
1867 self.register_stepinto = False
1868 self.tb.end('Functions')
1870 # 2.3 compatibility. Serialize global variables
1871 self.tb.start('Globals')
1874 for k, v in node.scope.entries.iteritems():
1875 if (v.qualified_name not in self.visited and not
1876 v.name.startswith('__pyx_') and not
1877 v.type.is_cfunction and not
1878 v.type.is_extension_type):
1881 self.serialize_local_variables(entries)
1882 self.tb.end('Globals')
1883 # self.tb.end('Module') # end Module after the line number mapping in
1884 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1887 def visit_FuncDefNode(self, node):
1888 self.visited.add(node.local_scope.qualified_name)
1890 if getattr(node, 'is_wrapper', False):
1893 if self.register_stepinto:
1894 self.nested_funcdefs.append(node)
1897 # node.entry.visibility = 'extern'
1898 if node.py_func is None:
1901 pf_cname = node.py_func.entry.func_cname
1904 name=node.entry.name,
1905 cname=node.entry.func_cname,
1907 qualified_name=node.local_scope.qualified_name,
1908 lineno=str(node.pos[1]))
1910 self.tb.start('Function', attrs=attrs)
1912 self.tb.start('Locals')
1913 self.serialize_local_variables(node.local_scope.entries)
1914 self.tb.end('Locals')
1916 self.tb.start('Arguments')
1917 for arg in node.local_scope.arg_entries:
1918 self.tb.start(arg.name)
1919 self.tb.end(arg.name)
1920 self.tb.end('Arguments')
1922 self.tb.start('StepIntoFunctions')
1923 self.register_stepinto = True
1924 self.visitchildren(node)
1925 self.register_stepinto = False
1926 self.tb.end('StepIntoFunctions')
1927 self.tb.end('Function')
1931 def visit_NameNode(self, node):
1932 if (self.register_stepinto and
1933 node.type.is_cfunction and
1934 getattr(node, 'is_called', False) and
1935 node.entry.func_cname is not None):
1936 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1937 # declared functions are not 'in_cinclude'.
1938 # This means we will list called 'cdef' functions as
1939 # "step into functions", but this is not an issue as they will be
1940 # recognized as Cython functions anyway.
1941 attrs = dict(name=node.entry.func_cname)
1942 self.tb.start('StepIntoFunction', attrs=attrs)
1943 self.tb.end('StepIntoFunction')
1945 self.visitchildren(node)
1948 def serialize_modulenode_as_function(self, node):
1950 Serialize the module-level code as a function so the debugger will know
1951 it's a "relevant frame" and it will know where to set the breakpoint
1952 for 'break modulename'.
1954 name = node.full_module_name.rpartition('.')[-1]
1956 cname_py2 = 'init' + name
1957 cname_py3 = 'PyInit_' + name
1963 # Ignore the qualified_name, breakpoints should be set using
1964 # `cy break modulename:lineno` for module-level breakpoints.
1967 is_initmodule_function="True",
1970 py3_attrs = dict(py2_attrs, cname=cname_py3)
1972 self._serialize_modulenode_as_function(node, py2_attrs)
1973 self._serialize_modulenode_as_function(node, py3_attrs)
1975 def _serialize_modulenode_as_function(self, node, attrs):
1976 self.tb.start('Function', attrs=attrs)
1978 self.tb.start('Locals')
1979 self.serialize_local_variables(node.scope.entries)
1980 self.tb.end('Locals')
1982 self.tb.start('Arguments')
1983 self.tb.end('Arguments')
1985 self.tb.start('StepIntoFunctions')
1986 self.register_stepinto = True
1987 self.visitchildren(node)
1988 self.register_stepinto = False
1989 self.tb.end('StepIntoFunctions')
1991 self.tb.end('Function')
1993 def serialize_local_variables(self, entries):
1994 for entry in entries.values():
1995 if entry.type.is_pyobject:
1996 vartype = 'PythonObject'
2000 if entry.from_closure:
2001 # We're dealing with a closure where a variable from an outer
2002 # scope is accessed, get it from the scope object.
2003 cname = '%s->%s' % (Naming.cur_scope_cname,
2004 entry.outer_entry.cname)
2006 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2009 elif entry.in_closure:
2010 cname = '%s->%s' % (Naming.cur_scope_cname,
2012 qname = entry.qualified_name
2015 qname = entry.qualified_name
2018 # this happens for variables that are not in the user's code,
2019 # e.g. for the global __builtins__, __doc__, etc. We can just
2020 # set the lineno to 0 for those.
2023 lineno = str(entry.pos[1])
2028 qualified_name=qname,
2032 self.tb.start('LocalVar', attrs)
2033 self.tb.end('LocalVar')