3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
16 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
17 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
18 from Cython.Compiler.ModuleNode import ModuleNode
19 from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
20 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
21 from Cython.Compiler.StringEncoding import EncodedString
22 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
27 class NameNodeCollector(TreeVisitor):
28 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
32 super(NameNodeCollector, self).__init__()
35 def visit_NameNode(self, node):
36 self.name_nodes.append(node)
38 def visit_Node(self, node):
39 self._visitchildren(node, None)
42 class SkipDeclarations(object):
44 Variable and function declarations can often have a deep tree structure,
45 and yet most transformations don't need to descend to this depth.
47 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
48 is no need to use this for transformations after that point.
50 def visit_CTypeDefNode(self, node):
53 def visit_CVarDefNode(self, node):
56 def visit_CDeclaratorNode(self, node):
59 def visit_CBaseTypeNode(self, node):
62 def visit_CEnumDefNode(self, node):
65 def visit_CStructOrUnionDefNode(self, node):
68 class NormalizeTree(CythonTransform):
70 This transform fixes up a few things after parsing
71 in order to make the parse tree more suitable for
74 a) After parsing, blocks with only one statement will
75 be represented by that statement, not by a StatListNode.
76 When doing transforms this is annoying and inconsistent,
77 as one cannot in general remove a statement in a consistent
78 way and so on. This transform wraps any single statements
79 in a StatListNode containing a single statement.
81 b) The PassStatNode is a noop and serves no purpose beyond
82 plugging such one-statement blocks; i.e., once parsed a
83 ` "pass" can just as well be represented using an empty
84 StatListNode. This means less special cases to worry about
85 in subsequent transforms (one always checks to see if a
86 StatListNode has no children to see if the block is empty).
89 def __init__(self, context):
90 super(NormalizeTree, self).__init__(context)
91 self.is_in_statlist = False
92 self.is_in_expr = False
94 def visit_ExprNode(self, node):
95 stacktmp = self.is_in_expr
96 self.is_in_expr = True
97 self.visitchildren(node)
98 self.is_in_expr = stacktmp
101 def visit_StatNode(self, node, is_listcontainer=False):
102 stacktmp = self.is_in_statlist
103 self.is_in_statlist = is_listcontainer
104 self.visitchildren(node)
105 self.is_in_statlist = stacktmp
106 if not self.is_in_statlist and not self.is_in_expr:
107 return Nodes.StatListNode(pos=node.pos, stats=[node])
111 def visit_StatListNode(self, node):
112 self.is_in_statlist = True
113 self.visitchildren(node)
114 self.is_in_statlist = False
117 def visit_ParallelAssignmentNode(self, node):
118 return self.visit_StatNode(node, True)
120 def visit_CEnumDefNode(self, node):
121 return self.visit_StatNode(node, True)
123 def visit_CStructOrUnionDefNode(self, node):
124 return self.visit_StatNode(node, True)
126 # Eliminate PassStatNode
127 def visit_PassStatNode(self, node):
128 if not self.is_in_statlist:
129 return Nodes.StatListNode(pos=node.pos, stats=[])
133 def visit_CDeclaratorNode(self, node):
137 class PostParseError(CompileError): pass
139 # error strings checked by unit tests, so define them
140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
143 class PostParse(ScopeTrackingTransform):
145 Basic interpretation of the parse tree, as well as validity
146 checking that can be done on a very basic level on the parse
147 tree (while still not being a problem with the basic syntax,
151 - Default values to cdef assignments are turned into single
152 assignments following the declaration (everywhere but in class
153 bodies, where they raise a compile error)
155 - Interpret some node structures into Python runtime values.
156 Some nodes take compile-time arguments (currently:
157 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
158 which should be interpreted. This happens in a general way
159 and other steps should be taken to ensure validity.
161 Type arguments cannot be interpreted in this way.
163 - For __cythonbufferdefaults__ the arguments are checked for
166 TemplatedTypeNode has its directives interpreted:
167 Any first positional argument goes into the "dtype" attribute,
168 any "ndim" keyword argument goes into the "ndim" attribute and
169 so on. Also it is checked that the directive combination is valid.
170 - __cythonbufferdefaults__ attributes are parsed and put into the
173 Note: Currently Parsing.py does a lot of interpretation and
174 reorganization that can be refactored into this transform
175 if a more pure Abstract Syntax Tree is wanted.
178 def __init__(self, context):
179 super(PostParse, self).__init__(context)
180 self.specialattribute_handlers = {
181 '__cythonbufferdefaults__' : self.handle_bufferdefaults
184 def visit_ModuleNode(self, node):
185 self.lambda_counter = 1
186 self.genexpr_counter = 1
187 return super(PostParse, self).visit_ModuleNode(node)
189 def visit_LambdaNode(self, node):
190 # unpack a lambda expression into the corresponding DefNode
191 lambda_id = self.lambda_counter
192 self.lambda_counter += 1
193 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
194 collector = YieldNodeCollector()
195 collector.visitchildren(node.result_expr)
196 if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
197 body = Nodes.ExprStatNode(
198 node.result_expr.pos, expr=node.result_expr)
200 body = Nodes.ReturnStatNode(
201 node.result_expr.pos, value=node.result_expr)
202 node.def_node = Nodes.DefNode(
203 node.pos, name=node.name, lambda_name=node.lambda_name,
204 args=node.args, star_arg=node.star_arg,
205 starstar_arg=node.starstar_arg,
207 self.visitchildren(node)
210 def visit_GeneratorExpressionNode(self, node):
211 # unpack a generator expression into the corresponding DefNode
212 genexpr_id = self.genexpr_counter
213 self.genexpr_counter += 1
214 node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
216 node.def_node = Nodes.DefNode(node.pos, name=node.name,
218 args=[], star_arg=None,
221 self.visitchildren(node)
225 def handle_bufferdefaults(self, decl):
226 if not isinstance(decl.default, ExprNodes.DictNode):
227 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
228 self.scope_node.buffer_defaults_node = decl.default
229 self.scope_node.buffer_defaults_pos = decl.pos
231 def visit_CVarDefNode(self, node):
232 # This assumes only plain names and pointers are assignable on
233 # declaration. Also, it makes use of the fact that a cdef decl
234 # must appear before the first use, so we don't have to deal with
235 # "i = 3; cdef int i = i" and can simply move the nodes around.
237 self.visitchildren(node)
240 for decl in node.declarators:
242 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
243 declbase = declbase.base
244 if isinstance(declbase, Nodes.CNameDeclaratorNode):
245 if declbase.default is not None:
246 if self.scope_type in ('cclass', 'pyclass', 'struct'):
247 if isinstance(self.scope_node, Nodes.CClassDefNode):
248 handler = self.specialattribute_handlers.get(decl.name)
250 if decl is not declbase:
251 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
253 continue # Remove declaration
254 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
255 first_assignment = self.scope_type != 'module'
256 stats.append(Nodes.SingleAssignmentNode(node.pos,
257 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
258 rhs=declbase.default, first=first_assignment))
259 declbase.default = None
260 newdecls.append(decl)
261 node.declarators = newdecls
263 except PostParseError, e:
264 # An error in a cdef clause is ok, simply remove the declaration
265 # and try to move on to report more errors
266 self.context.nonfatal_error(e)
269 # Split parallel assignments (a,b = b,a) into separate partial
270 # assignments that are executed rhs-first using temps. This
271 # restructuring must be applied before type analysis so that known
272 # types on rhs and lhs can be matched directly. It is required in
273 # the case that the types cannot be coerced to a Python type in
274 # order to assign from a tuple.
276 def visit_SingleAssignmentNode(self, node):
277 self.visitchildren(node)
278 return self._visit_assignment_node(node, [node.lhs, node.rhs])
280 def visit_CascadedAssignmentNode(self, node):
281 self.visitchildren(node)
282 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
284 def _visit_assignment_node(self, node, expr_list):
285 """Flatten parallel assignments into separate single
286 assignments or cascaded assignments.
288 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
289 # no parallel assignments => nothing to do
293 flatten_parallel_assignments(expr_list, expr_list_list)
295 eliminate_rhs_duplicates(expr_list_list, temp_refs)
298 for expr_list in expr_list_list:
299 lhs_list = expr_list[:-1]
301 if len(lhs_list) == 1:
302 node = Nodes.SingleAssignmentNode(rhs.pos,
303 lhs = lhs_list[0], rhs = rhs)
305 node = Nodes.CascadedAssignmentNode(rhs.pos,
306 lhs_list = lhs_list, rhs = rhs)
310 assign_node = nodes[0]
312 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
315 duplicates_and_temps = [ (temp.expression, temp)
316 for temp in temp_refs ]
317 sort_common_subsequences(duplicates_and_temps)
318 for _, temp_ref in duplicates_and_temps[::-1]:
319 assign_node = LetNode(temp_ref, assign_node)
323 def _flatten_sequence(self, seq, result):
325 if arg.is_sequence_constructor:
326 self._flatten_sequence(arg, result)
331 def visit_DelStatNode(self, node):
332 self.visitchildren(node)
333 node.args = self._flatten_sequence(node, [])
337 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
338 """Replace rhs items by LetRefNodes if they appear more than once.
339 Creates a sequence of LetRefNodes that set up the required temps
340 and appends them to ref_node_sequence. The input list is modified
343 seen_nodes = cython.set()
345 def find_duplicates(node):
346 if node.is_literal or node.is_name:
347 # no need to replace those; can't include attributes here
348 # as their access is not necessarily side-effect free
350 if node in seen_nodes:
351 if node not in ref_nodes:
352 ref_node = LetRefNode(node)
353 ref_nodes[node] = ref_node
354 ref_node_sequence.append(ref_node)
357 if node.is_sequence_constructor:
358 for item in node.args:
359 find_duplicates(item)
361 for expr_list in expr_list_list:
367 def substitute_nodes(node):
368 if node in ref_nodes:
369 return ref_nodes[node]
370 elif node.is_sequence_constructor:
371 node.args = list(map(substitute_nodes, node.args))
374 # replace nodes inside of the common subexpressions
375 for node in ref_nodes:
376 if node.is_sequence_constructor:
377 node.args = list(map(substitute_nodes, node.args))
379 # replace common subexpressions on all rhs items
380 for expr_list in expr_list_list:
381 expr_list[-1] = substitute_nodes(expr_list[-1])
383 def sort_common_subsequences(items):
384 """Sort items/subsequences so that all items and subsequences that
385 an item contains appear before the item itself. This is needed
386 because each rhs item must only be evaluated once, so its value
387 must be evaluated first and then reused when packing sequences
390 This implies a partial order, and the sort must be stable to
391 preserve the original order as much as possible, so we use a
392 simple insertion sort (which is very fast for short sequences, the
393 normal case in practice).
395 def contains(seq, x):
399 elif item.is_sequence_constructor and contains(item.args, x):
403 return b.is_sequence_constructor and contains(b.args, a)
405 for pos, item in enumerate(items):
406 key = item[1] # the ResultRefNode which has already been injected into the sequences
408 for i in xrange(pos-1, -1, -1):
409 if lower_than(key, items[i][0]):
412 for i in xrange(pos, new_pos, -1):
413 items[i] = items[i-1]
414 items[new_pos] = item
416 def flatten_parallel_assignments(input, output):
417 # The input is a list of expression nodes, representing the LHSs
418 # and RHS of one (possibly cascaded) assignment statement. For
419 # sequence constructors, rearranges the matching parts of both
420 # sides into a list of equivalent assignments between the
421 # individual elements. This transformation is applied
422 # recursively, so that nested structures get matched as well.
424 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
428 complete_assignments = []
430 rhs_size = len(rhs.args)
431 lhs_targets = [ [] for _ in xrange(rhs_size) ]
432 starred_assignments = []
433 for lhs in input[:-1]:
434 if not lhs.is_sequence_constructor:
436 error(lhs.pos, "starred assignment target must be in a list or tuple")
437 complete_assignments.append(lhs)
439 lhs_size = len(lhs.args)
440 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
441 if starred_targets > 1:
442 error(lhs.pos, "more than 1 starred expression in assignment")
443 output.append([lhs,rhs])
445 elif lhs_size - starred_targets > rhs_size:
446 error(lhs.pos, "need more than %d value%s to unpack"
447 % (rhs_size, (rhs_size != 1) and 's' or ''))
448 output.append([lhs,rhs])
450 elif starred_targets:
451 map_starred_assignment(lhs_targets, starred_assignments,
453 elif lhs_size < rhs_size:
454 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
455 % (lhs_size, rhs_size))
456 output.append([lhs,rhs])
459 for targets, expr in zip(lhs_targets, lhs.args):
462 if complete_assignments:
463 complete_assignments.append(rhs)
464 output.append(complete_assignments)
466 # recursively flatten partial assignments
467 for cascade, rhs in zip(lhs_targets, rhs.args):
470 flatten_parallel_assignments(cascade, output)
472 # recursively flatten starred assignments
473 for cascade in starred_assignments:
474 if cascade[0].is_sequence_constructor:
475 flatten_parallel_assignments(cascade, output)
477 output.append(cascade)
479 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
480 # Appends the fixed-position LHS targets to the target list that
481 # appear left and right of the starred argument.
483 # The starred_assignments list receives a new tuple
484 # (lhs_target, rhs_values_list) that maps the remaining arguments
485 # (those that match the starred target) to a list.
487 # left side of the starred target
488 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
491 lhs_remaining = len(lhs_args) - i - 1
495 raise InternalError("no starred arg found when splitting starred assignment")
497 # right side of the starred target
498 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
499 lhs_args[starred + 1:])):
502 # the starred target itself, must be assigned a (potentially empty) list
503 target = lhs_args[starred].target # unpack starred node
504 starred_rhs = rhs_args[starred:]
506 starred_rhs = starred_rhs[:-lhs_remaining]
508 pos = starred_rhs[0].pos
511 starred_assignments.append([
512 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
515 class PxdPostParse(CythonTransform, SkipDeclarations):
517 Basic interpretation/validity checking that should only be
520 A lot of this checking currently happens in the parser; but
521 what is listed below happens here.
523 - "def" functions are let through only if they fill the
524 getbuffer/releasebuffer slots
526 - cdef functions are let through only if they are on the
527 top level and are declared "inline"
529 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
530 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
532 def __call__(self, node):
533 self.scope_type = 'pxd'
534 return super(PxdPostParse, self).__call__(node)
536 def visit_CClassDefNode(self, node):
537 old = self.scope_type
538 self.scope_type = 'cclass'
539 self.visitchildren(node)
540 self.scope_type = old
543 def visit_FuncDefNode(self, node):
544 # FuncDefNode always come with an implementation (without
545 # an imp they are CVarDefNodes..)
546 err = self.ERR_INLINE_ONLY
548 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
549 and node.name in ('__getbuffer__', '__releasebuffer__')):
550 err = None # allow these slots
552 if isinstance(node, Nodes.CFuncDefNode):
553 if u'inline' in node.modifiers and self.scope_type == 'pxd':
554 node.inline_in_pxd = True
555 if node.visibility != 'private':
556 err = self.ERR_NOGO_WITH_INLINE % node.visibility
558 err = self.ERR_NOGO_WITH_INLINE % 'api'
560 err = None # allow inline function
562 err = self.ERR_INLINE_ONLY
565 self.context.nonfatal_error(PostParseError(node.pos, err))
570 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
572 After parsing, directives can be stored in a number of places:
573 - #cython-comments at the top of the file (stored in ModuleNode)
574 - Command-line arguments overriding these
575 - @cython.directivename decorators
576 - with cython.directivename: statements
578 This transform is responsible for interpreting these various sources
579 and store the directive in two ways:
580 - Set the directives attribute of the ModuleNode for global directives.
581 - Use a CompilerDirectivesNode to override directives for a subtree.
583 (The first one is primarily to not have to modify with the tree
584 structure, so that ModuleNode stay on top.)
586 The directives are stored in dictionaries from name to value in effect.
587 Each such dictionary is always filled in for all possible directives,
588 using default values where no value is given by the user.
590 The available directives are controlled in Options.py.
592 Note that we have to run this prior to analysis, and so some minor
593 duplication of functionality has to occur: We manually track cimports
594 and which names the "cython" module may have been imported to.
596 unop_method_nodes = {
597 'typeof': ExprNodes.TypeofNode,
599 'operator.address': ExprNodes.AmpersandNode,
600 'operator.dereference': ExprNodes.DereferenceNode,
601 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
602 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
603 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
604 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
606 # For backwards compatability.
607 'address': ExprNodes.AmpersandNode,
610 binop_method_nodes = {
611 'operator.comma' : ExprNodes.c_binop_constructor(','),
614 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
615 'cast', 'pointer', 'compiled', 'NULL'])
616 special_methods.update(unop_method_nodes.keys())
618 def __init__(self, context, compilation_directive_defaults):
619 super(InterpretCompilerDirectives, self).__init__(context)
620 self.compilation_directive_defaults = {}
621 for key, value in compilation_directive_defaults.items():
622 self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
623 self.cython_module_names = cython.set()
624 self.directive_names = {}
626 def check_directive_scope(self, pos, directive, scope):
627 legal_scopes = Options.directive_scopes.get(directive, None)
628 if legal_scopes and scope not in legal_scopes:
629 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
630 'is not allowed in %s scope' % (directive, scope)))
635 # Set up processing and handle the cython: comments.
636 def visit_ModuleNode(self, node):
637 for key, value in node.directive_comments.items():
638 if not self.check_directive_scope(node.pos, key, 'module'):
639 self.wrong_scope_error(node.pos, key, 'module')
640 del node.directive_comments[key]
642 directives = copy.deepcopy(Options.directive_defaults)
643 directives.update(copy.deepcopy(self.compilation_directive_defaults))
644 directives.update(node.directive_comments)
645 self.directives = directives
646 node.directives = directives
647 self.visitchildren(node)
648 node.cython_module_names = self.cython_module_names
651 # The following four functions track imports and cimports that
652 # begin with "cython"
653 def is_cython_directive(self, name):
654 return (name in Options.directive_types or
655 name in self.special_methods or
656 PyrexTypes.parse_basic_type(name))
658 def visit_CImportStatNode(self, node):
659 if node.module_name == u"cython":
660 self.cython_module_names.add(node.as_name or u"cython")
661 elif node.module_name.startswith(u"cython."):
663 self.directive_names[node.as_name] = node.module_name[7:]
665 self.cython_module_names.add(u"cython")
666 # if this cimport was a compiler directive, we don't
667 # want to leave the cimport node sitting in the tree
671 def visit_FromCImportStatNode(self, node):
672 if (node.module_name == u"cython") or \
673 node.module_name.startswith(u"cython."):
674 submodule = (node.module_name + u".")[7:]
676 for pos, name, as_name, kind in node.imported_names:
677 full_name = submodule + name
678 if self.is_cython_directive(full_name):
681 self.directive_names[as_name] = full_name
683 self.context.nonfatal_error(PostParseError(pos,
684 "Compiler directive imports must be plain imports"))
686 newimp.append((pos, name, as_name, kind))
689 node.imported_names = newimp
692 def visit_FromImportStatNode(self, node):
693 if (node.module.module_name.value == u"cython") or \
694 node.module.module_name.value.startswith(u"cython."):
695 submodule = (node.module.module_name.value + u".")[7:]
697 for name, name_node in node.items:
698 full_name = submodule + name
699 if self.is_cython_directive(full_name):
700 self.directive_names[name_node.name] = full_name
702 newimp.append((name, name_node))
708 def visit_SingleAssignmentNode(self, node):
709 if (isinstance(node.rhs, ExprNodes.ImportNode) and
710 node.rhs.module_name.value == u'cython'):
711 node = Nodes.CImportStatNode(node.pos,
712 module_name = u'cython',
713 as_name = node.lhs.name)
714 self.visit_CImportStatNode(node)
716 self.visitchildren(node)
719 def visit_NameNode(self, node):
720 if node.name in self.cython_module_names:
721 node.is_cython_module = True
723 node.cython_attribute = self.directive_names.get(node.name)
726 def try_to_parse_directives(self, node):
727 # If node is the contents of an directive (in a with statement or
728 # decorator), returns a list of (directivename, value) pairs.
729 # Otherwise, returns None
730 if isinstance(node, ExprNodes.CallNode):
731 self.visit(node.function)
732 optname = node.function.as_cython_attribute()
734 directivetype = Options.directive_types.get(optname)
736 args, kwds = node.explicit_args_kwds()
739 if kwds is not None and directivetype is not dict:
740 for keyvalue in kwds.key_value_pairs:
741 key, value = keyvalue
742 sub_optname = "%s.%s" % (optname, key.value)
743 if Options.directive_types.get(sub_optname):
744 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
746 key_value_pairs.append(keyvalue)
747 if not key_value_pairs:
750 kwds.key_value_pairs = key_value_pairs
751 if directives and not kwds and not args:
753 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
755 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
757 optname = node.as_cython_attribute()
759 directivetype = Options.directive_types.get(optname)
760 if directivetype is bool:
761 return [(optname, True)]
762 elif directivetype is None:
763 return [(optname, None)]
765 raise PostParseError(
766 node.pos, "The '%s' directive should be used as a function call." % optname)
769 def try_to_parse_directive(self, optname, args, kwds, pos):
770 directivetype = Options.directive_types.get(optname)
771 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
772 return optname, Options.directive_defaults[optname]
773 elif directivetype is bool:
774 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
775 raise PostParseError(pos,
776 'The %s directive takes one compile-time boolean argument' % optname)
777 return (optname, args[0].value)
778 elif directivetype is str:
779 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
780 ExprNodes.UnicodeNode)):
781 raise PostParseError(pos,
782 'The %s directive takes one compile-time string argument' % optname)
783 return (optname, str(args[0].value))
784 elif directivetype is dict:
786 raise PostParseError(pos,
787 'The %s directive takes no prepositional arguments' % optname)
788 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
789 elif directivetype is list:
790 if kwds and len(kwds) != 0:
791 raise PostParseError(pos,
792 'The %s directive takes no keyword arguments' % optname)
793 return optname, [ str(arg.value) for arg in args ]
797 def visit_with_directives(self, body, directives):
798 olddirectives = self.directives
799 newdirectives = copy.copy(olddirectives)
800 newdirectives.update(directives)
801 self.directives = newdirectives
802 assert isinstance(body, Nodes.StatListNode), body
803 retbody = self.visit_Node(body)
804 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
805 directives=newdirectives)
806 self.directives = olddirectives
810 def visit_FuncDefNode(self, node):
811 directives = self._extract_directives(node, 'function')
813 return self.visit_Node(node)
814 body = Nodes.StatListNode(node.pos, stats=[node])
815 return self.visit_with_directives(body, directives)
817 def visit_CVarDefNode(self, node):
818 if not node.decorators:
820 for dec in node.decorators:
821 for directive in self.try_to_parse_directives(dec.decorator) or ():
822 if directive is not None and directive[0] == u'locals':
823 node.directive_locals = directive[1]
825 self.context.nonfatal_error(PostParseError(dec.pos,
826 "Cdef functions can only take cython.locals() decorator."))
829 def visit_CClassDefNode(self, node):
830 directives = self._extract_directives(node, 'cclass')
832 return self.visit_Node(node)
833 body = Nodes.StatListNode(node.pos, stats=[node])
834 return self.visit_with_directives(body, directives)
836 def visit_PyClassDefNode(self, node):
837 directives = self._extract_directives(node, 'class')
839 return self.visit_Node(node)
840 body = Nodes.StatListNode(node.pos, stats=[node])
841 return self.visit_with_directives(body, directives)
843 def _extract_directives(self, node, scope_name):
844 if not node.decorators:
846 # Split the decorators into two lists -- real decorators and directives
849 for dec in node.decorators:
850 new_directives = self.try_to_parse_directives(dec.decorator)
851 if new_directives is not None:
852 for directive in new_directives:
853 if self.check_directive_scope(node.pos, directive[0], scope_name):
854 directives.append(directive)
857 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
858 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
860 node.decorators = realdecs
861 # merge or override repeated directives
863 directives.reverse() # Decorators coming first take precedence
864 for directive in directives:
865 name, value = directive
867 old_value = optdict[name]
868 # keywords and arg lists can be merged, everything
869 # else overrides completely
870 if isinstance(old_value, dict):
871 old_value.update(value)
872 elif isinstance(old_value, list):
873 old_value.extend(value)
875 optdict[name] = value
877 optdict[name] = value
880 # Handle with statements
881 def visit_WithStatNode(self, node):
883 for directive in self.try_to_parse_directives(node.manager) or []:
884 if directive is not None:
885 if node.target is not None:
886 self.context.nonfatal_error(
887 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
889 name, value = directive
891 # special case: in pure mode, "with nogil" spells "with cython.nogil"
892 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
893 return self.visit_Node(node)
894 if self.check_directive_scope(node.pos, name, 'with statement'):
895 directive_dict[name] = value
897 return self.visit_with_directives(node.body, directive_dict)
898 return self.visit_Node(node)
901 class DecoratorTransform(CythonTransform, SkipDeclarations):
903 def visit_DefNode(self, func_node):
904 self.visitchildren(func_node)
905 if not func_node.decorators:
907 return self._handle_decorators(
908 func_node, func_node.name)
910 def visit_CClassDefNode(self, class_node):
911 # This doesn't currently work, so it's disabled.
913 # Problem: assignments to cdef class names do not work. They
914 # would require an additional check anyway, as the extension
915 # type must not change its C type, so decorators cannot
916 # replace an extension type, just alter it and return it.
918 self.visitchildren(class_node)
919 if not class_node.decorators:
921 error(class_node.pos,
922 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
924 #return self._handle_decorators(
925 # class_node, class_node.class_name)
927 def visit_ClassDefNode(self, class_node):
928 self.visitchildren(class_node)
929 if not class_node.decorators:
931 return self._handle_decorators(
932 class_node, class_node.name)
934 def _handle_decorators(self, node, name):
935 decorator_result = ExprNodes.NameNode(node.pos, name = name)
936 for decorator in node.decorators[::-1]:
937 decorator_result = ExprNodes.SimpleCallNode(
939 function = decorator.decorator,
940 args = [decorator_result])
942 name_node = ExprNodes.NameNode(node.pos, name = name)
943 reassignment = Nodes.SingleAssignmentNode(
946 rhs = decorator_result)
947 return [node, reassignment]
950 class AnalyseDeclarationsTransform(CythonTransform):
952 basic_property = TreeFragment(u"""
956 def __set__(self, value):
958 """, level='c_class')
959 basic_pyobject_property = TreeFragment(u"""
963 def __set__(self, value):
967 """, level='c_class')
968 basic_property_ro = TreeFragment(u"""
972 """, level='c_class')
974 struct_or_union_wrapper = TreeFragment(u"""
977 def __init__(self, MEMBER=None):
981 if IS_UNION and count > 1:
982 raise ValueError, "At most one union member should be specified."
984 return STR_FORMAT % MEMBER_TUPLE
986 return REPR_FORMAT % MEMBER_TUPLE
989 init_assignment = TreeFragment(u"""
990 if VALUE is not None:
995 def __call__(self, root):
996 self.env_stack = [root.scope]
997 # needed to determine if a cdef var is declared after it's used.
998 self.seen_vars_stack = []
999 return super(AnalyseDeclarationsTransform, self).__call__(root)
1001 def visit_NameNode(self, node):
1002 self.seen_vars_stack[-1].add(node.name)
1005 def visit_ModuleNode(self, node):
1006 self.seen_vars_stack.append(cython.set())
1007 node.analyse_declarations(self.env_stack[-1])
1008 self.visitchildren(node)
1009 self.seen_vars_stack.pop()
1012 def visit_LambdaNode(self, node):
1013 node.analyse_declarations(self.env_stack[-1])
1014 self.visitchildren(node)
1017 def visit_ClassDefNode(self, node):
1018 self.env_stack.append(node.scope)
1019 self.visitchildren(node)
1020 self.env_stack.pop()
1023 def visit_CClassDefNode(self, node):
1024 node = self.visit_ClassDefNode(node)
1025 if node.scope and node.scope.implemented:
1027 for entry in node.scope.var_entries:
1028 if entry.needs_property:
1029 property = self.create_Property(entry)
1030 property.analyse_declarations(node.scope)
1031 self.visit(property)
1032 stats.append(property)
1034 node.body.stats += stats
1037 def visit_FuncDefNode(self, node):
1038 self.seen_vars_stack.append(cython.set())
1039 lenv = node.local_scope
1040 node.body.analyse_control_flow(lenv) # this will be totally refactored
1041 node.declare_arguments(lenv)
1042 for var, type_node in node.directive_locals.items():
1043 if not lenv.lookup_here(var): # don't redeclare args
1044 type = type_node.analyse_as_type(lenv)
1046 lenv.declare_var(var, type, type_node.pos)
1048 error(type_node.pos, "Not a type")
1049 node.body.analyse_declarations(lenv)
1050 self.env_stack.append(lenv)
1051 self.visitchildren(node)
1052 self.env_stack.pop()
1053 self.seen_vars_stack.pop()
1056 def visit_ScopedExprNode(self, node):
1057 env = self.env_stack[-1]
1058 node.analyse_declarations(env)
1059 # the node may or may not have a local scope
1060 if node.has_local_scope:
1061 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1062 self.env_stack.append(node.expr_scope)
1063 node.analyse_scoped_declarations(node.expr_scope)
1064 self.visitchildren(node)
1065 self.env_stack.pop()
1066 self.seen_vars_stack.pop()
1068 node.analyse_scoped_declarations(env)
1069 self.visitchildren(node)
1072 def visit_TempResultFromStatNode(self, node):
1073 self.visitchildren(node)
1074 node.analyse_declarations(self.env_stack[-1])
1077 def visit_CStructOrUnionDefNode(self, node):
1078 # Create a wrapper node if needed.
1079 # We want to use the struct type information (so it can't happen
1080 # before this phase) but also create new objects to be declared
1081 # (so it can't happen later).
1082 # Note that we don't return the original node, as it is
1083 # never used after this phase.
1084 if True: # private (default)
1087 self_value = ExprNodes.AttributeNode(
1089 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1090 attribute = EncodedString(u"value"))
1091 var_entries = node.entry.type.scope.var_entries
1093 for entry in var_entries:
1094 attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1096 attribute = entry.name))
1097 # __init__ assignments
1098 init_assignments = []
1099 for entry, attr in zip(var_entries, attributes):
1100 # TODO: branch on visibility
1101 init_assignments.append(self.init_assignment.substitute({
1102 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1104 }, pos = entry.pos))
1107 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1108 wrapper_class = self.struct_or_union_wrapper.substitute({
1109 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1110 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1111 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1112 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1113 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1114 }, pos = node.pos).stats[0]
1115 wrapper_class.class_name = node.name
1116 wrapper_class.shadow = True
1117 class_body = wrapper_class.body.stats
1120 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1121 class_body[0].base_type.name = node.name
1123 # fix __init__ arguments
1124 init_method = class_body[1]
1125 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1126 arg_template = init_method.args[1]
1127 if not node.entry.type.is_struct:
1128 arg_template.kw_only = True
1129 del init_method.args[1]
1130 for entry, attr in zip(var_entries, attributes):
1131 arg = copy.deepcopy(arg_template)
1132 arg.declarator.name = entry.name
1133 init_method.args.append(arg)
1136 for entry, attr in zip(var_entries, attributes):
1137 # TODO: branch on visibility
1138 if entry.type.is_pyobject:
1139 template = self.basic_pyobject_property
1141 template = self.basic_property
1142 property = template.substitute({
1144 }, pos = entry.pos).stats[0]
1145 property.name = entry.name
1146 wrapper_class.body.stats.append(property)
1148 wrapper_class.analyse_declarations(self.env_stack[-1])
1149 return self.visit_CClassDefNode(wrapper_class)
1151 # Some nodes are no longer needed after declaration
1152 # analysis and can be dropped. The analysis was performed
1153 # on these nodes in a seperate recursive process from the
1154 # enclosing function or module, so we can simply drop them.
1155 def visit_CDeclaratorNode(self, node):
1156 # necessary to ensure that all CNameDeclaratorNodes are visited.
1157 self.visitchildren(node)
1160 def visit_CTypeDefNode(self, node):
1163 def visit_CBaseTypeNode(self, node):
1166 def visit_CEnumDefNode(self, node):
1167 if node.visibility == 'public':
1172 def visit_CNameDeclaratorNode(self, node):
1173 if node.name in self.seen_vars_stack[-1]:
1174 entry = self.env_stack[-1].lookup(node.name)
1175 if (entry is None or entry.visibility != 'extern'
1176 and not entry.scope.is_c_class_scope):
1177 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1178 self.visitchildren(node)
1181 def visit_CVarDefNode(self, node):
1182 # to ensure all CNameDeclaratorNodes are visited.
1183 self.visitchildren(node)
1186 def create_Property(self, entry):
1187 if entry.visibility == 'public':
1188 if entry.type.is_pyobject:
1189 template = self.basic_pyobject_property
1191 template = self.basic_property
1192 elif entry.visibility == 'readonly':
1193 template = self.basic_property_ro
1194 property = template.substitute({
1195 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1196 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1197 attribute=entry.name),
1198 }, pos=entry.pos).stats[0]
1199 property.name = entry.name
1200 # ---------------------------------------
1201 # XXX This should go to AutoDocTransforms
1202 # ---------------------------------------
1203 if (Options.docstrings and
1204 self.current_directives['embedsignature']):
1205 attr_name = entry.name
1206 type_name = entry.type.declaration_code("", for_display=1)
1208 if not entry.type.is_pyobject:
1209 type_name = "'%s'" % type_name
1210 elif entry.type.is_extension_type:
1211 type_name = entry.type.module_name + '.' + type_name
1212 if entry.init is not None:
1213 default_value = ' = ' + entry.init
1214 elif entry.init_to_none:
1215 default_value = ' = ' + repr(None)
1216 docstring = attr_name + ': ' + type_name + default_value
1217 property.doc = EncodedString(docstring)
1218 # ---------------------------------------
1221 class AnalyseExpressionsTransform(CythonTransform):
1223 def visit_ModuleNode(self, node):
1224 node.scope.infer_types()
1225 node.body.analyse_expressions(node.scope)
1226 self.visitchildren(node)
1229 def visit_FuncDefNode(self, node):
1230 node.local_scope.infer_types()
1231 node.body.analyse_expressions(node.local_scope)
1232 self.visitchildren(node)
1235 def visit_ScopedExprNode(self, node):
1236 if node.has_local_scope:
1237 node.expr_scope.infer_types()
1238 node.analyse_scoped_expressions(node.expr_scope)
1239 self.visitchildren(node)
1242 class ExpandInplaceOperators(EnvTransform):
1244 def visit_InPlaceAssignmentNode(self, node):
1247 if lhs.type.is_cpp_class:
1248 # No getting around this exact operator here.
1250 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1251 # There is code to handle this case.
1254 env = self.current_env()
1255 def side_effect_free_reference(node, setting=False):
1256 if isinstance(node, ExprNodes.NameNode):
1258 elif node.type.is_pyobject and not setting:
1259 node = LetRefNode(node)
1261 elif isinstance(node, ExprNodes.IndexNode):
1262 if node.is_buffer_access:
1263 raise ValueError, "Buffer access"
1264 base, temps = side_effect_free_reference(node.base)
1265 index = LetRefNode(node.index)
1266 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1267 elif isinstance(node, ExprNodes.AttributeNode):
1268 obj, temps = side_effect_free_reference(node.obj)
1269 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1271 node = LetRefNode(node)
1274 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1277 dup = lhs.__class__(**lhs.__dict__)
1278 binop = ExprNodes.binop_node(node.pos,
1279 operator = node.operator,
1283 # Manually analyse types for new node.
1284 lhs.analyse_target_types(env)
1285 dup.analyse_types(env)
1286 binop.analyse_operation(env)
1287 node = Nodes.SingleAssignmentNode(
1290 rhs=binop.coerce_to(lhs.type, env))
1291 # Use LetRefNode to avoid side effects.
1292 let_ref_nodes.reverse()
1293 for t in let_ref_nodes:
1294 node = LetNode(t, node)
1297 def visit_ExprNode(self, node):
1298 # In-place assignments can't happen within an expression.
1302 class AlignFunctionDefinitions(CythonTransform):
1304 This class takes the signatures from a .pxd file and applies them to
1305 the def methods in a .py file.
1308 def visit_ModuleNode(self, node):
1309 self.scope = node.scope
1310 self.directives = node.directives
1311 self.visitchildren(node)
1314 def visit_PyClassDefNode(self, node):
1315 pxd_def = self.scope.lookup(node.name)
1317 if pxd_def.is_cclass:
1318 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1320 error(node.pos, "'%s' redeclared" % node.name)
1322 error(pxd_def.pos, "previous declaration here")
1327 def visit_CClassDefNode(self, node, pxd_def=None):
1329 pxd_def = self.scope.lookup(node.class_name)
1331 outer_scope = self.scope
1332 self.scope = pxd_def.type.scope
1333 self.visitchildren(node)
1335 self.scope = outer_scope
1338 def visit_DefNode(self, node):
1339 pxd_def = self.scope.lookup(node.name)
1340 if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
1341 if not pxd_def.is_cfunction:
1342 error(node.pos, "'%s' redeclared" % node.name)
1344 error(pxd_def.pos, "previous declaration here")
1346 node = node.as_cfunction(pxd_def)
1347 elif (self.scope.is_module_scope and self.directives['auto_cpdef']
1348 and node.is_cdef_func_compatible()):
1349 node = node.as_cfunction(scope=self.scope)
1350 # Enable this when nested cdef functions are allowed.
1351 # self.visitchildren(node)
1355 class YieldNodeCollector(TreeVisitor):
1358 super(YieldNodeCollector, self).__init__()
1361 self.has_return_value = False
1363 def visit_Node(self, node):
1364 return self.visitchildren(node)
1366 def visit_YieldExprNode(self, node):
1367 if self.has_return_value:
1368 error(node.pos, "'yield' outside function")
1369 self.yields.append(node)
1370 self.visitchildren(node)
1372 def visit_ReturnStatNode(self, node):
1374 self.has_return_value = True
1376 error(node.pos, "'return' with argument inside generator")
1377 self.returns.append(node)
1379 def visit_ClassDefNode(self, node):
1382 def visit_FuncDefNode(self, node):
1385 def visit_LambdaNode(self, node):
1388 def visit_GeneratorExpressionNode(self, node):
1391 class MarkClosureVisitor(CythonTransform):
1393 def visit_ModuleNode(self, node):
1394 self.needs_closure = False
1395 self.visitchildren(node)
1398 def visit_FuncDefNode(self, node):
1399 self.needs_closure = False
1400 self.visitchildren(node)
1401 node.needs_closure = self.needs_closure
1402 self.needs_closure = True
1404 collector = YieldNodeCollector()
1405 collector.visitchildren(node)
1407 if collector.yields:
1408 for i, yield_expr in enumerate(collector.yields):
1409 yield_expr.label_num = i + 1
1411 gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1414 generator = Nodes.GeneratorDefNode(pos=node.pos,
1417 star_arg=node.star_arg,
1418 starstar_arg=node.starstar_arg,
1420 decorators=node.decorators,
1422 lambda_name=node.lambda_name)
1426 def visit_CFuncDefNode(self, node):
1427 self.visit_FuncDefNode(node)
1428 if node.needs_closure:
1429 error(node.pos, "closures inside cdef functions not yet supported")
1432 def visit_LambdaNode(self, node):
1433 self.needs_closure = False
1434 self.visitchildren(node)
1435 node.needs_closure = self.needs_closure
1436 self.needs_closure = True
1439 def visit_ClassDefNode(self, node):
1440 self.visitchildren(node)
1441 self.needs_closure = True
1444 class CreateClosureClasses(CythonTransform):
1445 # Output closure classes in module scope for all functions
1446 # that really need it.
1448 def __init__(self, context):
1449 super(CreateClosureClasses, self).__init__(context)
1451 self.in_lambda = False
1452 self.generator_class = None
1454 def visit_ModuleNode(self, node):
1455 self.module_scope = node.scope
1456 self.visitchildren(node)
1459 def create_generator_class(self, target_module_scope, pos):
1460 if self.generator_class:
1461 return self.generator_class
1462 # XXX: make generator class creation cleaner
1463 entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1464 objstruct_cname='__pyx_Generator_object',
1465 typeobj_cname='__pyx_Generator_type',
1466 pos=pos, defining=True, implementing=True)
1467 klass = entry.type.scope
1468 klass.is_internal = True
1469 klass.directives = {'final': True}
1471 body_type = PyrexTypes.create_typedef_type('generator_body',
1472 PyrexTypes.c_void_ptr_type,
1473 '__pyx_generator_body_t')
1474 klass.declare_var(pos=pos, name='body', cname='body',
1475 type=body_type, is_cdef=True)
1476 klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1478 klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1480 klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
1481 type=PyrexTypes.py_object_type, is_cdef=True)
1482 klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
1483 type=PyrexTypes.py_object_type, is_cdef=True)
1484 klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
1485 type=PyrexTypes.py_object_type, is_cdef=True)
1488 e = klass.declare_pyfunction('send', pos)
1489 e.func_cname = '__Pyx_Generator_Send'
1490 e.signature = TypeSlots.binaryfunc
1492 e = klass.declare_pyfunction('close', pos)
1493 e.func_cname = '__Pyx_Generator_Close'
1494 e.signature = TypeSlots.unaryfunc
1496 e = klass.declare_pyfunction('throw', pos)
1497 e.func_cname = '__Pyx_Generator_Throw'
1498 e.signature = TypeSlots.pyfunction_signature
1500 e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1501 e.func_cname = 'PyObject_SelfIter'
1503 e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1504 e.func_cname = '__Pyx_Generator_Next'
1506 self.generator_class = entry.type
1507 return self.generator_class
1509 def find_entries_used_in_closures(self, node):
1512 for name, entry in node.local_scope.entries.items():
1513 if entry.from_closure:
1514 from_closure.append((name, entry))
1515 elif entry.in_closure:
1516 in_closure.append((name, entry))
1517 return from_closure, in_closure
1519 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1520 # skip generator body
1521 if node.is_generator_body:
1523 # move local variables into closure
1524 if node.is_generator:
1525 for entry in node.local_scope.entries.values():
1526 if not entry.from_closure:
1527 entry.in_closure = True
1529 from_closure, in_closure = self.find_entries_used_in_closures(node)
1532 # Now from the begining
1533 node.needs_closure = False
1534 node.needs_outer_scope = False
1536 func_scope = node.local_scope
1537 cscope = node.entry.scope
1538 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1539 cscope = cscope.outer_scope
1541 if not from_closure and (self.path or inner_node):
1544 raise InternalError, "DefNode does not have assignment node"
1545 inner_node = node.assmt.rhs
1546 inner_node.needs_self_code = False
1547 node.needs_outer_scope = False
1550 if node.is_generator:
1551 base_type = self.create_generator_class(target_module_scope, node.pos)
1552 elif not in_closure and not from_closure:
1554 elif not in_closure:
1555 func_scope.is_passthrough = True
1556 func_scope.scope_class = cscope.scope_class
1557 node.needs_outer_scope = True
1560 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1562 entry = target_module_scope.declare_c_class(
1563 name=as_name, pos=node.pos, defining=True,
1564 implementing=True, base_type=base_type)
1566 func_scope.scope_class = entry
1567 class_scope = entry.type.scope
1568 class_scope.is_internal = True
1569 class_scope.directives = {'final': True}
1572 assert cscope.is_closure_scope
1573 class_scope.declare_var(pos=node.pos,
1574 name=Naming.outer_scope_cname,
1575 cname=Naming.outer_scope_cname,
1576 type=cscope.scope_class.type,
1578 node.needs_outer_scope = True
1579 for name, entry in in_closure:
1580 closure_entry = class_scope.declare_var(pos=entry.pos,
1585 if entry.is_declared_generic:
1586 closure_entry.is_declared_generic = 1
1587 node.needs_closure = True
1588 # Do it here because other classes are already checked
1589 target_module_scope.check_c_class(func_scope.scope_class)
1591 def visit_LambdaNode(self, node):
1592 was_in_lambda = self.in_lambda
1593 self.in_lambda = True
1594 self.create_class_from_scope(node.def_node, self.module_scope, node)
1595 self.visitchildren(node)
1596 self.in_lambda = was_in_lambda
1599 def visit_FuncDefNode(self, node):
1601 self.visitchildren(node)
1603 if node.needs_closure or self.path:
1604 self.create_class_from_scope(node, self.module_scope)
1605 self.path.append(node)
1606 self.visitchildren(node)
1611 class GilCheck(VisitorTransform):
1613 Call `node.gil_check(env)` on each node to make sure we hold the
1614 GIL when we need it. Raise an error when on Python operations
1615 inside a `nogil` environment.
1617 def __call__(self, root):
1618 self.env_stack = [root.scope]
1620 return super(GilCheck, self).__call__(root)
1622 def visit_FuncDefNode(self, node):
1623 self.env_stack.append(node.local_scope)
1624 was_nogil = self.nogil
1625 self.nogil = node.local_scope.nogil
1626 if self.nogil and node.nogil_check:
1627 node.nogil_check(node.local_scope)
1628 self.visitchildren(node)
1629 self.env_stack.pop()
1630 self.nogil = was_nogil
1633 def visit_GILStatNode(self, node):
1634 env = self.env_stack[-1]
1635 if self.nogil and node.nogil_check: node.nogil_check()
1636 was_nogil = self.nogil
1637 self.nogil = (node.state == 'nogil')
1638 self.visitchildren(node)
1639 self.nogil = was_nogil
1642 def visit_Node(self, node):
1643 if self.env_stack and self.nogil and node.nogil_check:
1644 node.nogil_check(self.env_stack[-1])
1645 self.visitchildren(node)
1649 class TransformBuiltinMethods(EnvTransform):
1651 def visit_SingleAssignmentNode(self, node):
1652 if node.declaration_only:
1655 self.visitchildren(node)
1658 def visit_AttributeNode(self, node):
1659 self.visitchildren(node)
1660 return self.visit_cython_attribute(node)
1662 def visit_NameNode(self, node):
1663 return self.visit_cython_attribute(node)
1665 def visit_cython_attribute(self, node):
1666 attribute = node.as_cython_attribute()
1668 if attribute == u'compiled':
1669 node = ExprNodes.BoolNode(node.pos, value=True)
1670 elif attribute == u'NULL':
1671 node = ExprNodes.NullNode(node.pos)
1672 elif attribute in (u'set', u'frozenset'):
1673 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1674 entry=self.current_env().builtin_scope().lookup_here(attribute))
1675 elif not PyrexTypes.parse_basic_type(attribute):
1676 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1679 def _inject_locals(self, node, func_name):
1680 # locals()/dir() builtins
1681 lenv = self.current_env()
1682 entry = lenv.lookup_here(func_name)
1687 if func_name == 'locals':
1688 if len(node.args) > 0:
1689 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
1692 items = [ ExprNodes.DictItemNode(pos,
1693 key=ExprNodes.StringNode(pos, value=var),
1694 value=ExprNodes.NameNode(pos, name=var))
1695 for var in lenv.entries ]
1696 return ExprNodes.DictNode(pos, key_value_pairs=items)
1698 if len(node.args) > 1:
1699 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
1702 elif len(node.args) == 1:
1703 # optimised in Builtin.py
1705 items = [ ExprNodes.StringNode(pos, value=var) for var in lenv.entries ]
1706 return ExprNodes.ListNode(pos, args=items)
1708 def visit_SimpleCallNode(self, node):
1709 if isinstance(node.function, ExprNodes.NameNode):
1710 func_name = node.function.name
1711 if func_name in ('dir', 'locals'):
1712 return self._inject_locals(node, func_name)
1715 function = node.function.as_cython_attribute()
1717 if function in InterpretCompilerDirectives.unop_method_nodes:
1718 if len(node.args) != 1:
1719 error(node.function.pos, u"%s() takes exactly one argument" % function)
1721 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1722 elif function in InterpretCompilerDirectives.binop_method_nodes:
1723 if len(node.args) != 2:
1724 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1726 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1727 elif function == u'cast':
1728 if len(node.args) != 2:
1729 error(node.function.pos, u"cast() takes exactly two arguments")
1731 type = node.args[0].analyse_as_type(self.current_env())
1733 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1735 error(node.args[0].pos, "Not a type")
1736 elif function == u'sizeof':
1737 if len(node.args) != 1:
1738 error(node.function.pos, u"sizeof() takes exactly one argument")
1740 type = node.args[0].analyse_as_type(self.current_env())
1742 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1744 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1745 elif function == 'cmod':
1746 if len(node.args) != 2:
1747 error(node.function.pos, u"cmod() takes exactly two arguments")
1749 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1750 node.cdivision = True
1751 elif function == 'cdiv':
1752 if len(node.args) != 2:
1753 error(node.function.pos, u"cdiv() takes exactly two arguments")
1755 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1756 node.cdivision = True
1757 elif function == u'set':
1758 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1760 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1762 self.visitchildren(node)
1766 class DebugTransform(CythonTransform):
1768 Create debug information and all functions' visibility to extern in order
1769 to enable debugging.
1772 def __init__(self, context, options, result):
1773 super(DebugTransform, self).__init__(context)
1774 self.visited = cython.set()
1775 # our treebuilder and debug output writer
1776 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1777 self.tb = self.context.gdb_debug_outputwriter
1778 #self.c_output_file = options.output_file
1779 self.c_output_file = result.c_file
1781 # Closure support, basically treat nested functions as if the AST were
1783 self.nested_funcdefs = []
1785 # tells visit_NameNode whether it should register step-into functions
1786 self.register_stepinto = False
1788 def visit_ModuleNode(self, node):
1789 self.tb.module_name = node.full_module_name
1791 module_name=node.full_module_name,
1792 filename=node.pos[0].filename,
1793 c_filename=self.c_output_file)
1795 self.tb.start('Module', attrs)
1797 # serialize functions
1798 self.tb.start('Functions')
1799 # First, serialize functions normally...
1800 self.visitchildren(node)
1802 # ... then, serialize nested functions
1803 for nested_funcdef in self.nested_funcdefs:
1804 self.visit_FuncDefNode(nested_funcdef)
1806 self.register_stepinto = True
1807 self.serialize_modulenode_as_function(node)
1808 self.register_stepinto = False
1809 self.tb.end('Functions')
1811 # 2.3 compatibility. Serialize global variables
1812 self.tb.start('Globals')
1815 for k, v in node.scope.entries.iteritems():
1816 if (v.qualified_name not in self.visited and not
1817 v.name.startswith('__pyx_') and not
1818 v.type.is_cfunction and not
1819 v.type.is_extension_type):
1822 self.serialize_local_variables(entries)
1823 self.tb.end('Globals')
1824 # self.tb.end('Module') # end Module after the line number mapping in
1825 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1828 def visit_FuncDefNode(self, node):
1829 self.visited.add(node.local_scope.qualified_name)
1831 if getattr(node, 'is_wrapper', False):
1834 if self.register_stepinto:
1835 self.nested_funcdefs.append(node)
1838 # node.entry.visibility = 'extern'
1839 if node.py_func is None:
1842 pf_cname = node.py_func.entry.func_cname
1845 name=node.entry.name,
1846 cname=node.entry.func_cname,
1848 qualified_name=node.local_scope.qualified_name,
1849 lineno=str(node.pos[1]))
1851 self.tb.start('Function', attrs=attrs)
1853 self.tb.start('Locals')
1854 self.serialize_local_variables(node.local_scope.entries)
1855 self.tb.end('Locals')
1857 self.tb.start('Arguments')
1858 for arg in node.local_scope.arg_entries:
1859 self.tb.start(arg.name)
1860 self.tb.end(arg.name)
1861 self.tb.end('Arguments')
1863 self.tb.start('StepIntoFunctions')
1864 self.register_stepinto = True
1865 self.visitchildren(node)
1866 self.register_stepinto = False
1867 self.tb.end('StepIntoFunctions')
1868 self.tb.end('Function')
1872 def visit_NameNode(self, node):
1873 if (self.register_stepinto and
1874 node.type.is_cfunction and
1875 getattr(node, 'is_called', False) and
1876 node.entry.func_cname is not None):
1877 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1878 # declared functions are not 'in_cinclude'.
1879 # This means we will list called 'cdef' functions as
1880 # "step into functions", but this is not an issue as they will be
1881 # recognized as Cython functions anyway.
1882 attrs = dict(name=node.entry.func_cname)
1883 self.tb.start('StepIntoFunction', attrs=attrs)
1884 self.tb.end('StepIntoFunction')
1886 self.visitchildren(node)
1889 def serialize_modulenode_as_function(self, node):
1891 Serialize the module-level code as a function so the debugger will know
1892 it's a "relevant frame" and it will know where to set the breakpoint
1893 for 'break modulename'.
1895 name = node.full_module_name.rpartition('.')[-1]
1897 cname_py2 = 'init' + name
1898 cname_py3 = 'PyInit_' + name
1904 # Ignore the qualified_name, breakpoints should be set using
1905 # `cy break modulename:lineno` for module-level breakpoints.
1908 is_initmodule_function="True",
1911 py3_attrs = dict(py2_attrs, cname=cname_py3)
1913 self._serialize_modulenode_as_function(node, py2_attrs)
1914 self._serialize_modulenode_as_function(node, py3_attrs)
1916 def _serialize_modulenode_as_function(self, node, attrs):
1917 self.tb.start('Function', attrs=attrs)
1919 self.tb.start('Locals')
1920 self.serialize_local_variables(node.scope.entries)
1921 self.tb.end('Locals')
1923 self.tb.start('Arguments')
1924 self.tb.end('Arguments')
1926 self.tb.start('StepIntoFunctions')
1927 self.register_stepinto = True
1928 self.visitchildren(node)
1929 self.register_stepinto = False
1930 self.tb.end('StepIntoFunctions')
1932 self.tb.end('Function')
1934 def serialize_local_variables(self, entries):
1935 for entry in entries.values():
1936 if entry.type.is_pyobject:
1937 vartype = 'PythonObject'
1941 if entry.from_closure:
1942 # We're dealing with a closure where a variable from an outer
1943 # scope is accessed, get it from the scope object.
1944 cname = '%s->%s' % (Naming.cur_scope_cname,
1945 entry.outer_entry.cname)
1947 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1950 elif entry.in_closure:
1951 cname = '%s->%s' % (Naming.cur_scope_cname,
1953 qname = entry.qualified_name
1956 qname = entry.qualified_name
1959 # this happens for variables that are not in the user's code,
1960 # e.g. for the global __builtins__, __doc__, etc. We can just
1961 # set the lineno to 0 for those.
1964 lineno = str(entry.pos[1])
1969 qualified_name=qname,
1973 self.tb.start('LocalVar', attrs)
1974 self.tb.end('LocalVar')