3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
16 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
17 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
18 from Cython.Compiler.ModuleNode import ModuleNode
19 from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
20 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
21 from Cython.Compiler.StringEncoding import EncodedString
22 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
27 class NameNodeCollector(TreeVisitor):
28 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
32 super(NameNodeCollector, self).__init__()
35 def visit_NameNode(self, node):
36 self.name_nodes.append(node)
38 def visit_Node(self, node):
39 self._visitchildren(node, None)
42 class SkipDeclarations(object):
44 Variable and function declarations can often have a deep tree structure,
45 and yet most transformations don't need to descend to this depth.
47 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
48 is no need to use this for transformations after that point.
50 def visit_CTypeDefNode(self, node):
53 def visit_CVarDefNode(self, node):
56 def visit_CDeclaratorNode(self, node):
59 def visit_CBaseTypeNode(self, node):
62 def visit_CEnumDefNode(self, node):
65 def visit_CStructOrUnionDefNode(self, node):
68 class NormalizeTree(CythonTransform):
70 This transform fixes up a few things after parsing
71 in order to make the parse tree more suitable for
74 a) After parsing, blocks with only one statement will
75 be represented by that statement, not by a StatListNode.
76 When doing transforms this is annoying and inconsistent,
77 as one cannot in general remove a statement in a consistent
78 way and so on. This transform wraps any single statements
79 in a StatListNode containing a single statement.
81 b) The PassStatNode is a noop and serves no purpose beyond
82 plugging such one-statement blocks; i.e., once parsed a
83 ` "pass" can just as well be represented using an empty
84 StatListNode. This means less special cases to worry about
85 in subsequent transforms (one always checks to see if a
86 StatListNode has no children to see if the block is empty).
89 def __init__(self, context):
90 super(NormalizeTree, self).__init__(context)
91 self.is_in_statlist = False
92 self.is_in_expr = False
94 def visit_ExprNode(self, node):
95 stacktmp = self.is_in_expr
96 self.is_in_expr = True
97 self.visitchildren(node)
98 self.is_in_expr = stacktmp
101 def visit_StatNode(self, node, is_listcontainer=False):
102 stacktmp = self.is_in_statlist
103 self.is_in_statlist = is_listcontainer
104 self.visitchildren(node)
105 self.is_in_statlist = stacktmp
106 if not self.is_in_statlist and not self.is_in_expr:
107 return Nodes.StatListNode(pos=node.pos, stats=[node])
111 def visit_StatListNode(self, node):
112 self.is_in_statlist = True
113 self.visitchildren(node)
114 self.is_in_statlist = False
117 def visit_ParallelAssignmentNode(self, node):
118 return self.visit_StatNode(node, True)
120 def visit_CEnumDefNode(self, node):
121 return self.visit_StatNode(node, True)
123 def visit_CStructOrUnionDefNode(self, node):
124 return self.visit_StatNode(node, True)
126 # Eliminate PassStatNode
127 def visit_PassStatNode(self, node):
128 if not self.is_in_statlist:
129 return Nodes.StatListNode(pos=node.pos, stats=[])
133 def visit_CDeclaratorNode(self, node):
137 class PostParseError(CompileError): pass
139 # error strings checked by unit tests, so define them
140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
143 class PostParse(ScopeTrackingTransform):
145 Basic interpretation of the parse tree, as well as validity
146 checking that can be done on a very basic level on the parse
147 tree (while still not being a problem with the basic syntax,
151 - Default values to cdef assignments are turned into single
152 assignments following the declaration (everywhere but in class
153 bodies, where they raise a compile error)
155 - Interpret some node structures into Python runtime values.
156 Some nodes take compile-time arguments (currently:
157 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
158 which should be interpreted. This happens in a general way
159 and other steps should be taken to ensure validity.
161 Type arguments cannot be interpreted in this way.
163 - For __cythonbufferdefaults__ the arguments are checked for
166 TemplatedTypeNode has its directives interpreted:
167 Any first positional argument goes into the "dtype" attribute,
168 any "ndim" keyword argument goes into the "ndim" attribute and
169 so on. Also it is checked that the directive combination is valid.
170 - __cythonbufferdefaults__ attributes are parsed and put into the
173 Note: Currently Parsing.py does a lot of interpretation and
174 reorganization that can be refactored into this transform
175 if a more pure Abstract Syntax Tree is wanted.
178 def __init__(self, context):
179 super(PostParse, self).__init__(context)
180 self.specialattribute_handlers = {
181 '__cythonbufferdefaults__' : self.handle_bufferdefaults
184 def visit_ModuleNode(self, node):
185 self.lambda_counter = 1
186 self.genexpr_counter = 1
187 return super(PostParse, self).visit_ModuleNode(node)
189 def visit_LambdaNode(self, node):
190 # unpack a lambda expression into the corresponding DefNode
191 lambda_id = self.lambda_counter
192 self.lambda_counter += 1
193 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
194 collector = YieldNodeCollector()
195 collector.visitchildren(node.result_expr)
196 if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
197 body = Nodes.ExprStatNode(
198 node.result_expr.pos, expr=node.result_expr)
200 body = Nodes.ReturnStatNode(
201 node.result_expr.pos, value=node.result_expr)
202 node.def_node = Nodes.DefNode(
203 node.pos, name=node.name, lambda_name=node.lambda_name,
204 args=node.args, star_arg=node.star_arg,
205 starstar_arg=node.starstar_arg,
207 self.visitchildren(node)
210 def visit_GeneratorExpressionNode(self, node):
211 # unpack a generator expression into the corresponding DefNode
212 genexpr_id = self.genexpr_counter
213 self.genexpr_counter += 1
214 node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
216 node.def_node = Nodes.DefNode(node.pos, name=node.name,
218 args=[], star_arg=None,
221 self.visitchildren(node)
225 def handle_bufferdefaults(self, decl):
226 if not isinstance(decl.default, ExprNodes.DictNode):
227 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
228 self.scope_node.buffer_defaults_node = decl.default
229 self.scope_node.buffer_defaults_pos = decl.pos
231 def visit_CVarDefNode(self, node):
232 # This assumes only plain names and pointers are assignable on
233 # declaration. Also, it makes use of the fact that a cdef decl
234 # must appear before the first use, so we don't have to deal with
235 # "i = 3; cdef int i = i" and can simply move the nodes around.
237 self.visitchildren(node)
240 for decl in node.declarators:
242 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
243 declbase = declbase.base
244 if isinstance(declbase, Nodes.CNameDeclaratorNode):
245 if declbase.default is not None:
246 if self.scope_type in ('cclass', 'pyclass', 'struct'):
247 if isinstance(self.scope_node, Nodes.CClassDefNode):
248 handler = self.specialattribute_handlers.get(decl.name)
250 if decl is not declbase:
251 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
253 continue # Remove declaration
254 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
255 first_assignment = self.scope_type != 'module'
256 stats.append(Nodes.SingleAssignmentNode(node.pos,
257 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
258 rhs=declbase.default, first=first_assignment))
259 declbase.default = None
260 newdecls.append(decl)
261 node.declarators = newdecls
263 except PostParseError, e:
264 # An error in a cdef clause is ok, simply remove the declaration
265 # and try to move on to report more errors
266 self.context.nonfatal_error(e)
269 # Split parallel assignments (a,b = b,a) into separate partial
270 # assignments that are executed rhs-first using temps. This
271 # restructuring must be applied before type analysis so that known
272 # types on rhs and lhs can be matched directly. It is required in
273 # the case that the types cannot be coerced to a Python type in
274 # order to assign from a tuple.
276 def visit_SingleAssignmentNode(self, node):
277 self.visitchildren(node)
278 return self._visit_assignment_node(node, [node.lhs, node.rhs])
280 def visit_CascadedAssignmentNode(self, node):
281 self.visitchildren(node)
282 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
284 def _visit_assignment_node(self, node, expr_list):
285 """Flatten parallel assignments into separate single
286 assignments or cascaded assignments.
288 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
289 # no parallel assignments => nothing to do
293 flatten_parallel_assignments(expr_list, expr_list_list)
295 eliminate_rhs_duplicates(expr_list_list, temp_refs)
298 for expr_list in expr_list_list:
299 lhs_list = expr_list[:-1]
301 if len(lhs_list) == 1:
302 node = Nodes.SingleAssignmentNode(rhs.pos,
303 lhs = lhs_list[0], rhs = rhs)
305 node = Nodes.CascadedAssignmentNode(rhs.pos,
306 lhs_list = lhs_list, rhs = rhs)
310 assign_node = nodes[0]
312 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
315 duplicates_and_temps = [ (temp.expression, temp)
316 for temp in temp_refs ]
317 sort_common_subsequences(duplicates_and_temps)
318 for _, temp_ref in duplicates_and_temps[::-1]:
319 assign_node = LetNode(temp_ref, assign_node)
323 def _flatten_sequence(self, seq, result):
325 if arg.is_sequence_constructor:
326 self._flatten_sequence(arg, result)
331 def visit_DelStatNode(self, node):
332 self.visitchildren(node)
333 node.args = self._flatten_sequence(node, [])
337 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
338 """Replace rhs items by LetRefNodes if they appear more than once.
339 Creates a sequence of LetRefNodes that set up the required temps
340 and appends them to ref_node_sequence. The input list is modified
343 seen_nodes = cython.set()
345 def find_duplicates(node):
346 if node.is_literal or node.is_name:
347 # no need to replace those; can't include attributes here
348 # as their access is not necessarily side-effect free
350 if node in seen_nodes:
351 if node not in ref_nodes:
352 ref_node = LetRefNode(node)
353 ref_nodes[node] = ref_node
354 ref_node_sequence.append(ref_node)
357 if node.is_sequence_constructor:
358 for item in node.args:
359 find_duplicates(item)
361 for expr_list in expr_list_list:
367 def substitute_nodes(node):
368 if node in ref_nodes:
369 return ref_nodes[node]
370 elif node.is_sequence_constructor:
371 node.args = list(map(substitute_nodes, node.args))
374 # replace nodes inside of the common subexpressions
375 for node in ref_nodes:
376 if node.is_sequence_constructor:
377 node.args = list(map(substitute_nodes, node.args))
379 # replace common subexpressions on all rhs items
380 for expr_list in expr_list_list:
381 expr_list[-1] = substitute_nodes(expr_list[-1])
383 def sort_common_subsequences(items):
384 """Sort items/subsequences so that all items and subsequences that
385 an item contains appear before the item itself. This is needed
386 because each rhs item must only be evaluated once, so its value
387 must be evaluated first and then reused when packing sequences
390 This implies a partial order, and the sort must be stable to
391 preserve the original order as much as possible, so we use a
392 simple insertion sort (which is very fast for short sequences, the
393 normal case in practice).
395 def contains(seq, x):
399 elif item.is_sequence_constructor and contains(item.args, x):
403 return b.is_sequence_constructor and contains(b.args, a)
405 for pos, item in enumerate(items):
406 key = item[1] # the ResultRefNode which has already been injected into the sequences
408 for i in xrange(pos-1, -1, -1):
409 if lower_than(key, items[i][0]):
412 for i in xrange(pos, new_pos, -1):
413 items[i] = items[i-1]
414 items[new_pos] = item
416 def flatten_parallel_assignments(input, output):
417 # The input is a list of expression nodes, representing the LHSs
418 # and RHS of one (possibly cascaded) assignment statement. For
419 # sequence constructors, rearranges the matching parts of both
420 # sides into a list of equivalent assignments between the
421 # individual elements. This transformation is applied
422 # recursively, so that nested structures get matched as well.
424 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
428 complete_assignments = []
430 rhs_size = len(rhs.args)
431 lhs_targets = [ [] for _ in xrange(rhs_size) ]
432 starred_assignments = []
433 for lhs in input[:-1]:
434 if not lhs.is_sequence_constructor:
436 error(lhs.pos, "starred assignment target must be in a list or tuple")
437 complete_assignments.append(lhs)
439 lhs_size = len(lhs.args)
440 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
441 if starred_targets > 1:
442 error(lhs.pos, "more than 1 starred expression in assignment")
443 output.append([lhs,rhs])
445 elif lhs_size - starred_targets > rhs_size:
446 error(lhs.pos, "need more than %d value%s to unpack"
447 % (rhs_size, (rhs_size != 1) and 's' or ''))
448 output.append([lhs,rhs])
450 elif starred_targets:
451 map_starred_assignment(lhs_targets, starred_assignments,
453 elif lhs_size < rhs_size:
454 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
455 % (lhs_size, rhs_size))
456 output.append([lhs,rhs])
459 for targets, expr in zip(lhs_targets, lhs.args):
462 if complete_assignments:
463 complete_assignments.append(rhs)
464 output.append(complete_assignments)
466 # recursively flatten partial assignments
467 for cascade, rhs in zip(lhs_targets, rhs.args):
470 flatten_parallel_assignments(cascade, output)
472 # recursively flatten starred assignments
473 for cascade in starred_assignments:
474 if cascade[0].is_sequence_constructor:
475 flatten_parallel_assignments(cascade, output)
477 output.append(cascade)
479 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
480 # Appends the fixed-position LHS targets to the target list that
481 # appear left and right of the starred argument.
483 # The starred_assignments list receives a new tuple
484 # (lhs_target, rhs_values_list) that maps the remaining arguments
485 # (those that match the starred target) to a list.
487 # left side of the starred target
488 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
491 lhs_remaining = len(lhs_args) - i - 1
495 raise InternalError("no starred arg found when splitting starred assignment")
497 # right side of the starred target
498 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
499 lhs_args[starred + 1:])):
502 # the starred target itself, must be assigned a (potentially empty) list
503 target = lhs_args[starred].target # unpack starred node
504 starred_rhs = rhs_args[starred:]
506 starred_rhs = starred_rhs[:-lhs_remaining]
508 pos = starred_rhs[0].pos
511 starred_assignments.append([
512 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
515 class PxdPostParse(CythonTransform, SkipDeclarations):
517 Basic interpretation/validity checking that should only be
520 A lot of this checking currently happens in the parser; but
521 what is listed below happens here.
523 - "def" functions are let through only if they fill the
524 getbuffer/releasebuffer slots
526 - cdef functions are let through only if they are on the
527 top level and are declared "inline"
529 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
530 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
532 def __call__(self, node):
533 self.scope_type = 'pxd'
534 return super(PxdPostParse, self).__call__(node)
536 def visit_CClassDefNode(self, node):
537 old = self.scope_type
538 self.scope_type = 'cclass'
539 self.visitchildren(node)
540 self.scope_type = old
543 def visit_FuncDefNode(self, node):
544 # FuncDefNode always come with an implementation (without
545 # an imp they are CVarDefNodes..)
546 err = self.ERR_INLINE_ONLY
548 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
549 and node.name in ('__getbuffer__', '__releasebuffer__')):
550 err = None # allow these slots
552 if isinstance(node, Nodes.CFuncDefNode):
553 if u'inline' in node.modifiers and self.scope_type == 'pxd':
554 node.inline_in_pxd = True
555 if node.visibility != 'private':
556 err = self.ERR_NOGO_WITH_INLINE % node.visibility
558 err = self.ERR_NOGO_WITH_INLINE % 'api'
560 err = None # allow inline function
562 err = self.ERR_INLINE_ONLY
565 self.context.nonfatal_error(PostParseError(node.pos, err))
570 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
572 After parsing, directives can be stored in a number of places:
573 - #cython-comments at the top of the file (stored in ModuleNode)
574 - Command-line arguments overriding these
575 - @cython.directivename decorators
576 - with cython.directivename: statements
578 This transform is responsible for interpreting these various sources
579 and store the directive in two ways:
580 - Set the directives attribute of the ModuleNode for global directives.
581 - Use a CompilerDirectivesNode to override directives for a subtree.
583 (The first one is primarily to not have to modify with the tree
584 structure, so that ModuleNode stay on top.)
586 The directives are stored in dictionaries from name to value in effect.
587 Each such dictionary is always filled in for all possible directives,
588 using default values where no value is given by the user.
590 The available directives are controlled in Options.py.
592 Note that we have to run this prior to analysis, and so some minor
593 duplication of functionality has to occur: We manually track cimports
594 and which names the "cython" module may have been imported to.
596 unop_method_nodes = {
597 'typeof': ExprNodes.TypeofNode,
599 'operator.address': ExprNodes.AmpersandNode,
600 'operator.dereference': ExprNodes.DereferenceNode,
601 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
602 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
603 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
604 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
606 # For backwards compatability.
607 'address': ExprNodes.AmpersandNode,
610 binop_method_nodes = {
611 'operator.comma' : ExprNodes.c_binop_constructor(','),
614 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
615 'cast', 'pointer', 'compiled', 'NULL'])
616 special_methods.update(unop_method_nodes.keys())
618 def __init__(self, context, compilation_directive_defaults):
619 super(InterpretCompilerDirectives, self).__init__(context)
620 self.compilation_directive_defaults = {}
621 for key, value in compilation_directive_defaults.items():
622 self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
623 self.cython_module_names = cython.set()
624 self.directive_names = {}
626 def check_directive_scope(self, pos, directive, scope):
627 legal_scopes = Options.directive_scopes.get(directive, None)
628 if legal_scopes and scope not in legal_scopes:
629 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
630 'is not allowed in %s scope' % (directive, scope)))
635 # Set up processing and handle the cython: comments.
636 def visit_ModuleNode(self, node):
637 for key, value in node.directive_comments.items():
638 if not self.check_directive_scope(node.pos, key, 'module'):
639 self.wrong_scope_error(node.pos, key, 'module')
640 del node.directive_comments[key]
642 directives = copy.deepcopy(Options.directive_defaults)
643 directives.update(copy.deepcopy(self.compilation_directive_defaults))
644 directives.update(node.directive_comments)
645 self.directives = directives
646 node.directives = directives
647 self.visitchildren(node)
648 node.cython_module_names = self.cython_module_names
651 # The following four functions track imports and cimports that
652 # begin with "cython"
653 def is_cython_directive(self, name):
654 return (name in Options.directive_types or
655 name in self.special_methods or
656 PyrexTypes.parse_basic_type(name))
658 def visit_CImportStatNode(self, node):
659 if node.module_name == u"cython":
660 self.cython_module_names.add(node.as_name or u"cython")
661 elif node.module_name.startswith(u"cython."):
663 self.directive_names[node.as_name] = node.module_name[7:]
665 self.cython_module_names.add(u"cython")
666 # if this cimport was a compiler directive, we don't
667 # want to leave the cimport node sitting in the tree
671 def visit_FromCImportStatNode(self, node):
672 if (node.module_name == u"cython") or \
673 node.module_name.startswith(u"cython."):
674 submodule = (node.module_name + u".")[7:]
676 for pos, name, as_name, kind in node.imported_names:
677 full_name = submodule + name
678 if self.is_cython_directive(full_name):
681 self.directive_names[as_name] = full_name
683 self.context.nonfatal_error(PostParseError(pos,
684 "Compiler directive imports must be plain imports"))
686 newimp.append((pos, name, as_name, kind))
689 node.imported_names = newimp
692 def visit_FromImportStatNode(self, node):
693 if (node.module.module_name.value == u"cython") or \
694 node.module.module_name.value.startswith(u"cython."):
695 submodule = (node.module.module_name.value + u".")[7:]
697 for name, name_node in node.items:
698 full_name = submodule + name
699 if self.is_cython_directive(full_name):
700 self.directive_names[name_node.name] = full_name
702 newimp.append((name, name_node))
708 def visit_SingleAssignmentNode(self, node):
709 if (isinstance(node.rhs, ExprNodes.ImportNode) and
710 node.rhs.module_name.value == u'cython'):
711 node = Nodes.CImportStatNode(node.pos,
712 module_name = u'cython',
713 as_name = node.lhs.name)
714 self.visit_CImportStatNode(node)
716 self.visitchildren(node)
719 def visit_NameNode(self, node):
720 if node.name in self.cython_module_names:
721 node.is_cython_module = True
723 node.cython_attribute = self.directive_names.get(node.name)
726 def try_to_parse_directives(self, node):
727 # If node is the contents of an directive (in a with statement or
728 # decorator), returns a list of (directivename, value) pairs.
729 # Otherwise, returns None
730 if isinstance(node, ExprNodes.CallNode):
731 self.visit(node.function)
732 optname = node.function.as_cython_attribute()
734 directivetype = Options.directive_types.get(optname)
736 args, kwds = node.explicit_args_kwds()
739 if kwds is not None and directivetype is not dict:
740 for keyvalue in kwds.key_value_pairs:
741 key, value = keyvalue
742 sub_optname = "%s.%s" % (optname, key.value)
743 if Options.directive_types.get(sub_optname):
744 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
746 key_value_pairs.append(keyvalue)
747 if not key_value_pairs:
750 kwds.key_value_pairs = key_value_pairs
751 if directives and not kwds and not args:
753 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
755 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
757 optname = node.as_cython_attribute()
759 directivetype = Options.directive_types.get(optname)
760 if directivetype is bool:
761 return [(optname, True)]
762 elif directivetype is None:
763 return [(optname, None)]
765 raise PostParseError(
766 node.pos, "The '%s' directive should be used as a function call." % optname)
769 def try_to_parse_directive(self, optname, args, kwds, pos):
770 directivetype = Options.directive_types.get(optname)
771 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
772 return optname, Options.directive_defaults[optname]
773 elif directivetype is bool:
774 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
775 raise PostParseError(pos,
776 'The %s directive takes one compile-time boolean argument' % optname)
777 return (optname, args[0].value)
778 elif directivetype is str:
779 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
780 ExprNodes.UnicodeNode)):
781 raise PostParseError(pos,
782 'The %s directive takes one compile-time string argument' % optname)
783 return (optname, str(args[0].value))
784 elif directivetype is dict:
786 raise PostParseError(pos,
787 'The %s directive takes no prepositional arguments' % optname)
788 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
789 elif directivetype is list:
790 if kwds and len(kwds) != 0:
791 raise PostParseError(pos,
792 'The %s directive takes no keyword arguments' % optname)
793 return optname, [ str(arg.value) for arg in args ]
797 def visit_with_directives(self, body, directives):
798 olddirectives = self.directives
799 newdirectives = copy.copy(olddirectives)
800 newdirectives.update(directives)
801 self.directives = newdirectives
802 assert isinstance(body, Nodes.StatListNode), body
803 retbody = self.visit_Node(body)
804 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
805 directives=newdirectives)
806 self.directives = olddirectives
810 def visit_FuncDefNode(self, node):
811 directives = self._extract_directives(node, 'function')
813 return self.visit_Node(node)
814 body = Nodes.StatListNode(node.pos, stats=[node])
815 return self.visit_with_directives(body, directives)
817 def visit_CVarDefNode(self, node):
818 if not node.decorators:
820 for dec in node.decorators:
821 for directive in self.try_to_parse_directives(dec.decorator) or ():
822 if directive is not None and directive[0] == u'locals':
823 node.directive_locals = directive[1]
825 self.context.nonfatal_error(PostParseError(dec.pos,
826 "Cdef functions can only take cython.locals() decorator."))
829 def visit_CClassDefNode(self, node):
830 directives = self._extract_directives(node, 'cclass')
832 return self.visit_Node(node)
833 body = Nodes.StatListNode(node.pos, stats=[node])
834 return self.visit_with_directives(body, directives)
836 def visit_PyClassDefNode(self, node):
837 directives = self._extract_directives(node, 'class')
839 return self.visit_Node(node)
840 body = Nodes.StatListNode(node.pos, stats=[node])
841 return self.visit_with_directives(body, directives)
843 def _extract_directives(self, node, scope_name):
844 if not node.decorators:
846 # Split the decorators into two lists -- real decorators and directives
849 for dec in node.decorators:
850 new_directives = self.try_to_parse_directives(dec.decorator)
851 if new_directives is not None:
852 for directive in new_directives:
853 if self.check_directive_scope(node.pos, directive[0], scope_name):
854 directives.append(directive)
857 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
858 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
860 node.decorators = realdecs
861 # merge or override repeated directives
863 directives.reverse() # Decorators coming first take precedence
864 for directive in directives:
865 name, value = directive
867 old_value = optdict[name]
868 # keywords and arg lists can be merged, everything
869 # else overrides completely
870 if isinstance(old_value, dict):
871 old_value.update(value)
872 elif isinstance(old_value, list):
873 old_value.extend(value)
875 optdict[name] = value
877 optdict[name] = value
880 # Handle with statements
881 def visit_WithStatNode(self, node):
883 for directive in self.try_to_parse_directives(node.manager) or []:
884 if directive is not None:
885 if node.target is not None:
886 self.context.nonfatal_error(
887 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
889 name, value = directive
891 # special case: in pure mode, "with nogil" spells "with cython.nogil"
892 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
893 return self.visit_Node(node)
894 if self.check_directive_scope(node.pos, name, 'with statement'):
895 directive_dict[name] = value
897 return self.visit_with_directives(node.body, directive_dict)
898 return self.visit_Node(node)
900 class WithTransform(CythonTransform, SkipDeclarations):
901 def visit_WithStatNode(self, node):
902 self.visitchildren(node, 'body')
904 body, target, manager = node.body, node.target, node.manager
905 node.target_temp = ExprNodes.TempNode(pos, type=PyrexTypes.py_object_type)
906 if target is not None:
907 node.has_target = True
908 body = Nodes.StatListNode(
910 Nodes.WithTargetAssignmentStatNode(
911 pos, lhs = target, rhs = node.target_temp),
916 excinfo_target = ResultRefNode(
917 pos=pos, type=Builtin.tuple_type, may_hold_none=False)
918 except_clause = Nodes.ExceptClauseNode(
919 pos, body = Nodes.IfStatNode(
922 pos, condition = ExprNodes.NotNode(
923 pos, operand = ExprNodes.WithExitCallNode(
924 pos, with_stat = node,
925 args = excinfo_target)),
926 body = Nodes.ReraiseStatNode(pos),
932 excinfo_target = excinfo_target,
935 node.body = Nodes.TryFinallyStatNode(
936 pos, body = Nodes.TryExceptStatNode(
938 except_clauses = [except_clause],
941 finally_clause = Nodes.ExprStatNode(
942 pos, expr = ExprNodes.WithExitCallNode(
943 pos, with_stat = node,
944 args = ExprNodes.TupleNode(
945 pos, args = [ExprNodes.NoneNode(pos) for _ in range(3)]
947 handle_error_case = False,
951 def visit_ExprNode(self, node):
952 # With statements are never inside expressions.
956 class DecoratorTransform(CythonTransform, SkipDeclarations):
958 def visit_DefNode(self, func_node):
959 self.visitchildren(func_node)
960 if not func_node.decorators:
962 return self._handle_decorators(
963 func_node, func_node.name)
965 def visit_CClassDefNode(self, class_node):
966 # This doesn't currently work, so it's disabled.
968 # Problem: assignments to cdef class names do not work. They
969 # would require an additional check anyway, as the extension
970 # type must not change its C type, so decorators cannot
971 # replace an extension type, just alter it and return it.
973 self.visitchildren(class_node)
974 if not class_node.decorators:
976 error(class_node.pos,
977 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
979 #return self._handle_decorators(
980 # class_node, class_node.class_name)
982 def visit_ClassDefNode(self, class_node):
983 self.visitchildren(class_node)
984 if not class_node.decorators:
986 return self._handle_decorators(
987 class_node, class_node.name)
989 def _handle_decorators(self, node, name):
990 decorator_result = ExprNodes.NameNode(node.pos, name = name)
991 for decorator in node.decorators[::-1]:
992 decorator_result = ExprNodes.SimpleCallNode(
994 function = decorator.decorator,
995 args = [decorator_result])
997 name_node = ExprNodes.NameNode(node.pos, name = name)
998 reassignment = Nodes.SingleAssignmentNode(
1001 rhs = decorator_result)
1002 return [node, reassignment]
1005 class AnalyseDeclarationsTransform(CythonTransform):
1007 basic_property = TreeFragment(u"""
1011 def __set__(self, value):
1013 """, level='c_class')
1014 basic_pyobject_property = TreeFragment(u"""
1018 def __set__(self, value):
1022 """, level='c_class')
1023 basic_property_ro = TreeFragment(u"""
1027 """, level='c_class')
1029 struct_or_union_wrapper = TreeFragment(u"""
1032 def __init__(self, MEMBER=None):
1036 if IS_UNION and count > 1:
1037 raise ValueError, "At most one union member should be specified."
1039 return STR_FORMAT % MEMBER_TUPLE
1041 return REPR_FORMAT % MEMBER_TUPLE
1044 init_assignment = TreeFragment(u"""
1045 if VALUE is not None:
1050 def __call__(self, root):
1051 self.env_stack = [root.scope]
1052 # needed to determine if a cdef var is declared after it's used.
1053 self.seen_vars_stack = []
1054 return super(AnalyseDeclarationsTransform, self).__call__(root)
1056 def visit_NameNode(self, node):
1057 self.seen_vars_stack[-1].add(node.name)
1060 def visit_ModuleNode(self, node):
1061 self.seen_vars_stack.append(cython.set())
1062 node.analyse_declarations(self.env_stack[-1])
1063 self.visitchildren(node)
1064 self.seen_vars_stack.pop()
1067 def visit_LambdaNode(self, node):
1068 node.analyse_declarations(self.env_stack[-1])
1069 self.visitchildren(node)
1072 def visit_ClassDefNode(self, node):
1073 self.env_stack.append(node.scope)
1074 self.visitchildren(node)
1075 self.env_stack.pop()
1078 def visit_CClassDefNode(self, node):
1079 node = self.visit_ClassDefNode(node)
1080 if node.scope and node.scope.implemented:
1082 for entry in node.scope.var_entries:
1083 if entry.needs_property:
1084 property = self.create_Property(entry)
1085 property.analyse_declarations(node.scope)
1086 self.visit(property)
1087 stats.append(property)
1089 node.body.stats += stats
1092 def visit_FuncDefNode(self, node):
1093 self.seen_vars_stack.append(cython.set())
1094 lenv = node.local_scope
1095 node.body.analyse_control_flow(lenv) # this will be totally refactored
1096 node.declare_arguments(lenv)
1097 for var, type_node in node.directive_locals.items():
1098 if not lenv.lookup_here(var): # don't redeclare args
1099 type = type_node.analyse_as_type(lenv)
1101 lenv.declare_var(var, type, type_node.pos)
1103 error(type_node.pos, "Not a type")
1104 node.body.analyse_declarations(lenv)
1105 self.env_stack.append(lenv)
1106 self.visitchildren(node)
1107 self.env_stack.pop()
1108 self.seen_vars_stack.pop()
1111 def visit_ScopedExprNode(self, node):
1112 env = self.env_stack[-1]
1113 node.analyse_declarations(env)
1114 # the node may or may not have a local scope
1115 if node.has_local_scope:
1116 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1117 self.env_stack.append(node.expr_scope)
1118 node.analyse_scoped_declarations(node.expr_scope)
1119 self.visitchildren(node)
1120 self.env_stack.pop()
1121 self.seen_vars_stack.pop()
1123 node.analyse_scoped_declarations(env)
1124 self.visitchildren(node)
1127 def visit_TempResultFromStatNode(self, node):
1128 self.visitchildren(node)
1129 node.analyse_declarations(self.env_stack[-1])
1132 def visit_CStructOrUnionDefNode(self, node):
1133 # Create a wrapper node if needed.
1134 # We want to use the struct type information (so it can't happen
1135 # before this phase) but also create new objects to be declared
1136 # (so it can't happen later).
1137 # Note that we don't return the original node, as it is
1138 # never used after this phase.
1139 if True: # private (default)
1142 self_value = ExprNodes.AttributeNode(
1144 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1145 attribute = EncodedString(u"value"))
1146 var_entries = node.entry.type.scope.var_entries
1148 for entry in var_entries:
1149 attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1151 attribute = entry.name))
1152 # __init__ assignments
1153 init_assignments = []
1154 for entry, attr in zip(var_entries, attributes):
1155 # TODO: branch on visibility
1156 init_assignments.append(self.init_assignment.substitute({
1157 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1159 }, pos = entry.pos))
1162 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1163 wrapper_class = self.struct_or_union_wrapper.substitute({
1164 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1165 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1166 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1167 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1168 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1169 }, pos = node.pos).stats[0]
1170 wrapper_class.class_name = node.name
1171 wrapper_class.shadow = True
1172 class_body = wrapper_class.body.stats
1175 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1176 class_body[0].base_type.name = node.name
1178 # fix __init__ arguments
1179 init_method = class_body[1]
1180 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1181 arg_template = init_method.args[1]
1182 if not node.entry.type.is_struct:
1183 arg_template.kw_only = True
1184 del init_method.args[1]
1185 for entry, attr in zip(var_entries, attributes):
1186 arg = copy.deepcopy(arg_template)
1187 arg.declarator.name = entry.name
1188 init_method.args.append(arg)
1191 for entry, attr in zip(var_entries, attributes):
1192 # TODO: branch on visibility
1193 if entry.type.is_pyobject:
1194 template = self.basic_pyobject_property
1196 template = self.basic_property
1197 property = template.substitute({
1199 }, pos = entry.pos).stats[0]
1200 property.name = entry.name
1201 wrapper_class.body.stats.append(property)
1203 wrapper_class.analyse_declarations(self.env_stack[-1])
1204 return self.visit_CClassDefNode(wrapper_class)
1206 # Some nodes are no longer needed after declaration
1207 # analysis and can be dropped. The analysis was performed
1208 # on these nodes in a seperate recursive process from the
1209 # enclosing function or module, so we can simply drop them.
1210 def visit_CDeclaratorNode(self, node):
1211 # necessary to ensure that all CNameDeclaratorNodes are visited.
1212 self.visitchildren(node)
1215 def visit_CTypeDefNode(self, node):
1218 def visit_CBaseTypeNode(self, node):
1221 def visit_CEnumDefNode(self, node):
1222 if node.visibility == 'public':
1227 def visit_CNameDeclaratorNode(self, node):
1228 if node.name in self.seen_vars_stack[-1]:
1229 entry = self.env_stack[-1].lookup(node.name)
1230 if (entry is None or entry.visibility != 'extern'
1231 and not entry.scope.is_c_class_scope):
1232 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1233 self.visitchildren(node)
1236 def visit_CVarDefNode(self, node):
1237 # to ensure all CNameDeclaratorNodes are visited.
1238 self.visitchildren(node)
1241 def create_Property(self, entry):
1242 if entry.visibility == 'public':
1243 if entry.type.is_pyobject:
1244 template = self.basic_pyobject_property
1246 template = self.basic_property
1247 elif entry.visibility == 'readonly':
1248 template = self.basic_property_ro
1249 property = template.substitute({
1250 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1251 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1252 attribute=entry.name),
1253 }, pos=entry.pos).stats[0]
1254 property.name = entry.name
1255 # ---------------------------------------
1256 # XXX This should go to AutoDocTransforms
1257 # ---------------------------------------
1258 if (Options.docstrings and
1259 self.current_directives['embedsignature']):
1260 attr_name = entry.name
1261 type_name = entry.type.declaration_code("", for_display=1)
1263 if not entry.type.is_pyobject:
1264 type_name = "'%s'" % type_name
1265 elif entry.type.is_extension_type:
1266 type_name = entry.type.module_name + '.' + type_name
1267 if entry.init is not None:
1268 default_value = ' = ' + entry.init
1269 elif entry.init_to_none:
1270 default_value = ' = ' + repr(None)
1271 docstring = attr_name + ': ' + type_name + default_value
1272 property.doc = EncodedString(docstring)
1273 # ---------------------------------------
1276 class AnalyseExpressionsTransform(CythonTransform):
1278 def visit_ModuleNode(self, node):
1279 node.scope.infer_types()
1280 node.body.analyse_expressions(node.scope)
1281 self.visitchildren(node)
1284 def visit_FuncDefNode(self, node):
1285 node.local_scope.infer_types()
1286 node.body.analyse_expressions(node.local_scope)
1287 self.visitchildren(node)
1290 def visit_ScopedExprNode(self, node):
1291 if node.has_local_scope:
1292 node.expr_scope.infer_types()
1293 node.analyse_scoped_expressions(node.expr_scope)
1294 self.visitchildren(node)
1297 class ExpandInplaceOperators(EnvTransform):
1299 def visit_InPlaceAssignmentNode(self, node):
1302 if lhs.type.is_cpp_class:
1303 # No getting around this exact operator here.
1305 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1306 # There is code to handle this case.
1309 env = self.current_env()
1310 def side_effect_free_reference(node, setting=False):
1311 if isinstance(node, ExprNodes.NameNode):
1313 elif node.type.is_pyobject and not setting:
1314 node = LetRefNode(node)
1316 elif isinstance(node, ExprNodes.IndexNode):
1317 if node.is_buffer_access:
1318 raise ValueError, "Buffer access"
1319 base, temps = side_effect_free_reference(node.base)
1320 index = LetRefNode(node.index)
1321 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1322 elif isinstance(node, ExprNodes.AttributeNode):
1323 obj, temps = side_effect_free_reference(node.obj)
1324 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1326 node = LetRefNode(node)
1329 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1332 dup = lhs.__class__(**lhs.__dict__)
1333 binop = ExprNodes.binop_node(node.pos,
1334 operator = node.operator,
1338 # Manually analyse types for new node.
1339 lhs.analyse_target_types(env)
1340 dup.analyse_types(env)
1341 binop.analyse_operation(env)
1342 node = Nodes.SingleAssignmentNode(
1345 rhs=binop.coerce_to(lhs.type, env))
1346 # Use LetRefNode to avoid side effects.
1347 let_ref_nodes.reverse()
1348 for t in let_ref_nodes:
1349 node = LetNode(t, node)
1352 def visit_ExprNode(self, node):
1353 # In-place assignments can't happen within an expression.
1357 class AlignFunctionDefinitions(CythonTransform):
1359 This class takes the signatures from a .pxd file and applies them to
1360 the def methods in a .py file.
1363 def visit_ModuleNode(self, node):
1364 self.scope = node.scope
1365 self.directives = node.directives
1366 self.visitchildren(node)
1369 def visit_PyClassDefNode(self, node):
1370 pxd_def = self.scope.lookup(node.name)
1372 if pxd_def.is_cclass:
1373 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1375 error(node.pos, "'%s' redeclared" % node.name)
1377 error(pxd_def.pos, "previous declaration here")
1382 def visit_CClassDefNode(self, node, pxd_def=None):
1384 pxd_def = self.scope.lookup(node.class_name)
1386 outer_scope = self.scope
1387 self.scope = pxd_def.type.scope
1388 self.visitchildren(node)
1390 self.scope = outer_scope
1393 def visit_DefNode(self, node):
1394 pxd_def = self.scope.lookup(node.name)
1395 if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
1396 if not pxd_def.is_cfunction:
1397 error(node.pos, "'%s' redeclared" % node.name)
1399 error(pxd_def.pos, "previous declaration here")
1401 node = node.as_cfunction(pxd_def)
1402 elif (self.scope.is_module_scope and self.directives['auto_cpdef']
1403 and node.is_cdef_func_compatible()):
1404 node = node.as_cfunction(scope=self.scope)
1405 # Enable this when nested cdef functions are allowed.
1406 # self.visitchildren(node)
1410 class YieldNodeCollector(TreeVisitor):
1413 super(YieldNodeCollector, self).__init__()
1416 self.has_return_value = False
1418 def visit_Node(self, node):
1419 return self.visitchildren(node)
1421 def visit_YieldExprNode(self, node):
1422 if self.has_return_value:
1423 error(node.pos, "'yield' outside function")
1424 self.yields.append(node)
1425 self.visitchildren(node)
1427 def visit_ReturnStatNode(self, node):
1429 self.has_return_value = True
1431 error(node.pos, "'return' with argument inside generator")
1432 self.returns.append(node)
1434 def visit_ClassDefNode(self, node):
1437 def visit_FuncDefNode(self, node):
1440 def visit_LambdaNode(self, node):
1443 def visit_GeneratorExpressionNode(self, node):
1446 class MarkClosureVisitor(CythonTransform):
1448 def visit_ModuleNode(self, node):
1449 self.needs_closure = False
1450 self.visitchildren(node)
1453 def visit_FuncDefNode(self, node):
1454 self.needs_closure = False
1455 self.visitchildren(node)
1456 node.needs_closure = self.needs_closure
1457 self.needs_closure = True
1459 collector = YieldNodeCollector()
1460 collector.visitchildren(node)
1462 if collector.yields:
1463 for i, yield_expr in enumerate(collector.yields):
1464 yield_expr.label_num = i + 1
1466 gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1469 generator = Nodes.GeneratorDefNode(pos=node.pos,
1472 star_arg=node.star_arg,
1473 starstar_arg=node.starstar_arg,
1475 decorators=node.decorators,
1477 lambda_name=node.lambda_name)
1481 def visit_CFuncDefNode(self, node):
1482 self.visit_FuncDefNode(node)
1483 if node.needs_closure:
1484 error(node.pos, "closures inside cdef functions not yet supported")
1487 def visit_LambdaNode(self, node):
1488 self.needs_closure = False
1489 self.visitchildren(node)
1490 node.needs_closure = self.needs_closure
1491 self.needs_closure = True
1494 def visit_ClassDefNode(self, node):
1495 self.visitchildren(node)
1496 self.needs_closure = True
1499 class CreateClosureClasses(CythonTransform):
1500 # Output closure classes in module scope for all functions
1501 # that really need it.
1503 def __init__(self, context):
1504 super(CreateClosureClasses, self).__init__(context)
1506 self.in_lambda = False
1507 self.generator_class = None
1509 def visit_ModuleNode(self, node):
1510 self.module_scope = node.scope
1511 self.visitchildren(node)
1514 def create_generator_class(self, target_module_scope, pos):
1515 if self.generator_class:
1516 return self.generator_class
1517 # XXX: make generator class creation cleaner
1518 entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1519 objstruct_cname='__pyx_Generator_object',
1520 typeobj_cname='__pyx_Generator_type',
1521 pos=pos, defining=True, implementing=True)
1522 klass = entry.type.scope
1523 klass.is_internal = True
1524 klass.directives = {'final': True}
1526 body_type = PyrexTypes.create_typedef_type('generator_body',
1527 PyrexTypes.c_void_ptr_type,
1528 '__pyx_generator_body_t')
1529 klass.declare_var(pos=pos, name='body', cname='body',
1530 type=body_type, is_cdef=True)
1531 klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1533 klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1535 klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
1536 type=PyrexTypes.py_object_type, is_cdef=True)
1537 klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
1538 type=PyrexTypes.py_object_type, is_cdef=True)
1539 klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
1540 type=PyrexTypes.py_object_type, is_cdef=True)
1543 e = klass.declare_pyfunction('send', pos)
1544 e.func_cname = '__Pyx_Generator_Send'
1545 e.signature = TypeSlots.binaryfunc
1547 e = klass.declare_pyfunction('close', pos)
1548 e.func_cname = '__Pyx_Generator_Close'
1549 e.signature = TypeSlots.unaryfunc
1551 e = klass.declare_pyfunction('throw', pos)
1552 e.func_cname = '__Pyx_Generator_Throw'
1553 e.signature = TypeSlots.pyfunction_signature
1555 e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1556 e.func_cname = 'PyObject_SelfIter'
1558 e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1559 e.func_cname = '__Pyx_Generator_Next'
1561 self.generator_class = entry.type
1562 return self.generator_class
1564 def find_entries_used_in_closures(self, node):
1567 for name, entry in node.local_scope.entries.items():
1568 if entry.from_closure:
1569 from_closure.append((name, entry))
1570 elif entry.in_closure:
1571 in_closure.append((name, entry))
1572 return from_closure, in_closure
1574 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1575 # skip generator body
1576 if node.is_generator_body:
1578 # move local variables into closure
1579 if node.is_generator:
1580 for entry in node.local_scope.entries.values():
1581 if not entry.from_closure:
1582 entry.in_closure = True
1584 from_closure, in_closure = self.find_entries_used_in_closures(node)
1587 # Now from the begining
1588 node.needs_closure = False
1589 node.needs_outer_scope = False
1591 func_scope = node.local_scope
1592 cscope = node.entry.scope
1593 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1594 cscope = cscope.outer_scope
1596 if not from_closure and (self.path or inner_node):
1599 raise InternalError, "DefNode does not have assignment node"
1600 inner_node = node.assmt.rhs
1601 inner_node.needs_self_code = False
1602 node.needs_outer_scope = False
1605 if node.is_generator:
1606 base_type = self.create_generator_class(target_module_scope, node.pos)
1607 elif not in_closure and not from_closure:
1609 elif not in_closure:
1610 func_scope.is_passthrough = True
1611 func_scope.scope_class = cscope.scope_class
1612 node.needs_outer_scope = True
1615 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1617 entry = target_module_scope.declare_c_class(
1618 name=as_name, pos=node.pos, defining=True,
1619 implementing=True, base_type=base_type)
1621 func_scope.scope_class = entry
1622 class_scope = entry.type.scope
1623 class_scope.is_internal = True
1624 class_scope.directives = {'final': True}
1627 assert cscope.is_closure_scope
1628 class_scope.declare_var(pos=node.pos,
1629 name=Naming.outer_scope_cname,
1630 cname=Naming.outer_scope_cname,
1631 type=cscope.scope_class.type,
1633 node.needs_outer_scope = True
1634 for name, entry in in_closure:
1635 closure_entry = class_scope.declare_var(pos=entry.pos,
1640 if entry.is_declared_generic:
1641 closure_entry.is_declared_generic = 1
1642 node.needs_closure = True
1643 # Do it here because other classes are already checked
1644 target_module_scope.check_c_class(func_scope.scope_class)
1646 def visit_LambdaNode(self, node):
1647 was_in_lambda = self.in_lambda
1648 self.in_lambda = True
1649 self.create_class_from_scope(node.def_node, self.module_scope, node)
1650 self.visitchildren(node)
1651 self.in_lambda = was_in_lambda
1654 def visit_FuncDefNode(self, node):
1656 self.visitchildren(node)
1658 if node.needs_closure or self.path:
1659 self.create_class_from_scope(node, self.module_scope)
1660 self.path.append(node)
1661 self.visitchildren(node)
1666 class GilCheck(VisitorTransform):
1668 Call `node.gil_check(env)` on each node to make sure we hold the
1669 GIL when we need it. Raise an error when on Python operations
1670 inside a `nogil` environment.
1672 def __call__(self, root):
1673 self.env_stack = [root.scope]
1675 return super(GilCheck, self).__call__(root)
1677 def visit_FuncDefNode(self, node):
1678 self.env_stack.append(node.local_scope)
1679 was_nogil = self.nogil
1680 self.nogil = node.local_scope.nogil
1681 if self.nogil and node.nogil_check:
1682 node.nogil_check(node.local_scope)
1683 self.visitchildren(node)
1684 self.env_stack.pop()
1685 self.nogil = was_nogil
1688 def visit_GILStatNode(self, node):
1689 env = self.env_stack[-1]
1690 if self.nogil and node.nogil_check: node.nogil_check()
1691 was_nogil = self.nogil
1692 self.nogil = (node.state == 'nogil')
1693 self.visitchildren(node)
1694 self.nogil = was_nogil
1697 def visit_Node(self, node):
1698 if self.env_stack and self.nogil and node.nogil_check:
1699 node.nogil_check(self.env_stack[-1])
1700 self.visitchildren(node)
1704 class TransformBuiltinMethods(EnvTransform):
1706 def visit_SingleAssignmentNode(self, node):
1707 if node.declaration_only:
1710 self.visitchildren(node)
1713 def visit_AttributeNode(self, node):
1714 self.visitchildren(node)
1715 return self.visit_cython_attribute(node)
1717 def visit_NameNode(self, node):
1718 return self.visit_cython_attribute(node)
1720 def visit_cython_attribute(self, node):
1721 attribute = node.as_cython_attribute()
1723 if attribute == u'compiled':
1724 node = ExprNodes.BoolNode(node.pos, value=True)
1725 elif attribute == u'NULL':
1726 node = ExprNodes.NullNode(node.pos)
1727 elif attribute in (u'set', u'frozenset'):
1728 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1729 entry=self.current_env().builtin_scope().lookup_here(attribute))
1730 elif not PyrexTypes.parse_basic_type(attribute):
1731 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1734 def _inject_locals(self, node, func_name):
1735 # locals()/dir() builtins
1736 lenv = self.current_env()
1737 entry = lenv.lookup_here(func_name)
1742 if func_name == 'locals':
1743 if len(node.args) > 0:
1744 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
1747 items = [ ExprNodes.DictItemNode(pos,
1748 key=ExprNodes.StringNode(pos, value=var),
1749 value=ExprNodes.NameNode(pos, name=var))
1750 for var in lenv.entries ]
1751 return ExprNodes.DictNode(pos, key_value_pairs=items)
1753 if len(node.args) > 1:
1754 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
1757 elif len(node.args) == 1:
1758 # optimised in Builtin.py
1760 items = [ ExprNodes.StringNode(pos, value=var) for var in lenv.entries ]
1761 return ExprNodes.ListNode(pos, args=items)
1763 def visit_SimpleCallNode(self, node):
1764 if isinstance(node.function, ExprNodes.NameNode):
1765 func_name = node.function.name
1766 if func_name in ('dir', 'locals'):
1767 return self._inject_locals(node, func_name)
1770 function = node.function.as_cython_attribute()
1772 if function in InterpretCompilerDirectives.unop_method_nodes:
1773 if len(node.args) != 1:
1774 error(node.function.pos, u"%s() takes exactly one argument" % function)
1776 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1777 elif function in InterpretCompilerDirectives.binop_method_nodes:
1778 if len(node.args) != 2:
1779 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1781 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1782 elif function == u'cast':
1783 if len(node.args) != 2:
1784 error(node.function.pos, u"cast() takes exactly two arguments")
1786 type = node.args[0].analyse_as_type(self.current_env())
1788 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1790 error(node.args[0].pos, "Not a type")
1791 elif function == u'sizeof':
1792 if len(node.args) != 1:
1793 error(node.function.pos, u"sizeof() takes exactly one argument")
1795 type = node.args[0].analyse_as_type(self.current_env())
1797 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1799 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1800 elif function == 'cmod':
1801 if len(node.args) != 2:
1802 error(node.function.pos, u"cmod() takes exactly two arguments")
1804 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1805 node.cdivision = True
1806 elif function == 'cdiv':
1807 if len(node.args) != 2:
1808 error(node.function.pos, u"cdiv() takes exactly two arguments")
1810 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1811 node.cdivision = True
1812 elif function == u'set':
1813 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1815 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1817 self.visitchildren(node)
1821 class DebugTransform(CythonTransform):
1823 Create debug information and all functions' visibility to extern in order
1824 to enable debugging.
1827 def __init__(self, context, options, result):
1828 super(DebugTransform, self).__init__(context)
1829 self.visited = cython.set()
1830 # our treebuilder and debug output writer
1831 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1832 self.tb = self.context.gdb_debug_outputwriter
1833 #self.c_output_file = options.output_file
1834 self.c_output_file = result.c_file
1836 # Closure support, basically treat nested functions as if the AST were
1838 self.nested_funcdefs = []
1840 # tells visit_NameNode whether it should register step-into functions
1841 self.register_stepinto = False
1843 def visit_ModuleNode(self, node):
1844 self.tb.module_name = node.full_module_name
1846 module_name=node.full_module_name,
1847 filename=node.pos[0].filename,
1848 c_filename=self.c_output_file)
1850 self.tb.start('Module', attrs)
1852 # serialize functions
1853 self.tb.start('Functions')
1854 # First, serialize functions normally...
1855 self.visitchildren(node)
1857 # ... then, serialize nested functions
1858 for nested_funcdef in self.nested_funcdefs:
1859 self.visit_FuncDefNode(nested_funcdef)
1861 self.register_stepinto = True
1862 self.serialize_modulenode_as_function(node)
1863 self.register_stepinto = False
1864 self.tb.end('Functions')
1866 # 2.3 compatibility. Serialize global variables
1867 self.tb.start('Globals')
1870 for k, v in node.scope.entries.iteritems():
1871 if (v.qualified_name not in self.visited and not
1872 v.name.startswith('__pyx_') and not
1873 v.type.is_cfunction and not
1874 v.type.is_extension_type):
1877 self.serialize_local_variables(entries)
1878 self.tb.end('Globals')
1879 # self.tb.end('Module') # end Module after the line number mapping in
1880 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1883 def visit_FuncDefNode(self, node):
1884 self.visited.add(node.local_scope.qualified_name)
1886 if getattr(node, 'is_wrapper', False):
1889 if self.register_stepinto:
1890 self.nested_funcdefs.append(node)
1893 # node.entry.visibility = 'extern'
1894 if node.py_func is None:
1897 pf_cname = node.py_func.entry.func_cname
1900 name=node.entry.name,
1901 cname=node.entry.func_cname,
1903 qualified_name=node.local_scope.qualified_name,
1904 lineno=str(node.pos[1]))
1906 self.tb.start('Function', attrs=attrs)
1908 self.tb.start('Locals')
1909 self.serialize_local_variables(node.local_scope.entries)
1910 self.tb.end('Locals')
1912 self.tb.start('Arguments')
1913 for arg in node.local_scope.arg_entries:
1914 self.tb.start(arg.name)
1915 self.tb.end(arg.name)
1916 self.tb.end('Arguments')
1918 self.tb.start('StepIntoFunctions')
1919 self.register_stepinto = True
1920 self.visitchildren(node)
1921 self.register_stepinto = False
1922 self.tb.end('StepIntoFunctions')
1923 self.tb.end('Function')
1927 def visit_NameNode(self, node):
1928 if (self.register_stepinto and
1929 node.type.is_cfunction and
1930 getattr(node, 'is_called', False) and
1931 node.entry.func_cname is not None):
1932 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1933 # declared functions are not 'in_cinclude'.
1934 # This means we will list called 'cdef' functions as
1935 # "step into functions", but this is not an issue as they will be
1936 # recognized as Cython functions anyway.
1937 attrs = dict(name=node.entry.func_cname)
1938 self.tb.start('StepIntoFunction', attrs=attrs)
1939 self.tb.end('StepIntoFunction')
1941 self.visitchildren(node)
1944 def serialize_modulenode_as_function(self, node):
1946 Serialize the module-level code as a function so the debugger will know
1947 it's a "relevant frame" and it will know where to set the breakpoint
1948 for 'break modulename'.
1950 name = node.full_module_name.rpartition('.')[-1]
1952 cname_py2 = 'init' + name
1953 cname_py3 = 'PyInit_' + name
1959 # Ignore the qualified_name, breakpoints should be set using
1960 # `cy break modulename:lineno` for module-level breakpoints.
1963 is_initmodule_function="True",
1966 py3_attrs = dict(py2_attrs, cname=cname_py3)
1968 self._serialize_modulenode_as_function(node, py2_attrs)
1969 self._serialize_modulenode_as_function(node, py3_attrs)
1971 def _serialize_modulenode_as_function(self, node, attrs):
1972 self.tb.start('Function', attrs=attrs)
1974 self.tb.start('Locals')
1975 self.serialize_local_variables(node.scope.entries)
1976 self.tb.end('Locals')
1978 self.tb.start('Arguments')
1979 self.tb.end('Arguments')
1981 self.tb.start('StepIntoFunctions')
1982 self.register_stepinto = True
1983 self.visitchildren(node)
1984 self.register_stepinto = False
1985 self.tb.end('StepIntoFunctions')
1987 self.tb.end('Function')
1989 def serialize_local_variables(self, entries):
1990 for entry in entries.values():
1991 if entry.type.is_pyobject:
1992 vartype = 'PythonObject'
1996 if entry.from_closure:
1997 # We're dealing with a closure where a variable from an outer
1998 # scope is accessed, get it from the scope object.
1999 cname = '%s->%s' % (Naming.cur_scope_cname,
2000 entry.outer_entry.cname)
2002 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2005 elif entry.in_closure:
2006 cname = '%s->%s' % (Naming.cur_scope_cname,
2008 qname = entry.qualified_name
2011 qname = entry.qualified_name
2014 # this happens for variables that are not in the user's code,
2015 # e.g. for the global __builtins__, __doc__, etc. We can just
2016 # set the lineno to 0 for those.
2019 lineno = str(entry.pos[1])
2024 qualified_name=qname,
2028 self.tb.start('LocalVar', attrs)
2029 self.tb.end('LocalVar')