3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
17 from Cython.Compiler.ModuleNode import ModuleNode
18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
20 from Cython.Compiler.StringEncoding import EncodedString
21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
26 class NameNodeCollector(TreeVisitor):
27 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
31 super(NameNodeCollector, self).__init__()
34 def visit_NameNode(self, node):
35 self.name_nodes.append(node)
37 def visit_Node(self, node):
38 self._visitchildren(node, None)
41 class SkipDeclarations(object):
43 Variable and function declarations can often have a deep tree structure,
44 and yet most transformations don't need to descend to this depth.
46 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
47 is no need to use this for transformations after that point.
49 def visit_CTypeDefNode(self, node):
52 def visit_CVarDefNode(self, node):
55 def visit_CDeclaratorNode(self, node):
58 def visit_CBaseTypeNode(self, node):
61 def visit_CEnumDefNode(self, node):
64 def visit_CStructOrUnionDefNode(self, node):
68 class NormalizeTree(CythonTransform):
70 This transform fixes up a few things after parsing
71 in order to make the parse tree more suitable for
74 a) After parsing, blocks with only one statement will
75 be represented by that statement, not by a StatListNode.
76 When doing transforms this is annoying and inconsistent,
77 as one cannot in general remove a statement in a consistent
78 way and so on. This transform wraps any single statements
79 in a StatListNode containing a single statement.
81 b) The PassStatNode is a noop and serves no purpose beyond
82 plugging such one-statement blocks; i.e., once parsed a
83 ` "pass" can just as well be represented using an empty
84 StatListNode. This means less special cases to worry about
85 in subsequent transforms (one always checks to see if a
86 StatListNode has no children to see if the block is empty).
89 def __init__(self, context):
90 super(NormalizeTree, self).__init__(context)
91 self.is_in_statlist = False
92 self.is_in_expr = False
94 def visit_ExprNode(self, node):
95 stacktmp = self.is_in_expr
96 self.is_in_expr = True
97 self.visitchildren(node)
98 self.is_in_expr = stacktmp
101 def visit_StatNode(self, node, is_listcontainer=False):
102 stacktmp = self.is_in_statlist
103 self.is_in_statlist = is_listcontainer
104 self.visitchildren(node)
105 self.is_in_statlist = stacktmp
106 if not self.is_in_statlist and not self.is_in_expr:
107 return Nodes.StatListNode(pos=node.pos, stats=[node])
111 def visit_StatListNode(self, node):
112 self.is_in_statlist = True
113 self.visitchildren(node)
114 self.is_in_statlist = False
117 def visit_ParallelAssignmentNode(self, node):
118 return self.visit_StatNode(node, True)
120 def visit_CEnumDefNode(self, node):
121 return self.visit_StatNode(node, True)
123 def visit_CStructOrUnionDefNode(self, node):
124 return self.visit_StatNode(node, True)
126 # Eliminate PassStatNode
127 def visit_PassStatNode(self, node):
128 if not self.is_in_statlist:
129 return Nodes.StatListNode(pos=node.pos, stats=[])
133 def visit_CDeclaratorNode(self, node):
137 class PostParseError(CompileError): pass
139 # error strings checked by unit tests, so define them
140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
143 class PostParse(ScopeTrackingTransform):
145 Basic interpretation of the parse tree, as well as validity
146 checking that can be done on a very basic level on the parse
147 tree (while still not being a problem with the basic syntax,
151 - Default values to cdef assignments are turned into single
152 assignments following the declaration (everywhere but in class
153 bodies, where they raise a compile error)
155 - Interpret some node structures into Python runtime values.
156 Some nodes take compile-time arguments (currently:
157 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
158 which should be interpreted. This happens in a general way
159 and other steps should be taken to ensure validity.
161 Type arguments cannot be interpreted in this way.
163 - For __cythonbufferdefaults__ the arguments are checked for
166 TemplatedTypeNode has its directives interpreted:
167 Any first positional argument goes into the "dtype" attribute,
168 any "ndim" keyword argument goes into the "ndim" attribute and
169 so on. Also it is checked that the directive combination is valid.
170 - __cythonbufferdefaults__ attributes are parsed and put into the
173 Note: Currently Parsing.py does a lot of interpretation and
174 reorganization that can be refactored into this transform
175 if a more pure Abstract Syntax Tree is wanted.
178 def __init__(self, context):
179 super(PostParse, self).__init__(context)
180 self.specialattribute_handlers = {
181 '__cythonbufferdefaults__' : self.handle_bufferdefaults
184 def visit_ModuleNode(self, node):
185 self.lambda_counter = 1
186 self.genexpr_counter = 1
187 return super(PostParse, self).visit_ModuleNode(node)
189 def visit_LambdaNode(self, node):
190 # unpack a lambda expression into the corresponding DefNode
191 lambda_id = self.lambda_counter
192 self.lambda_counter += 1
193 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
194 collector = YieldNodeCollector()
195 collector.visitchildren(node.result_expr)
196 if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
197 body = ExprNodes.YieldExprNode(
198 node.result_expr.pos, arg=node.result_expr)
199 body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
201 body = Nodes.ReturnStatNode(
202 node.result_expr.pos, value=node.result_expr)
203 node.def_node = Nodes.DefNode(
204 node.pos, name=node.name, lambda_name=node.lambda_name,
205 args=node.args, star_arg=node.star_arg,
206 starstar_arg=node.starstar_arg,
208 self.visitchildren(node)
211 def visit_GeneratorExpressionNode(self, node):
212 # unpack a generator expression into the corresponding DefNode
213 genexpr_id = self.genexpr_counter
214 self.genexpr_counter += 1
215 node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
217 node.def_node = Nodes.DefNode(node.pos, name=node.name,
219 args=[], star_arg=None,
222 self.visitchildren(node)
226 def handle_bufferdefaults(self, decl):
227 if not isinstance(decl.default, ExprNodes.DictNode):
228 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
229 self.scope_node.buffer_defaults_node = decl.default
230 self.scope_node.buffer_defaults_pos = decl.pos
232 def visit_CVarDefNode(self, node):
233 # This assumes only plain names and pointers are assignable on
234 # declaration. Also, it makes use of the fact that a cdef decl
235 # must appear before the first use, so we don't have to deal with
236 # "i = 3; cdef int i = i" and can simply move the nodes around.
238 self.visitchildren(node)
241 for decl in node.declarators:
243 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
244 declbase = declbase.base
245 if isinstance(declbase, Nodes.CNameDeclaratorNode):
246 if declbase.default is not None:
247 if self.scope_type in ('cclass', 'pyclass', 'struct'):
248 if isinstance(self.scope_node, Nodes.CClassDefNode):
249 handler = self.specialattribute_handlers.get(decl.name)
251 if decl is not declbase:
252 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
254 continue # Remove declaration
255 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
256 first_assignment = self.scope_type != 'module'
257 stats.append(Nodes.SingleAssignmentNode(node.pos,
258 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
259 rhs=declbase.default, first=first_assignment))
260 declbase.default = None
261 newdecls.append(decl)
262 node.declarators = newdecls
264 except PostParseError, e:
265 # An error in a cdef clause is ok, simply remove the declaration
266 # and try to move on to report more errors
267 self.context.nonfatal_error(e)
270 # Split parallel assignments (a,b = b,a) into separate partial
271 # assignments that are executed rhs-first using temps. This
272 # restructuring must be applied before type analysis so that known
273 # types on rhs and lhs can be matched directly. It is required in
274 # the case that the types cannot be coerced to a Python type in
275 # order to assign from a tuple.
277 def visit_SingleAssignmentNode(self, node):
278 self.visitchildren(node)
279 return self._visit_assignment_node(node, [node.lhs, node.rhs])
281 def visit_CascadedAssignmentNode(self, node):
282 self.visitchildren(node)
283 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
285 def _visit_assignment_node(self, node, expr_list):
286 """Flatten parallel assignments into separate single
287 assignments or cascaded assignments.
289 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
290 # no parallel assignments => nothing to do
294 flatten_parallel_assignments(expr_list, expr_list_list)
296 eliminate_rhs_duplicates(expr_list_list, temp_refs)
299 for expr_list in expr_list_list:
300 lhs_list = expr_list[:-1]
302 if len(lhs_list) == 1:
303 node = Nodes.SingleAssignmentNode(rhs.pos,
304 lhs = lhs_list[0], rhs = rhs)
306 node = Nodes.CascadedAssignmentNode(rhs.pos,
307 lhs_list = lhs_list, rhs = rhs)
311 assign_node = nodes[0]
313 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
316 duplicates_and_temps = [ (temp.expression, temp)
317 for temp in temp_refs ]
318 sort_common_subsequences(duplicates_and_temps)
319 for _, temp_ref in duplicates_and_temps[::-1]:
320 assign_node = LetNode(temp_ref, assign_node)
324 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
325 """Replace rhs items by LetRefNodes if they appear more than once.
326 Creates a sequence of LetRefNodes that set up the required temps
327 and appends them to ref_node_sequence. The input list is modified
330 seen_nodes = cython.set()
332 def find_duplicates(node):
333 if node.is_literal or node.is_name:
334 # no need to replace those; can't include attributes here
335 # as their access is not necessarily side-effect free
337 if node in seen_nodes:
338 if node not in ref_nodes:
339 ref_node = LetRefNode(node)
340 ref_nodes[node] = ref_node
341 ref_node_sequence.append(ref_node)
344 if node.is_sequence_constructor:
345 for item in node.args:
346 find_duplicates(item)
348 for expr_list in expr_list_list:
354 def substitute_nodes(node):
355 if node in ref_nodes:
356 return ref_nodes[node]
357 elif node.is_sequence_constructor:
358 node.args = list(map(substitute_nodes, node.args))
361 # replace nodes inside of the common subexpressions
362 for node in ref_nodes:
363 if node.is_sequence_constructor:
364 node.args = list(map(substitute_nodes, node.args))
366 # replace common subexpressions on all rhs items
367 for expr_list in expr_list_list:
368 expr_list[-1] = substitute_nodes(expr_list[-1])
370 def sort_common_subsequences(items):
371 """Sort items/subsequences so that all items and subsequences that
372 an item contains appear before the item itself. This is needed
373 because each rhs item must only be evaluated once, so its value
374 must be evaluated first and then reused when packing sequences
377 This implies a partial order, and the sort must be stable to
378 preserve the original order as much as possible, so we use a
379 simple insertion sort (which is very fast for short sequences, the
380 normal case in practice).
382 def contains(seq, x):
386 elif item.is_sequence_constructor and contains(item.args, x):
390 return b.is_sequence_constructor and contains(b.args, a)
392 for pos, item in enumerate(items):
393 key = item[1] # the ResultRefNode which has already been injected into the sequences
395 for i in xrange(pos-1, -1, -1):
396 if lower_than(key, items[i][0]):
399 for i in xrange(pos, new_pos, -1):
400 items[i] = items[i-1]
401 items[new_pos] = item
403 def flatten_parallel_assignments(input, output):
404 # The input is a list of expression nodes, representing the LHSs
405 # and RHS of one (possibly cascaded) assignment statement. For
406 # sequence constructors, rearranges the matching parts of both
407 # sides into a list of equivalent assignments between the
408 # individual elements. This transformation is applied
409 # recursively, so that nested structures get matched as well.
411 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
415 complete_assignments = []
417 rhs_size = len(rhs.args)
418 lhs_targets = [ [] for _ in xrange(rhs_size) ]
419 starred_assignments = []
420 for lhs in input[:-1]:
421 if not lhs.is_sequence_constructor:
423 error(lhs.pos, "starred assignment target must be in a list or tuple")
424 complete_assignments.append(lhs)
426 lhs_size = len(lhs.args)
427 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
428 if starred_targets > 1:
429 error(lhs.pos, "more than 1 starred expression in assignment")
430 output.append([lhs,rhs])
432 elif lhs_size - starred_targets > rhs_size:
433 error(lhs.pos, "need more than %d value%s to unpack"
434 % (rhs_size, (rhs_size != 1) and 's' or ''))
435 output.append([lhs,rhs])
437 elif starred_targets:
438 map_starred_assignment(lhs_targets, starred_assignments,
440 elif lhs_size < rhs_size:
441 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
442 % (lhs_size, rhs_size))
443 output.append([lhs,rhs])
446 for targets, expr in zip(lhs_targets, lhs.args):
449 if complete_assignments:
450 complete_assignments.append(rhs)
451 output.append(complete_assignments)
453 # recursively flatten partial assignments
454 for cascade, rhs in zip(lhs_targets, rhs.args):
457 flatten_parallel_assignments(cascade, output)
459 # recursively flatten starred assignments
460 for cascade in starred_assignments:
461 if cascade[0].is_sequence_constructor:
462 flatten_parallel_assignments(cascade, output)
464 output.append(cascade)
466 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
467 # Appends the fixed-position LHS targets to the target list that
468 # appear left and right of the starred argument.
470 # The starred_assignments list receives a new tuple
471 # (lhs_target, rhs_values_list) that maps the remaining arguments
472 # (those that match the starred target) to a list.
474 # left side of the starred target
475 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
478 lhs_remaining = len(lhs_args) - i - 1
482 raise InternalError("no starred arg found when splitting starred assignment")
484 # right side of the starred target
485 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
486 lhs_args[starred + 1:])):
489 # the starred target itself, must be assigned a (potentially empty) list
490 target = lhs_args[starred].target # unpack starred node
491 starred_rhs = rhs_args[starred:]
493 starred_rhs = starred_rhs[:-lhs_remaining]
495 pos = starred_rhs[0].pos
498 starred_assignments.append([
499 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
502 class PxdPostParse(CythonTransform, SkipDeclarations):
504 Basic interpretation/validity checking that should only be
507 A lot of this checking currently happens in the parser; but
508 what is listed below happens here.
510 - "def" functions are let through only if they fill the
511 getbuffer/releasebuffer slots
513 - cdef functions are let through only if they are on the
514 top level and are declared "inline"
516 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
517 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
519 def __call__(self, node):
520 self.scope_type = 'pxd'
521 return super(PxdPostParse, self).__call__(node)
523 def visit_CClassDefNode(self, node):
524 old = self.scope_type
525 self.scope_type = 'cclass'
526 self.visitchildren(node)
527 self.scope_type = old
530 def visit_FuncDefNode(self, node):
531 # FuncDefNode always come with an implementation (without
532 # an imp they are CVarDefNodes..)
533 err = self.ERR_INLINE_ONLY
535 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
536 and node.name in ('__getbuffer__', '__releasebuffer__')):
537 err = None # allow these slots
539 if isinstance(node, Nodes.CFuncDefNode):
540 if u'inline' in node.modifiers and self.scope_type == 'pxd':
541 node.inline_in_pxd = True
542 if node.visibility != 'private':
543 err = self.ERR_NOGO_WITH_INLINE % node.visibility
545 err = self.ERR_NOGO_WITH_INLINE % 'api'
547 err = None # allow inline function
549 err = self.ERR_INLINE_ONLY
552 self.context.nonfatal_error(PostParseError(node.pos, err))
557 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
559 After parsing, directives can be stored in a number of places:
560 - #cython-comments at the top of the file (stored in ModuleNode)
561 - Command-line arguments overriding these
562 - @cython.directivename decorators
563 - with cython.directivename: statements
565 This transform is responsible for interpreting these various sources
566 and store the directive in two ways:
567 - Set the directives attribute of the ModuleNode for global directives.
568 - Use a CompilerDirectivesNode to override directives for a subtree.
570 (The first one is primarily to not have to modify with the tree
571 structure, so that ModuleNode stay on top.)
573 The directives are stored in dictionaries from name to value in effect.
574 Each such dictionary is always filled in for all possible directives,
575 using default values where no value is given by the user.
577 The available directives are controlled in Options.py.
579 Note that we have to run this prior to analysis, and so some minor
580 duplication of functionality has to occur: We manually track cimports
581 and which names the "cython" module may have been imported to.
583 unop_method_nodes = {
584 'typeof': ExprNodes.TypeofNode,
586 'operator.address': ExprNodes.AmpersandNode,
587 'operator.dereference': ExprNodes.DereferenceNode,
588 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
589 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
590 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
591 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
593 # For backwards compatability.
594 'address': ExprNodes.AmpersandNode,
597 binop_method_nodes = {
598 'operator.comma' : ExprNodes.c_binop_constructor(','),
601 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
602 'cast', 'pointer', 'compiled', 'NULL'])
603 special_methods.update(unop_method_nodes.keys())
605 def __init__(self, context, compilation_directive_defaults):
606 super(InterpretCompilerDirectives, self).__init__(context)
607 self.compilation_directive_defaults = {}
608 for key, value in compilation_directive_defaults.items():
609 self.compilation_directive_defaults[unicode(key)] = value
610 self.cython_module_names = cython.set()
611 self.directive_names = {}
613 def check_directive_scope(self, pos, directive, scope):
614 legal_scopes = Options.directive_scopes.get(directive, None)
615 if legal_scopes and scope not in legal_scopes:
616 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
617 'is not allowed in %s scope' % (directive, scope)))
622 # Set up processing and handle the cython: comments.
623 def visit_ModuleNode(self, node):
624 for key, value in node.directive_comments.items():
625 if not self.check_directive_scope(node.pos, key, 'module'):
626 self.wrong_scope_error(node.pos, key, 'module')
627 del node.directive_comments[key]
629 directives = copy.copy(Options.directive_defaults)
630 directives.update(self.compilation_directive_defaults)
631 directives.update(node.directive_comments)
632 self.directives = directives
633 node.directives = directives
634 self.visitchildren(node)
635 node.cython_module_names = self.cython_module_names
638 # The following four functions track imports and cimports that
639 # begin with "cython"
640 def is_cython_directive(self, name):
641 return (name in Options.directive_types or
642 name in self.special_methods or
643 PyrexTypes.parse_basic_type(name))
645 def visit_CImportStatNode(self, node):
646 if node.module_name == u"cython":
647 self.cython_module_names.add(node.as_name or u"cython")
648 elif node.module_name.startswith(u"cython."):
650 self.directive_names[node.as_name] = node.module_name[7:]
652 self.cython_module_names.add(u"cython")
653 # if this cimport was a compiler directive, we don't
654 # want to leave the cimport node sitting in the tree
658 def visit_FromCImportStatNode(self, node):
659 if (node.module_name == u"cython") or \
660 node.module_name.startswith(u"cython."):
661 submodule = (node.module_name + u".")[7:]
663 for pos, name, as_name, kind in node.imported_names:
664 full_name = submodule + name
665 if self.is_cython_directive(full_name):
668 self.directive_names[as_name] = full_name
670 self.context.nonfatal_error(PostParseError(pos,
671 "Compiler directive imports must be plain imports"))
673 newimp.append((pos, name, as_name, kind))
676 node.imported_names = newimp
679 def visit_FromImportStatNode(self, node):
680 if (node.module.module_name.value == u"cython") or \
681 node.module.module_name.value.startswith(u"cython."):
682 submodule = (node.module.module_name.value + u".")[7:]
684 for name, name_node in node.items:
685 full_name = submodule + name
686 if self.is_cython_directive(full_name):
687 self.directive_names[name_node.name] = full_name
689 newimp.append((name, name_node))
695 def visit_SingleAssignmentNode(self, node):
696 if (isinstance(node.rhs, ExprNodes.ImportNode) and
697 node.rhs.module_name.value == u'cython'):
698 node = Nodes.CImportStatNode(node.pos,
699 module_name = u'cython',
700 as_name = node.lhs.name)
701 self.visit_CImportStatNode(node)
703 self.visitchildren(node)
706 def visit_NameNode(self, node):
707 if node.name in self.cython_module_names:
708 node.is_cython_module = True
710 node.cython_attribute = self.directive_names.get(node.name)
713 def try_to_parse_directives(self, node):
714 # If node is the contents of an directive (in a with statement or
715 # decorator), returns a list of (directivename, value) pairs.
716 # Otherwise, returns None
717 if isinstance(node, ExprNodes.CallNode):
718 self.visit(node.function)
719 optname = node.function.as_cython_attribute()
721 directivetype = Options.directive_types.get(optname)
723 args, kwds = node.explicit_args_kwds()
726 if kwds is not None and directivetype is not dict:
727 for keyvalue in kwds.key_value_pairs:
728 key, value = keyvalue
729 sub_optname = "%s.%s" % (optname, key.value)
730 if Options.directive_types.get(sub_optname):
731 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
733 key_value_pairs.append(keyvalue)
734 if not key_value_pairs:
737 kwds.key_value_pairs = key_value_pairs
738 if directives and not kwds and not args:
740 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
742 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
744 optname = node.as_cython_attribute()
746 directivetype = Options.directive_types.get(optname)
747 if directivetype is bool:
748 return [(optname, True)]
749 elif directivetype is None:
750 return [(optname, None)]
752 raise PostParseError(
753 node.pos, "The '%s' directive should be used as a function call." % optname)
756 def try_to_parse_directive(self, optname, args, kwds, pos):
757 directivetype = Options.directive_types.get(optname)
758 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
759 return optname, Options.directive_defaults[optname]
760 elif directivetype is bool:
761 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
762 raise PostParseError(pos,
763 'The %s directive takes one compile-time boolean argument' % optname)
764 return (optname, args[0].value)
765 elif directivetype is str:
766 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
767 ExprNodes.UnicodeNode)):
768 raise PostParseError(pos,
769 'The %s directive takes one compile-time string argument' % optname)
770 return (optname, str(args[0].value))
771 elif directivetype is dict:
773 raise PostParseError(pos,
774 'The %s directive takes no prepositional arguments' % optname)
775 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
776 elif directivetype is list:
777 if kwds and len(kwds) != 0:
778 raise PostParseError(pos,
779 'The %s directive takes no keyword arguments' % optname)
780 return optname, [ str(arg.value) for arg in args ]
784 def visit_with_directives(self, body, directives):
785 olddirectives = self.directives
786 newdirectives = copy.copy(olddirectives)
787 newdirectives.update(directives)
788 self.directives = newdirectives
789 assert isinstance(body, Nodes.StatListNode), body
790 retbody = self.visit_Node(body)
791 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
792 directives=newdirectives)
793 self.directives = olddirectives
797 def visit_FuncDefNode(self, node):
798 directives = self._extract_directives(node, 'function')
800 return self.visit_Node(node)
801 body = Nodes.StatListNode(node.pos, stats=[node])
802 return self.visit_with_directives(body, directives)
804 def visit_CVarDefNode(self, node):
805 if not node.decorators:
807 for dec in node.decorators:
808 for directive in self.try_to_parse_directives(dec.decorator) or ():
809 if directive is not None and directive[0] == u'locals':
810 node.directive_locals = directive[1]
812 self.context.nonfatal_error(PostParseError(dec.pos,
813 "Cdef functions can only take cython.locals() decorator."))
816 def visit_CClassDefNode(self, node):
817 directives = self._extract_directives(node, 'cclass')
819 return self.visit_Node(node)
820 body = Nodes.StatListNode(node.pos, stats=[node])
821 return self.visit_with_directives(body, directives)
823 def visit_PyClassDefNode(self, node):
824 directives = self._extract_directives(node, 'class')
826 return self.visit_Node(node)
827 body = Nodes.StatListNode(node.pos, stats=[node])
828 return self.visit_with_directives(body, directives)
830 def _extract_directives(self, node, scope_name):
831 if not node.decorators:
833 # Split the decorators into two lists -- real decorators and directives
836 for dec in node.decorators:
837 new_directives = self.try_to_parse_directives(dec.decorator)
838 if new_directives is not None:
839 for directive in new_directives:
840 if self.check_directive_scope(node.pos, directive[0], scope_name):
841 directives.append(directive)
844 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
845 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
847 node.decorators = realdecs
848 # merge or override repeated directives
850 directives.reverse() # Decorators coming first take precedence
851 for directive in directives:
852 name, value = directive
854 old_value = optdict[name]
855 # keywords and arg lists can be merged, everything
856 # else overrides completely
857 if isinstance(old_value, dict):
858 old_value.update(value)
859 elif isinstance(old_value, list):
860 old_value.extend(value)
862 optdict[name] = value
864 optdict[name] = value
867 # Handle with statements
868 def visit_WithStatNode(self, node):
870 for directive in self.try_to_parse_directives(node.manager) or []:
871 if directive is not None:
872 if node.target is not None:
873 self.context.nonfatal_error(
874 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
876 name, value = directive
878 # special case: in pure mode, "with nogil" spells "with cython.nogil"
879 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
880 return self.visit_Node(node)
881 if self.check_directive_scope(node.pos, name, 'with statement'):
882 directive_dict[name] = value
884 return self.visit_with_directives(node.body, directive_dict)
885 return self.visit_Node(node)
887 class WithTransform(CythonTransform, SkipDeclarations):
889 # EXCINFO is manually set to a variable that contains
890 # the exc_info() tuple that can be generated by the enclosing except
892 template_without_target = TreeFragment(u"""
903 if not EXIT(*EXCINFO):
907 EXIT(None, None, None)
908 """, temps=[u'MGR', u'EXC', u"EXIT"],
909 pipeline=[NormalizeTree(None)])
911 template_with_target = TreeFragment(u"""
914 VALUE = MGR.__enter__()
923 if not EXIT(*EXCINFO):
927 EXIT(None, None, None)
928 MGR = EXIT = VALUE = EXC = None
930 """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
931 pipeline=[NormalizeTree(None)])
933 def visit_WithStatNode(self, node):
934 # TODO: Cleanup badly needed
935 TemplateTransform.temp_name_counter += 1
936 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
938 self.visitchildren(node, ['body'])
939 excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
940 if node.target is not None:
941 result = self.template_with_target.substitute({
942 u'EXPR' : node.manager,
944 u'TARGET' : node.target,
945 u'EXCINFO' : excinfo_temp
948 result = self.template_without_target.substitute({
949 u'EXPR' : node.manager,
951 u'EXCINFO' : excinfo_temp
954 # Set except excinfo target to EXCINFO
955 try_except = result.stats[-1].body.stats[-1]
956 try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
957 # excinfo_temp.ref(node.pos))
959 # result.stats[-1].body.stats[-1] = TempsBlockNode(
960 # node.pos, temps=[excinfo_temp], body=try_except)
964 def visit_ExprNode(self, node):
965 # With statements are never inside expressions.
969 class DecoratorTransform(CythonTransform, SkipDeclarations):
971 def visit_DefNode(self, func_node):
972 self.visitchildren(func_node)
973 if not func_node.decorators:
975 return self._handle_decorators(
976 func_node, func_node.name)
978 def visit_CClassDefNode(self, class_node):
979 # This doesn't currently work, so it's disabled.
981 # Problem: assignments to cdef class names do not work. They
982 # would require an additional check anyway, as the extension
983 # type must not change its C type, so decorators cannot
984 # replace an extension type, just alter it and return it.
986 self.visitchildren(class_node)
987 if not class_node.decorators:
989 error(class_node.pos,
990 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
992 #return self._handle_decorators(
993 # class_node, class_node.class_name)
995 def visit_ClassDefNode(self, class_node):
996 self.visitchildren(class_node)
997 if not class_node.decorators:
999 return self._handle_decorators(
1000 class_node, class_node.name)
1002 def _handle_decorators(self, node, name):
1003 decorator_result = ExprNodes.NameNode(node.pos, name = name)
1004 for decorator in node.decorators[::-1]:
1005 decorator_result = ExprNodes.SimpleCallNode(
1007 function = decorator.decorator,
1008 args = [decorator_result])
1010 name_node = ExprNodes.NameNode(node.pos, name = name)
1011 reassignment = Nodes.SingleAssignmentNode(
1014 rhs = decorator_result)
1015 return [node, reassignment]
1018 class AnalyseDeclarationsTransform(CythonTransform):
1020 basic_property = TreeFragment(u"""
1024 def __set__(self, value):
1026 """, level='c_class')
1027 basic_pyobject_property = TreeFragment(u"""
1031 def __set__(self, value):
1035 """, level='c_class')
1036 basic_property_ro = TreeFragment(u"""
1040 """, level='c_class')
1042 struct_or_union_wrapper = TreeFragment(u"""
1045 def __init__(self, MEMBER=None):
1049 if IS_UNION and count > 1:
1050 raise ValueError, "At most one union member should be specified."
1052 return STR_FORMAT % MEMBER_TUPLE
1054 return REPR_FORMAT % MEMBER_TUPLE
1057 init_assignment = TreeFragment(u"""
1058 if VALUE is not None:
1063 def __call__(self, root):
1064 self.env_stack = [root.scope]
1065 # needed to determine if a cdef var is declared after it's used.
1066 self.seen_vars_stack = []
1067 return super(AnalyseDeclarationsTransform, self).__call__(root)
1069 def visit_NameNode(self, node):
1070 self.seen_vars_stack[-1].add(node.name)
1073 def visit_ModuleNode(self, node):
1074 self.seen_vars_stack.append(cython.set())
1075 node.analyse_declarations(self.env_stack[-1])
1076 self.visitchildren(node)
1077 self.seen_vars_stack.pop()
1080 def visit_LambdaNode(self, node):
1081 node.analyse_declarations(self.env_stack[-1])
1082 self.visitchildren(node)
1085 def visit_ClassDefNode(self, node):
1086 self.env_stack.append(node.scope)
1087 self.visitchildren(node)
1088 self.env_stack.pop()
1091 def visit_CClassDefNode(self, node):
1092 node = self.visit_ClassDefNode(node)
1093 if node.scope and node.scope.implemented:
1095 for entry in node.scope.var_entries:
1096 if entry.needs_property:
1097 property = self.create_Property(entry)
1098 property.analyse_declarations(node.scope)
1099 self.visit(property)
1100 stats.append(property)
1102 node.body.stats += stats
1105 def visit_FuncDefNode(self, node):
1106 self.seen_vars_stack.append(cython.set())
1107 lenv = node.local_scope
1108 node.body.analyse_control_flow(lenv) # this will be totally refactored
1109 node.declare_arguments(lenv)
1110 for var, type_node in node.directive_locals.items():
1111 if not lenv.lookup_here(var): # don't redeclare args
1112 type = type_node.analyse_as_type(lenv)
1114 lenv.declare_var(var, type, type_node.pos)
1116 error(type_node.pos, "Not a type")
1117 node.body.analyse_declarations(lenv)
1118 self.env_stack.append(lenv)
1119 self.visitchildren(node)
1120 self.env_stack.pop()
1121 self.seen_vars_stack.pop()
1124 def visit_ScopedExprNode(self, node):
1125 env = self.env_stack[-1]
1126 node.analyse_declarations(env)
1127 # the node may or may not have a local scope
1128 if node.has_local_scope:
1129 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1130 self.env_stack.append(node.expr_scope)
1131 node.analyse_scoped_declarations(node.expr_scope)
1132 self.visitchildren(node)
1133 self.env_stack.pop()
1134 self.seen_vars_stack.pop()
1136 node.analyse_scoped_declarations(env)
1137 self.visitchildren(node)
1140 def visit_TempResultFromStatNode(self, node):
1141 self.visitchildren(node)
1142 node.analyse_declarations(self.env_stack[-1])
1145 def visit_CStructOrUnionDefNode(self, node):
1146 # Create a wrapper node if needed.
1147 # We want to use the struct type information (so it can't happen
1148 # before this phase) but also create new objects to be declared
1149 # (so it can't happen later).
1150 # Note that we don't return the original node, as it is
1151 # never used after this phase.
1152 if True: # private (default)
1155 self_value = ExprNodes.AttributeNode(
1157 obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1158 attribute = EncodedString(u"value"))
1159 var_entries = node.entry.type.scope.var_entries
1161 for entry in var_entries:
1162 attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1164 attribute = entry.name))
1165 # __init__ assignments
1166 init_assignments = []
1167 for entry, attr in zip(var_entries, attributes):
1168 # TODO: branch on visibility
1169 init_assignments.append(self.init_assignment.substitute({
1170 u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1172 }, pos = entry.pos))
1175 str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1176 wrapper_class = self.struct_or_union_wrapper.substitute({
1177 u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1178 u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1179 u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1180 u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1181 u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1182 }, pos = node.pos).stats[0]
1183 wrapper_class.class_name = node.name
1184 wrapper_class.shadow = True
1185 class_body = wrapper_class.body.stats
1188 assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1189 class_body[0].base_type.name = node.name
1191 # fix __init__ arguments
1192 init_method = class_body[1]
1193 assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1194 arg_template = init_method.args[1]
1195 if not node.entry.type.is_struct:
1196 arg_template.kw_only = True
1197 del init_method.args[1]
1198 for entry, attr in zip(var_entries, attributes):
1199 arg = copy.deepcopy(arg_template)
1200 arg.declarator.name = entry.name
1201 init_method.args.append(arg)
1204 for entry, attr in zip(var_entries, attributes):
1205 # TODO: branch on visibility
1206 if entry.type.is_pyobject:
1207 template = self.basic_pyobject_property
1209 template = self.basic_property
1210 property = template.substitute({
1212 }, pos = entry.pos).stats[0]
1213 property.name = entry.name
1214 wrapper_class.body.stats.append(property)
1216 wrapper_class.analyse_declarations(self.env_stack[-1])
1217 return self.visit_CClassDefNode(wrapper_class)
1219 # Some nodes are no longer needed after declaration
1220 # analysis and can be dropped. The analysis was performed
1221 # on these nodes in a seperate recursive process from the
1222 # enclosing function or module, so we can simply drop them.
1223 def visit_CDeclaratorNode(self, node):
1224 # necessary to ensure that all CNameDeclaratorNodes are visited.
1225 self.visitchildren(node)
1228 def visit_CTypeDefNode(self, node):
1231 def visit_CBaseTypeNode(self, node):
1234 def visit_CEnumDefNode(self, node):
1235 if node.visibility == 'public':
1240 def visit_CNameDeclaratorNode(self, node):
1241 if node.name in self.seen_vars_stack[-1]:
1242 entry = self.env_stack[-1].lookup(node.name)
1243 if entry is None or entry.visibility != 'extern':
1244 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1245 self.visitchildren(node)
1248 def visit_CVarDefNode(self, node):
1249 # to ensure all CNameDeclaratorNodes are visited.
1250 self.visitchildren(node)
1253 def create_Property(self, entry):
1254 if entry.visibility == 'public':
1255 if entry.type.is_pyobject:
1256 template = self.basic_pyobject_property
1258 template = self.basic_property
1259 elif entry.visibility == 'readonly':
1260 template = self.basic_property_ro
1261 property = template.substitute({
1262 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1263 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1264 attribute=entry.name),
1265 }, pos=entry.pos).stats[0]
1266 property.name = entry.name
1267 # ---------------------------------------
1268 # XXX This should go to AutoDocTransforms
1269 # ---------------------------------------
1270 if (Options.docstrings and
1271 self.current_directives['embedsignature']):
1272 attr_name = entry.name
1273 type_name = entry.type.declaration_code("", for_display=1)
1275 if not entry.type.is_pyobject:
1276 type_name = "'%s'" % type_name
1277 elif entry.type.is_extension_type:
1278 type_name = entry.type.module_name + '.' + type_name
1279 if entry.init is not None:
1280 default_value = ' = ' + entry.init
1281 elif entry.init_to_none:
1282 default_value = ' = ' + repr(None)
1283 docstring = attr_name + ': ' + type_name + default_value
1284 property.doc = EncodedString(docstring)
1285 # ---------------------------------------
1288 class AnalyseExpressionsTransform(CythonTransform):
1290 def visit_ModuleNode(self, node):
1291 node.scope.infer_types()
1292 node.body.analyse_expressions(node.scope)
1293 self.visitchildren(node)
1296 def visit_FuncDefNode(self, node):
1297 node.local_scope.infer_types()
1298 node.body.analyse_expressions(node.local_scope)
1299 self.visitchildren(node)
1302 def visit_ScopedExprNode(self, node):
1303 if node.has_local_scope:
1304 node.expr_scope.infer_types()
1305 node.analyse_scoped_expressions(node.expr_scope)
1306 self.visitchildren(node)
1309 class ExpandInplaceOperators(EnvTransform):
1311 def visit_InPlaceAssignmentNode(self, node):
1314 if lhs.type.is_cpp_class:
1315 # No getting around this exact operator here.
1317 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1318 # There is code to handle this case.
1321 env = self.current_env()
1322 def side_effect_free_reference(node, setting=False):
1323 if isinstance(node, ExprNodes.NameNode):
1325 elif node.type.is_pyobject and not setting:
1326 node = LetRefNode(node)
1328 elif isinstance(node, ExprNodes.IndexNode):
1329 if node.is_buffer_access:
1330 raise ValueError, "Buffer access"
1331 base, temps = side_effect_free_reference(node.base)
1332 index = LetRefNode(node.index)
1333 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1334 elif isinstance(node, ExprNodes.AttributeNode):
1335 obj, temps = side_effect_free_reference(node.obj)
1336 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1338 node = LetRefNode(node)
1341 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1344 dup = lhs.__class__(**lhs.__dict__)
1345 binop = ExprNodes.binop_node(node.pos,
1346 operator = node.operator,
1350 # Manually analyse types for new node.
1351 lhs.analyse_target_types(env)
1352 dup.analyse_types(env)
1353 binop.analyse_operation(env)
1354 node = Nodes.SingleAssignmentNode(
1357 rhs=binop.coerce_to(lhs.type, env))
1358 # Use LetRefNode to avoid side effects.
1359 let_ref_nodes.reverse()
1360 for t in let_ref_nodes:
1361 node = LetNode(t, node)
1364 def visit_ExprNode(self, node):
1365 # In-place assignments can't happen within an expression.
1369 class AlignFunctionDefinitions(CythonTransform):
1371 This class takes the signatures from a .pxd file and applies them to
1372 the def methods in a .py file.
1375 def visit_ModuleNode(self, node):
1376 self.scope = node.scope
1377 self.directives = node.directives
1378 self.visitchildren(node)
1381 def visit_PyClassDefNode(self, node):
1382 pxd_def = self.scope.lookup(node.name)
1384 if pxd_def.is_cclass:
1385 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1387 error(node.pos, "'%s' redeclared" % node.name)
1388 error(pxd_def.pos, "previous declaration here")
1393 def visit_CClassDefNode(self, node, pxd_def=None):
1395 pxd_def = self.scope.lookup(node.class_name)
1397 outer_scope = self.scope
1398 self.scope = pxd_def.type.scope
1399 self.visitchildren(node)
1401 self.scope = outer_scope
1404 def visit_DefNode(self, node):
1405 pxd_def = self.scope.lookup(node.name)
1407 if not pxd_def.is_cfunction:
1408 error(node.pos, "'%s' redeclared" % node.name)
1409 error(pxd_def.pos, "previous declaration here")
1411 node = node.as_cfunction(pxd_def)
1412 elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1413 node = node.as_cfunction(scope=self.scope)
1414 # Enable this when internal def functions are allowed.
1415 # self.visitchildren(node)
1419 class YieldNodeCollector(TreeVisitor):
1422 super(YieldNodeCollector, self).__init__()
1425 self.has_return_value = False
1427 def visit_Node(self, node):
1428 return self.visitchildren(node)
1430 def visit_YieldExprNode(self, node):
1431 if self.has_return_value:
1432 error(node.pos, "'yield' outside function")
1433 self.yields.append(node)
1434 self.visitchildren(node)
1436 def visit_ReturnStatNode(self, node):
1438 self.has_return_value = True
1440 error(node.pos, "'return' with argument inside generator")
1441 self.returns.append(node)
1443 def visit_ClassDefNode(self, node):
1446 def visit_DefNode(self, node):
1449 def visit_LambdaNode(self, node):
1452 def visit_GeneratorExpressionNode(self, node):
1455 class MarkClosureVisitor(CythonTransform):
1457 def visit_ModuleNode(self, node):
1458 self.needs_closure = False
1459 self.visitchildren(node)
1462 def visit_FuncDefNode(self, node):
1463 self.needs_closure = False
1464 self.visitchildren(node)
1465 node.needs_closure = self.needs_closure
1466 self.needs_closure = True
1468 collector = YieldNodeCollector()
1469 collector.visitchildren(node)
1471 if collector.yields:
1472 for i, yield_expr in enumerate(collector.yields):
1473 yield_expr.label_num = i + 1
1475 gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1478 generator = Nodes.GeneratorDefNode(pos=node.pos,
1481 star_arg=node.star_arg,
1482 starstar_arg=node.starstar_arg,
1484 decorators=node.decorators,
1486 lambda_name=node.lambda_name)
1490 def visit_CFuncDefNode(self, node):
1491 self.visit_FuncDefNode(node)
1492 if node.needs_closure:
1493 error(node.pos, "closures inside cdef functions not yet supported")
1496 def visit_LambdaNode(self, node):
1497 self.needs_closure = False
1498 self.visitchildren(node)
1499 node.needs_closure = self.needs_closure
1500 self.needs_closure = True
1503 def visit_ClassDefNode(self, node):
1504 self.visitchildren(node)
1505 self.needs_closure = True
1508 class CreateClosureClasses(CythonTransform):
1509 # Output closure classes in module scope for all functions
1510 # that really need it.
1512 def __init__(self, context):
1513 super(CreateClosureClasses, self).__init__(context)
1515 self.in_lambda = False
1516 self.generator_class = None
1518 def visit_ModuleNode(self, node):
1519 self.module_scope = node.scope
1520 self.visitchildren(node)
1523 def create_generator_class(self, target_module_scope, pos):
1524 if self.generator_class:
1525 return self.generator_class
1526 # XXX: make generator class creation cleaner
1527 entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1528 objstruct_cname='__pyx_Generator_object',
1529 typeobj_cname='__pyx_Generator_type',
1530 pos=pos, defining=True, implementing=True)
1531 klass = entry.type.scope
1532 klass.is_internal = True
1533 klass.directives = {'final': True}
1535 body_type = PyrexTypes.create_typedef_type('generator_body',
1536 PyrexTypes.c_void_ptr_type,
1537 '__pyx_generator_body_t')
1538 klass.declare_var(pos=pos, name='body', cname='body',
1539 type=body_type, is_cdef=True)
1540 klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1542 klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1546 e = klass.declare_pyfunction('send', pos)
1547 e.func_cname = '__Pyx_Generator_Send'
1548 e.signature = TypeSlots.binaryfunc
1550 e = klass.declare_pyfunction('close', pos)
1551 e.func_cname = '__Pyx_Generator_Close'
1552 e.signature = TypeSlots.unaryfunc
1554 e = klass.declare_pyfunction('throw', pos)
1555 e.func_cname = '__Pyx_Generator_Throw'
1556 e.signature = TypeSlots.pyfunction_signature
1558 e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1559 e.func_cname = 'PyObject_SelfIter'
1561 e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1562 e.func_cname = '__Pyx_Generator_Next'
1564 self.generator_class = entry.type
1565 return self.generator_class
1567 def get_scope_use(self, node):
1570 for name, entry in node.local_scope.entries.items():
1571 if entry.from_closure:
1572 from_closure.append((name, entry))
1573 elif entry.in_closure and not entry.from_closure:
1574 in_closure.append((name, entry))
1575 return from_closure, in_closure
1577 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1578 # skip generator body
1579 if node.is_generator_body:
1581 # move local variables into closure
1582 if node.is_generator:
1583 for entry in node.local_scope.entries.values():
1584 if not entry.from_closure:
1585 entry.in_closure = True
1587 from_closure, in_closure = self.get_scope_use(node)
1590 # Now from the begining
1591 node.needs_closure = False
1592 node.needs_outer_scope = False
1594 func_scope = node.local_scope
1595 cscope = node.entry.scope
1596 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1597 cscope = cscope.outer_scope
1599 if not from_closure and (self.path or inner_node):
1602 raise InternalError, "DefNode does not have assignment node"
1603 inner_node = node.assmt.rhs
1604 inner_node.needs_self_code = False
1605 node.needs_outer_scope = False
1607 if node.is_generator:
1608 generator_class = self.create_generator_class(target_module_scope, node.pos)
1609 elif not in_closure and not from_closure:
1611 elif not in_closure:
1612 func_scope.is_passthrough = True
1613 func_scope.scope_class = cscope.scope_class
1614 node.needs_outer_scope = True
1617 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1619 if node.is_generator:
1620 entry = target_module_scope.declare_c_class(name = as_name,
1621 pos = node.pos, defining = True, implementing = True, base_type=generator_class)
1623 entry = target_module_scope.declare_c_class(name = as_name,
1624 pos = node.pos, defining = True, implementing = True)
1625 func_scope.scope_class = entry
1626 class_scope = entry.type.scope
1627 class_scope.is_internal = True
1628 class_scope.directives = {'final': True}
1631 assert cscope.is_closure_scope
1632 class_scope.declare_var(pos=node.pos,
1633 name=Naming.outer_scope_cname,
1634 cname=Naming.outer_scope_cname,
1635 type=cscope.scope_class.type,
1637 node.needs_outer_scope = True
1638 for name, entry in in_closure:
1639 class_scope.declare_var(pos=entry.pos,
1644 node.needs_closure = True
1645 # Do it here because other classes are already checked
1646 target_module_scope.check_c_class(func_scope.scope_class)
1648 def visit_LambdaNode(self, node):
1649 was_in_lambda = self.in_lambda
1650 self.in_lambda = True
1651 self.create_class_from_scope(node.def_node, self.module_scope, node)
1652 self.visitchildren(node)
1653 self.in_lambda = was_in_lambda
1656 def visit_FuncDefNode(self, node):
1658 self.visitchildren(node)
1660 if node.needs_closure or self.path:
1661 self.create_class_from_scope(node, self.module_scope)
1662 self.path.append(node)
1663 self.visitchildren(node)
1668 class GilCheck(VisitorTransform):
1670 Call `node.gil_check(env)` on each node to make sure we hold the
1671 GIL when we need it. Raise an error when on Python operations
1672 inside a `nogil` environment.
1674 def __call__(self, root):
1675 self.env_stack = [root.scope]
1677 return super(GilCheck, self).__call__(root)
1679 def visit_FuncDefNode(self, node):
1680 self.env_stack.append(node.local_scope)
1681 was_nogil = self.nogil
1682 self.nogil = node.local_scope.nogil
1683 if self.nogil and node.nogil_check:
1684 node.nogil_check(node.local_scope)
1685 self.visitchildren(node)
1686 self.env_stack.pop()
1687 self.nogil = was_nogil
1690 def visit_GILStatNode(self, node):
1691 env = self.env_stack[-1]
1692 if self.nogil and node.nogil_check: node.nogil_check()
1693 was_nogil = self.nogil
1694 self.nogil = (node.state == 'nogil')
1695 self.visitchildren(node)
1696 self.nogil = was_nogil
1699 def visit_Node(self, node):
1700 if self.env_stack and self.nogil and node.nogil_check:
1701 node.nogil_check(self.env_stack[-1])
1702 self.visitchildren(node)
1706 class TransformBuiltinMethods(EnvTransform):
1708 def visit_SingleAssignmentNode(self, node):
1709 if node.declaration_only:
1712 self.visitchildren(node)
1715 def visit_AttributeNode(self, node):
1716 self.visitchildren(node)
1717 return self.visit_cython_attribute(node)
1719 def visit_NameNode(self, node):
1720 return self.visit_cython_attribute(node)
1722 def visit_cython_attribute(self, node):
1723 attribute = node.as_cython_attribute()
1725 if attribute == u'compiled':
1726 node = ExprNodes.BoolNode(node.pos, value=True)
1727 elif attribute == u'NULL':
1728 node = ExprNodes.NullNode(node.pos)
1729 elif attribute in (u'set', u'frozenset'):
1730 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1731 entry=self.current_env().builtin_scope().lookup_here(attribute))
1732 elif not PyrexTypes.parse_basic_type(attribute):
1733 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1736 def visit_SimpleCallNode(self, node):
1739 if isinstance(node.function, ExprNodes.NameNode):
1740 if node.function.name == 'locals':
1741 lenv = self.current_env()
1742 entry = lenv.lookup_here('locals')
1744 # not the builtin 'locals'
1746 if len(node.args) > 0:
1747 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1750 items = [ ExprNodes.DictItemNode(pos,
1751 key=ExprNodes.StringNode(pos, value=var),
1752 value=ExprNodes.NameNode(pos, name=var))
1753 for var in lenv.entries ]
1754 return ExprNodes.DictNode(pos, key_value_pairs=items)
1757 function = node.function.as_cython_attribute()
1759 if function in InterpretCompilerDirectives.unop_method_nodes:
1760 if len(node.args) != 1:
1761 error(node.function.pos, u"%s() takes exactly one argument" % function)
1763 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1764 elif function in InterpretCompilerDirectives.binop_method_nodes:
1765 if len(node.args) != 2:
1766 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1768 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1769 elif function == u'cast':
1770 if len(node.args) != 2:
1771 error(node.function.pos, u"cast() takes exactly two arguments")
1773 type = node.args[0].analyse_as_type(self.current_env())
1775 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1777 error(node.args[0].pos, "Not a type")
1778 elif function == u'sizeof':
1779 if len(node.args) != 1:
1780 error(node.function.pos, u"sizeof() takes exactly one argument")
1782 type = node.args[0].analyse_as_type(self.current_env())
1784 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1786 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1787 elif function == 'cmod':
1788 if len(node.args) != 2:
1789 error(node.function.pos, u"cmod() takes exactly two arguments")
1791 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1792 node.cdivision = True
1793 elif function == 'cdiv':
1794 if len(node.args) != 2:
1795 error(node.function.pos, u"cdiv() takes exactly two arguments")
1797 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1798 node.cdivision = True
1799 elif function == u'set':
1800 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1802 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1804 self.visitchildren(node)
1808 class DebugTransform(CythonTransform):
1810 Create debug information and all functions' visibility to extern in order
1811 to enable debugging.
1814 def __init__(self, context, options, result):
1815 super(DebugTransform, self).__init__(context)
1816 self.visited = cython.set()
1817 # our treebuilder and debug output writer
1818 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1819 self.tb = self.context.gdb_debug_outputwriter
1820 #self.c_output_file = options.output_file
1821 self.c_output_file = result.c_file
1823 # Closure support, basically treat nested functions as if the AST were
1825 self.nested_funcdefs = []
1827 # tells visit_NameNode whether it should register step-into functions
1828 self.register_stepinto = False
1830 def visit_ModuleNode(self, node):
1831 self.tb.module_name = node.full_module_name
1833 module_name=node.full_module_name,
1834 filename=node.pos[0].filename,
1835 c_filename=self.c_output_file)
1837 self.tb.start('Module', attrs)
1839 # serialize functions
1840 self.tb.start('Functions')
1841 # First, serialize functions normally...
1842 self.visitchildren(node)
1844 # ... then, serialize nested functions
1845 for nested_funcdef in self.nested_funcdefs:
1846 self.visit_FuncDefNode(nested_funcdef)
1848 self.register_stepinto = True
1849 self.serialize_modulenode_as_function(node)
1850 self.register_stepinto = False
1851 self.tb.end('Functions')
1853 # 2.3 compatibility. Serialize global variables
1854 self.tb.start('Globals')
1857 for k, v in node.scope.entries.iteritems():
1858 if (v.qualified_name not in self.visited and not
1859 v.name.startswith('__pyx_') and not
1860 v.type.is_cfunction and not
1861 v.type.is_extension_type):
1864 self.serialize_local_variables(entries)
1865 self.tb.end('Globals')
1866 # self.tb.end('Module') # end Module after the line number mapping in
1867 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1870 def visit_FuncDefNode(self, node):
1871 self.visited.add(node.local_scope.qualified_name)
1873 if getattr(node, 'is_wrapper', False):
1876 if self.register_stepinto:
1877 self.nested_funcdefs.append(node)
1880 # node.entry.visibility = 'extern'
1881 if node.py_func is None:
1884 pf_cname = node.py_func.entry.func_cname
1887 name=node.entry.name,
1888 cname=node.entry.func_cname,
1890 qualified_name=node.local_scope.qualified_name,
1891 lineno=str(node.pos[1]))
1893 self.tb.start('Function', attrs=attrs)
1895 self.tb.start('Locals')
1896 self.serialize_local_variables(node.local_scope.entries)
1897 self.tb.end('Locals')
1899 self.tb.start('Arguments')
1900 for arg in node.local_scope.arg_entries:
1901 self.tb.start(arg.name)
1902 self.tb.end(arg.name)
1903 self.tb.end('Arguments')
1905 self.tb.start('StepIntoFunctions')
1906 self.register_stepinto = True
1907 self.visitchildren(node)
1908 self.register_stepinto = False
1909 self.tb.end('StepIntoFunctions')
1910 self.tb.end('Function')
1914 def visit_NameNode(self, node):
1915 if (self.register_stepinto and
1916 node.type.is_cfunction and
1917 getattr(node, 'is_called', False) and
1918 node.entry.func_cname is not None):
1919 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1920 # declared functions are not 'in_cinclude'.
1921 # This means we will list called 'cdef' functions as
1922 # "step into functions", but this is not an issue as they will be
1923 # recognized as Cython functions anyway.
1924 attrs = dict(name=node.entry.func_cname)
1925 self.tb.start('StepIntoFunction', attrs=attrs)
1926 self.tb.end('StepIntoFunction')
1928 self.visitchildren(node)
1931 def serialize_modulenode_as_function(self, node):
1933 Serialize the module-level code as a function so the debugger will know
1934 it's a "relevant frame" and it will know where to set the breakpoint
1935 for 'break modulename'.
1937 name = node.full_module_name.rpartition('.')[-1]
1939 cname_py2 = 'init' + name
1940 cname_py3 = 'PyInit_' + name
1946 # Ignore the qualified_name, breakpoints should be set using
1947 # `cy break modulename:lineno` for module-level breakpoints.
1950 is_initmodule_function="True",
1953 py3_attrs = dict(py2_attrs, cname=cname_py3)
1955 self._serialize_modulenode_as_function(node, py2_attrs)
1956 self._serialize_modulenode_as_function(node, py3_attrs)
1958 def _serialize_modulenode_as_function(self, node, attrs):
1959 self.tb.start('Function', attrs=attrs)
1961 self.tb.start('Locals')
1962 self.serialize_local_variables(node.scope.entries)
1963 self.tb.end('Locals')
1965 self.tb.start('Arguments')
1966 self.tb.end('Arguments')
1968 self.tb.start('StepIntoFunctions')
1969 self.register_stepinto = True
1970 self.visitchildren(node)
1971 self.register_stepinto = False
1972 self.tb.end('StepIntoFunctions')
1974 self.tb.end('Function')
1976 def serialize_local_variables(self, entries):
1977 for entry in entries.values():
1978 if entry.type.is_pyobject:
1979 vartype = 'PythonObject'
1983 if entry.from_closure:
1984 # We're dealing with a closure where a variable from an outer
1985 # scope is accessed, get it from the scope object.
1986 cname = '%s->%s' % (Naming.cur_scope_cname,
1987 entry.outer_entry.cname)
1989 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1992 elif entry.in_closure:
1993 cname = '%s->%s' % (Naming.cur_scope_cname,
1995 qname = entry.qualified_name
1998 qname = entry.qualified_name
2001 # this happens for variables that are not in the user's code,
2002 # e.g. for the global __builtins__, __doc__, etc. We can just
2003 # set the lineno to 0 for those.
2006 lineno = str(entry.pos[1])
2011 qualified_name=qname,
2015 self.tb.start('LocalVar', attrs)
2016 self.tb.end('LocalVar')