3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
4 Options=object, UtilNodes=object, ModuleNode=object,
5 LetNode=object, LetRefNode=object, TreeFragment=object,
6 TemplateTransform=object, EncodedString=object,
7 error=object, warning=object, copy=object)
15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
17 from Cython.Compiler.ModuleNode import ModuleNode
18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
20 from Cython.Compiler.StringEncoding import EncodedString
21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
26 class NameNodeCollector(TreeVisitor):
27 """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
31 super(NameNodeCollector, self).__init__()
34 def visit_NameNode(self, node):
35 self.name_nodes.append(node)
37 def visit_Node(self, node):
38 self._visitchildren(node, None)
41 class SkipDeclarations(object):
43 Variable and function declarations can often have a deep tree structure,
44 and yet most transformations don't need to descend to this depth.
46 Declaration nodes are removed after AnalyseDeclarationsTransform, so there
47 is no need to use this for transformations after that point.
49 def visit_CTypeDefNode(self, node):
52 def visit_CVarDefNode(self, node):
55 def visit_CDeclaratorNode(self, node):
58 def visit_CBaseTypeNode(self, node):
61 def visit_CEnumDefNode(self, node):
64 def visit_CStructOrUnionDefNode(self, node):
68 class NormalizeTree(CythonTransform):
70 This transform fixes up a few things after parsing
71 in order to make the parse tree more suitable for
74 a) After parsing, blocks with only one statement will
75 be represented by that statement, not by a StatListNode.
76 When doing transforms this is annoying and inconsistent,
77 as one cannot in general remove a statement in a consistent
78 way and so on. This transform wraps any single statements
79 in a StatListNode containing a single statement.
81 b) The PassStatNode is a noop and serves no purpose beyond
82 plugging such one-statement blocks; i.e., once parsed a
83 ` "pass" can just as well be represented using an empty
84 StatListNode. This means less special cases to worry about
85 in subsequent transforms (one always checks to see if a
86 StatListNode has no children to see if the block is empty).
89 def __init__(self, context):
90 super(NormalizeTree, self).__init__(context)
91 self.is_in_statlist = False
92 self.is_in_expr = False
94 def visit_ExprNode(self, node):
95 stacktmp = self.is_in_expr
96 self.is_in_expr = True
97 self.visitchildren(node)
98 self.is_in_expr = stacktmp
101 def visit_StatNode(self, node, is_listcontainer=False):
102 stacktmp = self.is_in_statlist
103 self.is_in_statlist = is_listcontainer
104 self.visitchildren(node)
105 self.is_in_statlist = stacktmp
106 if not self.is_in_statlist and not self.is_in_expr:
107 return Nodes.StatListNode(pos=node.pos, stats=[node])
111 def visit_StatListNode(self, node):
112 self.is_in_statlist = True
113 self.visitchildren(node)
114 self.is_in_statlist = False
117 def visit_ParallelAssignmentNode(self, node):
118 return self.visit_StatNode(node, True)
120 def visit_CEnumDefNode(self, node):
121 return self.visit_StatNode(node, True)
123 def visit_CStructOrUnionDefNode(self, node):
124 return self.visit_StatNode(node, True)
126 # Eliminate PassStatNode
127 def visit_PassStatNode(self, node):
128 if not self.is_in_statlist:
129 return Nodes.StatListNode(pos=node.pos, stats=[])
133 def visit_CDeclaratorNode(self, node):
137 class PostParseError(CompileError): pass
139 # error strings checked by unit tests, so define them
140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
143 class PostParse(ScopeTrackingTransform):
145 Basic interpretation of the parse tree, as well as validity
146 checking that can be done on a very basic level on the parse
147 tree (while still not being a problem with the basic syntax,
151 - Default values to cdef assignments are turned into single
152 assignments following the declaration (everywhere but in class
153 bodies, where they raise a compile error)
155 - Interpret some node structures into Python runtime values.
156 Some nodes take compile-time arguments (currently:
157 TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
158 which should be interpreted. This happens in a general way
159 and other steps should be taken to ensure validity.
161 Type arguments cannot be interpreted in this way.
163 - For __cythonbufferdefaults__ the arguments are checked for
166 TemplatedTypeNode has its directives interpreted:
167 Any first positional argument goes into the "dtype" attribute,
168 any "ndim" keyword argument goes into the "ndim" attribute and
169 so on. Also it is checked that the directive combination is valid.
170 - __cythonbufferdefaults__ attributes are parsed and put into the
173 Note: Currently Parsing.py does a lot of interpretation and
174 reorganization that can be refactored into this transform
175 if a more pure Abstract Syntax Tree is wanted.
178 def __init__(self, context):
179 super(PostParse, self).__init__(context)
180 self.specialattribute_handlers = {
181 '__cythonbufferdefaults__' : self.handle_bufferdefaults
184 def visit_ModuleNode(self, node):
185 self.lambda_counter = 1
186 self.genexpr_counter = 1
187 return super(PostParse, self).visit_ModuleNode(node)
189 def visit_LambdaNode(self, node):
190 # unpack a lambda expression into the corresponding DefNode
191 lambda_id = self.lambda_counter
192 self.lambda_counter += 1
193 node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
194 collector = YieldNodeCollector()
195 collector.visitchildren(node.result_expr)
196 if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
197 body = ExprNodes.YieldExprNode(
198 node.result_expr.pos, arg=node.result_expr)
199 body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
201 body = Nodes.ReturnStatNode(
202 node.result_expr.pos, value=node.result_expr)
203 node.def_node = Nodes.DefNode(
204 node.pos, name=node.name, lambda_name=node.lambda_name,
205 args=node.args, star_arg=node.star_arg,
206 starstar_arg=node.starstar_arg,
208 self.visitchildren(node)
211 def visit_GeneratorExpressionNode(self, node):
212 # unpack a generator expression into the corresponding DefNode
213 genexpr_id = self.genexpr_counter
214 self.genexpr_counter += 1
215 node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
217 node.def_node = Nodes.DefNode(node.pos, name=node.name,
219 args=[], star_arg=None,
222 self.visitchildren(node)
226 def handle_bufferdefaults(self, decl):
227 if not isinstance(decl.default, ExprNodes.DictNode):
228 raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
229 self.scope_node.buffer_defaults_node = decl.default
230 self.scope_node.buffer_defaults_pos = decl.pos
232 def visit_CVarDefNode(self, node):
233 # This assumes only plain names and pointers are assignable on
234 # declaration. Also, it makes use of the fact that a cdef decl
235 # must appear before the first use, so we don't have to deal with
236 # "i = 3; cdef int i = i" and can simply move the nodes around.
238 self.visitchildren(node)
241 for decl in node.declarators:
243 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
244 declbase = declbase.base
245 if isinstance(declbase, Nodes.CNameDeclaratorNode):
246 if declbase.default is not None:
247 if self.scope_type in ('cclass', 'pyclass', 'struct'):
248 if isinstance(self.scope_node, Nodes.CClassDefNode):
249 handler = self.specialattribute_handlers.get(decl.name)
251 if decl is not declbase:
252 raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
254 continue # Remove declaration
255 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
256 first_assignment = self.scope_type != 'module'
257 stats.append(Nodes.SingleAssignmentNode(node.pos,
258 lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
259 rhs=declbase.default, first=first_assignment))
260 declbase.default = None
261 newdecls.append(decl)
262 node.declarators = newdecls
264 except PostParseError, e:
265 # An error in a cdef clause is ok, simply remove the declaration
266 # and try to move on to report more errors
267 self.context.nonfatal_error(e)
270 # Split parallel assignments (a,b = b,a) into separate partial
271 # assignments that are executed rhs-first using temps. This
272 # restructuring must be applied before type analysis so that known
273 # types on rhs and lhs can be matched directly. It is required in
274 # the case that the types cannot be coerced to a Python type in
275 # order to assign from a tuple.
277 def visit_SingleAssignmentNode(self, node):
278 self.visitchildren(node)
279 return self._visit_assignment_node(node, [node.lhs, node.rhs])
281 def visit_CascadedAssignmentNode(self, node):
282 self.visitchildren(node)
283 return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
285 def _visit_assignment_node(self, node, expr_list):
286 """Flatten parallel assignments into separate single
287 assignments or cascaded assignments.
289 if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
290 # no parallel assignments => nothing to do
294 flatten_parallel_assignments(expr_list, expr_list_list)
296 eliminate_rhs_duplicates(expr_list_list, temp_refs)
299 for expr_list in expr_list_list:
300 lhs_list = expr_list[:-1]
302 if len(lhs_list) == 1:
303 node = Nodes.SingleAssignmentNode(rhs.pos,
304 lhs = lhs_list[0], rhs = rhs)
306 node = Nodes.CascadedAssignmentNode(rhs.pos,
307 lhs_list = lhs_list, rhs = rhs)
311 assign_node = nodes[0]
313 assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
316 duplicates_and_temps = [ (temp.expression, temp)
317 for temp in temp_refs ]
318 sort_common_subsequences(duplicates_and_temps)
319 for _, temp_ref in duplicates_and_temps[::-1]:
320 assign_node = LetNode(temp_ref, assign_node)
324 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
325 """Replace rhs items by LetRefNodes if they appear more than once.
326 Creates a sequence of LetRefNodes that set up the required temps
327 and appends them to ref_node_sequence. The input list is modified
330 seen_nodes = cython.set()
332 def find_duplicates(node):
333 if node.is_literal or node.is_name:
334 # no need to replace those; can't include attributes here
335 # as their access is not necessarily side-effect free
337 if node in seen_nodes:
338 if node not in ref_nodes:
339 ref_node = LetRefNode(node)
340 ref_nodes[node] = ref_node
341 ref_node_sequence.append(ref_node)
344 if node.is_sequence_constructor:
345 for item in node.args:
346 find_duplicates(item)
348 for expr_list in expr_list_list:
354 def substitute_nodes(node):
355 if node in ref_nodes:
356 return ref_nodes[node]
357 elif node.is_sequence_constructor:
358 node.args = list(map(substitute_nodes, node.args))
361 # replace nodes inside of the common subexpressions
362 for node in ref_nodes:
363 if node.is_sequence_constructor:
364 node.args = list(map(substitute_nodes, node.args))
366 # replace common subexpressions on all rhs items
367 for expr_list in expr_list_list:
368 expr_list[-1] = substitute_nodes(expr_list[-1])
370 def sort_common_subsequences(items):
371 """Sort items/subsequences so that all items and subsequences that
372 an item contains appear before the item itself. This is needed
373 because each rhs item must only be evaluated once, so its value
374 must be evaluated first and then reused when packing sequences
377 This implies a partial order, and the sort must be stable to
378 preserve the original order as much as possible, so we use a
379 simple insertion sort (which is very fast for short sequences, the
380 normal case in practice).
382 def contains(seq, x):
386 elif item.is_sequence_constructor and contains(item.args, x):
390 return b.is_sequence_constructor and contains(b.args, a)
392 for pos, item in enumerate(items):
393 key = item[1] # the ResultRefNode which has already been injected into the sequences
395 for i in xrange(pos-1, -1, -1):
396 if lower_than(key, items[i][0]):
399 for i in xrange(pos, new_pos, -1):
400 items[i] = items[i-1]
401 items[new_pos] = item
403 def flatten_parallel_assignments(input, output):
404 # The input is a list of expression nodes, representing the LHSs
405 # and RHS of one (possibly cascaded) assignment statement. For
406 # sequence constructors, rearranges the matching parts of both
407 # sides into a list of equivalent assignments between the
408 # individual elements. This transformation is applied
409 # recursively, so that nested structures get matched as well.
411 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
415 complete_assignments = []
417 rhs_size = len(rhs.args)
418 lhs_targets = [ [] for _ in xrange(rhs_size) ]
419 starred_assignments = []
420 for lhs in input[:-1]:
421 if not lhs.is_sequence_constructor:
423 error(lhs.pos, "starred assignment target must be in a list or tuple")
424 complete_assignments.append(lhs)
426 lhs_size = len(lhs.args)
427 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
428 if starred_targets > 1:
429 error(lhs.pos, "more than 1 starred expression in assignment")
430 output.append([lhs,rhs])
432 elif lhs_size - starred_targets > rhs_size:
433 error(lhs.pos, "need more than %d value%s to unpack"
434 % (rhs_size, (rhs_size != 1) and 's' or ''))
435 output.append([lhs,rhs])
437 elif starred_targets:
438 map_starred_assignment(lhs_targets, starred_assignments,
440 elif lhs_size < rhs_size:
441 error(lhs.pos, "too many values to unpack (expected %d, got %d)"
442 % (lhs_size, rhs_size))
443 output.append([lhs,rhs])
446 for targets, expr in zip(lhs_targets, lhs.args):
449 if complete_assignments:
450 complete_assignments.append(rhs)
451 output.append(complete_assignments)
453 # recursively flatten partial assignments
454 for cascade, rhs in zip(lhs_targets, rhs.args):
457 flatten_parallel_assignments(cascade, output)
459 # recursively flatten starred assignments
460 for cascade in starred_assignments:
461 if cascade[0].is_sequence_constructor:
462 flatten_parallel_assignments(cascade, output)
464 output.append(cascade)
466 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
467 # Appends the fixed-position LHS targets to the target list that
468 # appear left and right of the starred argument.
470 # The starred_assignments list receives a new tuple
471 # (lhs_target, rhs_values_list) that maps the remaining arguments
472 # (those that match the starred target) to a list.
474 # left side of the starred target
475 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
478 lhs_remaining = len(lhs_args) - i - 1
482 raise InternalError("no starred arg found when splitting starred assignment")
484 # right side of the starred target
485 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
486 lhs_args[-lhs_remaining:])):
489 # the starred target itself, must be assigned a (potentially empty) list
490 target = lhs_args[starred].target # unpack starred node
491 starred_rhs = rhs_args[starred:]
493 starred_rhs = starred_rhs[:-lhs_remaining]
495 pos = starred_rhs[0].pos
498 starred_assignments.append([
499 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
502 class PxdPostParse(CythonTransform, SkipDeclarations):
504 Basic interpretation/validity checking that should only be
507 A lot of this checking currently happens in the parser; but
508 what is listed below happens here.
510 - "def" functions are let through only if they fill the
511 getbuffer/releasebuffer slots
513 - cdef functions are let through only if they are on the
514 top level and are declared "inline"
516 ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
517 ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
519 def __call__(self, node):
520 self.scope_type = 'pxd'
521 return super(PxdPostParse, self).__call__(node)
523 def visit_CClassDefNode(self, node):
524 old = self.scope_type
525 self.scope_type = 'cclass'
526 self.visitchildren(node)
527 self.scope_type = old
530 def visit_FuncDefNode(self, node):
531 # FuncDefNode always come with an implementation (without
532 # an imp they are CVarDefNodes..)
533 err = self.ERR_INLINE_ONLY
535 if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
536 and node.name in ('__getbuffer__', '__releasebuffer__')):
537 err = None # allow these slots
539 if isinstance(node, Nodes.CFuncDefNode):
540 if u'inline' in node.modifiers and self.scope_type == 'pxd':
541 node.inline_in_pxd = True
542 if node.visibility != 'private':
543 err = self.ERR_NOGO_WITH_INLINE % node.visibility
545 err = self.ERR_NOGO_WITH_INLINE % 'api'
547 err = None # allow inline function
549 err = self.ERR_INLINE_ONLY
552 self.context.nonfatal_error(PostParseError(node.pos, err))
557 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
559 After parsing, directives can be stored in a number of places:
560 - #cython-comments at the top of the file (stored in ModuleNode)
561 - Command-line arguments overriding these
562 - @cython.directivename decorators
563 - with cython.directivename: statements
565 This transform is responsible for interpreting these various sources
566 and store the directive in two ways:
567 - Set the directives attribute of the ModuleNode for global directives.
568 - Use a CompilerDirectivesNode to override directives for a subtree.
570 (The first one is primarily to not have to modify with the tree
571 structure, so that ModuleNode stay on top.)
573 The directives are stored in dictionaries from name to value in effect.
574 Each such dictionary is always filled in for all possible directives,
575 using default values where no value is given by the user.
577 The available directives are controlled in Options.py.
579 Note that we have to run this prior to analysis, and so some minor
580 duplication of functionality has to occur: We manually track cimports
581 and which names the "cython" module may have been imported to.
583 unop_method_nodes = {
584 'typeof': ExprNodes.TypeofNode,
586 'operator.address': ExprNodes.AmpersandNode,
587 'operator.dereference': ExprNodes.DereferenceNode,
588 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
589 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
590 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
591 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
593 # For backwards compatability.
594 'address': ExprNodes.AmpersandNode,
597 binop_method_nodes = {
598 'operator.comma' : ExprNodes.c_binop_constructor(','),
601 special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
602 'cast', 'pointer', 'compiled', 'NULL'])
603 special_methods.update(unop_method_nodes.keys())
605 def __init__(self, context, compilation_directive_defaults):
606 super(InterpretCompilerDirectives, self).__init__(context)
607 self.compilation_directive_defaults = {}
608 for key, value in compilation_directive_defaults.items():
609 self.compilation_directive_defaults[unicode(key)] = value
610 self.cython_module_names = cython.set()
611 self.directive_names = {}
613 def check_directive_scope(self, pos, directive, scope):
614 legal_scopes = Options.directive_scopes.get(directive, None)
615 if legal_scopes and scope not in legal_scopes:
616 self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
617 'is not allowed in %s scope' % (directive, scope)))
622 # Set up processing and handle the cython: comments.
623 def visit_ModuleNode(self, node):
624 for key, value in node.directive_comments.items():
625 if not self.check_directive_scope(node.pos, key, 'module'):
626 self.wrong_scope_error(node.pos, key, 'module')
627 del node.directive_comments[key]
629 directives = copy.copy(Options.directive_defaults)
630 directives.update(self.compilation_directive_defaults)
631 directives.update(node.directive_comments)
632 self.directives = directives
633 node.directives = directives
634 self.visitchildren(node)
635 node.cython_module_names = self.cython_module_names
638 # The following four functions track imports and cimports that
639 # begin with "cython"
640 def is_cython_directive(self, name):
641 return (name in Options.directive_types or
642 name in self.special_methods or
643 PyrexTypes.parse_basic_type(name))
645 def visit_CImportStatNode(self, node):
646 if node.module_name == u"cython":
647 self.cython_module_names.add(node.as_name or u"cython")
648 elif node.module_name.startswith(u"cython."):
650 self.directive_names[node.as_name] = node.module_name[7:]
652 self.cython_module_names.add(u"cython")
653 # if this cimport was a compiler directive, we don't
654 # want to leave the cimport node sitting in the tree
658 def visit_FromCImportStatNode(self, node):
659 if (node.module_name == u"cython") or \
660 node.module_name.startswith(u"cython."):
661 submodule = (node.module_name + u".")[7:]
663 for pos, name, as_name, kind in node.imported_names:
664 full_name = submodule + name
665 if self.is_cython_directive(full_name):
668 self.directive_names[as_name] = full_name
670 self.context.nonfatal_error(PostParseError(pos,
671 "Compiler directive imports must be plain imports"))
673 newimp.append((pos, name, as_name, kind))
676 node.imported_names = newimp
679 def visit_FromImportStatNode(self, node):
680 if (node.module.module_name.value == u"cython") or \
681 node.module.module_name.value.startswith(u"cython."):
682 submodule = (node.module.module_name.value + u".")[7:]
684 for name, name_node in node.items:
685 full_name = submodule + name
686 if self.is_cython_directive(full_name):
687 self.directive_names[name_node.name] = full_name
689 newimp.append((name, name_node))
695 def visit_SingleAssignmentNode(self, node):
696 if (isinstance(node.rhs, ExprNodes.ImportNode) and
697 node.rhs.module_name.value == u'cython'):
698 node = Nodes.CImportStatNode(node.pos,
699 module_name = u'cython',
700 as_name = node.lhs.name)
701 self.visit_CImportStatNode(node)
703 self.visitchildren(node)
706 def visit_NameNode(self, node):
707 if node.name in self.cython_module_names:
708 node.is_cython_module = True
710 node.cython_attribute = self.directive_names.get(node.name)
713 def try_to_parse_directives(self, node):
714 # If node is the contents of an directive (in a with statement or
715 # decorator), returns a list of (directivename, value) pairs.
716 # Otherwise, returns None
717 if isinstance(node, ExprNodes.CallNode):
718 self.visit(node.function)
719 optname = node.function.as_cython_attribute()
721 directivetype = Options.directive_types.get(optname)
723 args, kwds = node.explicit_args_kwds()
726 if kwds is not None and directivetype is not dict:
727 for keyvalue in kwds.key_value_pairs:
728 key, value = keyvalue
729 sub_optname = "%s.%s" % (optname, key.value)
730 if Options.directive_types.get(sub_optname):
731 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
733 key_value_pairs.append(keyvalue)
734 if not key_value_pairs:
737 kwds.key_value_pairs = key_value_pairs
738 if directives and not kwds and not args:
740 directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
742 elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
744 optname = node.as_cython_attribute()
746 directivetype = Options.directive_types.get(optname)
747 if directivetype is bool:
748 return [(optname, True)]
749 elif directivetype is None:
750 return [(optname, None)]
752 raise PostParseError(
753 node.pos, "The '%s' directive should be used as a function call." % optname)
756 def try_to_parse_directive(self, optname, args, kwds, pos):
757 directivetype = Options.directive_types.get(optname)
758 if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
759 return optname, Options.directive_defaults[optname]
760 elif directivetype is bool:
761 if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
762 raise PostParseError(pos,
763 'The %s directive takes one compile-time boolean argument' % optname)
764 return (optname, args[0].value)
765 elif directivetype is str:
766 if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
767 ExprNodes.UnicodeNode)):
768 raise PostParseError(pos,
769 'The %s directive takes one compile-time string argument' % optname)
770 return (optname, str(args[0].value))
771 elif directivetype is dict:
773 raise PostParseError(pos,
774 'The %s directive takes no prepositional arguments' % optname)
775 return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
776 elif directivetype is list:
777 if kwds and len(kwds) != 0:
778 raise PostParseError(pos,
779 'The %s directive takes no keyword arguments' % optname)
780 return optname, [ str(arg.value) for arg in args ]
784 def visit_with_directives(self, body, directives):
785 olddirectives = self.directives
786 newdirectives = copy.copy(olddirectives)
787 newdirectives.update(directives)
788 self.directives = newdirectives
789 assert isinstance(body, Nodes.StatListNode), body
790 retbody = self.visit_Node(body)
791 directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
792 directives=newdirectives)
793 self.directives = olddirectives
797 def visit_FuncDefNode(self, node):
798 directives = self._extract_directives(node, 'function')
800 return self.visit_Node(node)
801 body = Nodes.StatListNode(node.pos, stats=[node])
802 return self.visit_with_directives(body, directives)
804 def visit_CVarDefNode(self, node):
805 if not node.decorators:
807 for dec in node.decorators:
808 for directive in self.try_to_parse_directives(dec.decorator) or ():
809 if directive is not None and directive[0] == u'locals':
810 node.directive_locals = directive[1]
812 self.context.nonfatal_error(PostParseError(dec.pos,
813 "Cdef functions can only take cython.locals() decorator."))
816 def visit_CClassDefNode(self, node):
817 directives = self._extract_directives(node, 'cclass')
819 return self.visit_Node(node)
820 body = Nodes.StatListNode(node.pos, stats=[node])
821 return self.visit_with_directives(body, directives)
823 def visit_PyClassDefNode(self, node):
824 directives = self._extract_directives(node, 'class')
826 return self.visit_Node(node)
827 body = Nodes.StatListNode(node.pos, stats=[node])
828 return self.visit_with_directives(body, directives)
830 def _extract_directives(self, node, scope_name):
831 if not node.decorators:
833 # Split the decorators into two lists -- real decorators and directives
836 for dec in node.decorators:
837 new_directives = self.try_to_parse_directives(dec.decorator)
838 if new_directives is not None:
839 for directive in new_directives:
840 if self.check_directive_scope(node.pos, directive[0], scope_name):
841 directives.append(directive)
844 if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
845 raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
847 node.decorators = realdecs
848 # merge or override repeated directives
850 directives.reverse() # Decorators coming first take precedence
851 for directive in directives:
852 name, value = directive
854 old_value = optdict[name]
855 # keywords and arg lists can be merged, everything
856 # else overrides completely
857 if isinstance(old_value, dict):
858 old_value.update(value)
859 elif isinstance(old_value, list):
860 old_value.extend(value)
862 optdict[name] = value
864 optdict[name] = value
867 # Handle with statements
868 def visit_WithStatNode(self, node):
870 for directive in self.try_to_parse_directives(node.manager) or []:
871 if directive is not None:
872 if node.target is not None:
873 self.context.nonfatal_error(
874 PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
876 name, value = directive
878 # special case: in pure mode, "with nogil" spells "with cython.nogil"
879 node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
880 return self.visit_Node(node)
881 if self.check_directive_scope(node.pos, name, 'with statement'):
882 directive_dict[name] = value
884 return self.visit_with_directives(node.body, directive_dict)
885 return self.visit_Node(node)
887 class WithTransform(CythonTransform, SkipDeclarations):
889 # EXCINFO is manually set to a variable that contains
890 # the exc_info() tuple that can be generated by the enclosing except
892 template_without_target = TreeFragment(u"""
903 if not EXIT(*EXCINFO):
907 EXIT(None, None, None)
908 """, temps=[u'MGR', u'EXC', u"EXIT"],
909 pipeline=[NormalizeTree(None)])
911 template_with_target = TreeFragment(u"""
914 VALUE = MGR.__enter__()
923 if not EXIT(*EXCINFO):
927 EXIT(None, None, None)
928 MGR = EXIT = VALUE = EXC = None
930 """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
931 pipeline=[NormalizeTree(None)])
933 def visit_WithStatNode(self, node):
934 # TODO: Cleanup badly needed
935 TemplateTransform.temp_name_counter += 1
936 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
938 self.visitchildren(node, ['body'])
939 excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
940 if node.target is not None:
941 result = self.template_with_target.substitute({
942 u'EXPR' : node.manager,
944 u'TARGET' : node.target,
945 u'EXCINFO' : excinfo_temp
948 result = self.template_without_target.substitute({
949 u'EXPR' : node.manager,
951 u'EXCINFO' : excinfo_temp
954 # Set except excinfo target to EXCINFO
955 try_except = result.stats[-1].body.stats[-1]
956 try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
957 # excinfo_temp.ref(node.pos))
959 # result.stats[-1].body.stats[-1] = TempsBlockNode(
960 # node.pos, temps=[excinfo_temp], body=try_except)
964 def visit_ExprNode(self, node):
965 # With statements are never inside expressions.
969 class DecoratorTransform(CythonTransform, SkipDeclarations):
971 def visit_DefNode(self, func_node):
972 self.visitchildren(func_node)
973 if not func_node.decorators:
975 return self._handle_decorators(
976 func_node, func_node.name)
978 def visit_CClassDefNode(self, class_node):
979 # This doesn't currently work, so it's disabled.
981 # Problem: assignments to cdef class names do not work. They
982 # would require an additional check anyway, as the extension
983 # type must not change its C type, so decorators cannot
984 # replace an extension type, just alter it and return it.
986 self.visitchildren(class_node)
987 if not class_node.decorators:
989 error(class_node.pos,
990 "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
992 #return self._handle_decorators(
993 # class_node, class_node.class_name)
995 def visit_ClassDefNode(self, class_node):
996 self.visitchildren(class_node)
997 if not class_node.decorators:
999 return self._handle_decorators(
1000 class_node, class_node.name)
1002 def _handle_decorators(self, node, name):
1003 decorator_result = ExprNodes.NameNode(node.pos, name = name)
1004 for decorator in node.decorators[::-1]:
1005 decorator_result = ExprNodes.SimpleCallNode(
1007 function = decorator.decorator,
1008 args = [decorator_result])
1010 name_node = ExprNodes.NameNode(node.pos, name = name)
1011 reassignment = Nodes.SingleAssignmentNode(
1014 rhs = decorator_result)
1015 return [node, reassignment]
1018 class AnalyseDeclarationsTransform(CythonTransform):
1020 basic_property = TreeFragment(u"""
1024 def __set__(self, value):
1026 """, level='c_class')
1027 basic_pyobject_property = TreeFragment(u"""
1031 def __set__(self, value):
1035 """, level='c_class')
1036 basic_property_ro = TreeFragment(u"""
1040 """, level='c_class')
1042 def __call__(self, root):
1043 self.env_stack = [root.scope]
1044 # needed to determine if a cdef var is declared after it's used.
1045 self.seen_vars_stack = []
1046 return super(AnalyseDeclarationsTransform, self).__call__(root)
1048 def visit_NameNode(self, node):
1049 self.seen_vars_stack[-1].add(node.name)
1052 def visit_ModuleNode(self, node):
1053 self.seen_vars_stack.append(cython.set())
1054 node.analyse_declarations(self.env_stack[-1])
1055 self.visitchildren(node)
1056 self.seen_vars_stack.pop()
1059 def visit_LambdaNode(self, node):
1060 node.analyse_declarations(self.env_stack[-1])
1061 self.visitchildren(node)
1064 def visit_ClassDefNode(self, node):
1065 self.env_stack.append(node.scope)
1066 self.visitchildren(node)
1067 self.env_stack.pop()
1070 def visit_CClassDefNode(self, node):
1071 node = self.visit_ClassDefNode(node)
1072 if node.scope and node.scope.implemented:
1074 for entry in node.scope.var_entries:
1075 if entry.needs_property:
1076 property = self.create_Property(entry)
1077 property.analyse_declarations(node.scope)
1078 self.visit(property)
1079 stats.append(property)
1081 node.body.stats += stats
1084 def visit_FuncDefNode(self, node):
1085 self.seen_vars_stack.append(cython.set())
1086 lenv = node.local_scope
1087 node.body.analyse_control_flow(lenv) # this will be totally refactored
1088 node.declare_arguments(lenv)
1089 for var, type_node in node.directive_locals.items():
1090 if not lenv.lookup_here(var): # don't redeclare args
1091 type = type_node.analyse_as_type(lenv)
1093 lenv.declare_var(var, type, type_node.pos)
1095 error(type_node.pos, "Not a type")
1096 node.body.analyse_declarations(lenv)
1097 self.env_stack.append(lenv)
1098 self.visitchildren(node)
1099 self.env_stack.pop()
1100 self.seen_vars_stack.pop()
1103 def visit_ScopedExprNode(self, node):
1104 env = self.env_stack[-1]
1105 node.analyse_declarations(env)
1106 # the node may or may not have a local scope
1107 if node.has_local_scope:
1108 self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1109 self.env_stack.append(node.expr_scope)
1110 node.analyse_scoped_declarations(node.expr_scope)
1111 self.visitchildren(node)
1112 self.env_stack.pop()
1113 self.seen_vars_stack.pop()
1115 node.analyse_scoped_declarations(env)
1116 self.visitchildren(node)
1119 def visit_TempResultFromStatNode(self, node):
1120 self.visitchildren(node)
1121 node.analyse_declarations(self.env_stack[-1])
1124 # Some nodes are no longer needed after declaration
1125 # analysis and can be dropped. The analysis was performed
1126 # on these nodes in a seperate recursive process from the
1127 # enclosing function or module, so we can simply drop them.
1128 def visit_CDeclaratorNode(self, node):
1129 # necessary to ensure that all CNameDeclaratorNodes are visited.
1130 self.visitchildren(node)
1133 def visit_CTypeDefNode(self, node):
1136 def visit_CBaseTypeNode(self, node):
1139 def visit_CEnumDefNode(self, node):
1140 if node.visibility == 'public':
1145 def visit_CStructOrUnionDefNode(self, node):
1148 def visit_CNameDeclaratorNode(self, node):
1149 if node.name in self.seen_vars_stack[-1]:
1150 entry = self.env_stack[-1].lookup(node.name)
1151 if entry is None or entry.visibility != 'extern':
1152 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1153 self.visitchildren(node)
1156 def visit_CVarDefNode(self, node):
1157 # to ensure all CNameDeclaratorNodes are visited.
1158 self.visitchildren(node)
1161 def create_Property(self, entry):
1162 if entry.visibility == 'public':
1163 if entry.type.is_pyobject:
1164 template = self.basic_pyobject_property
1166 template = self.basic_property
1167 elif entry.visibility == 'readonly':
1168 template = self.basic_property_ro
1169 property = template.substitute({
1170 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1171 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1172 attribute=entry.name),
1173 }, pos=entry.pos).stats[0]
1174 property.name = entry.name
1175 # ---------------------------------------
1176 # XXX This should go to AutoDocTransforms
1177 # ---------------------------------------
1178 if (Options.docstrings and
1179 self.current_directives['embedsignature']):
1180 attr_name = entry.name
1181 type_name = entry.type.declaration_code("", for_display=1)
1183 if not entry.type.is_pyobject:
1184 type_name = "'%s'" % type_name
1185 elif entry.type.is_extension_type:
1186 type_name = entry.type.module_name + '.' + type_name
1187 if entry.init is not None:
1188 default_value = ' = ' + entry.init
1189 elif entry.init_to_none:
1190 default_value = ' = ' + repr(None)
1191 docstring = attr_name + ': ' + type_name + default_value
1192 property.doc = EncodedString(docstring)
1193 # ---------------------------------------
1196 class AnalyseExpressionsTransform(CythonTransform):
1198 def visit_ModuleNode(self, node):
1199 node.scope.infer_types()
1200 node.body.analyse_expressions(node.scope)
1201 self.visitchildren(node)
1204 def visit_FuncDefNode(self, node):
1205 node.local_scope.infer_types()
1206 node.body.analyse_expressions(node.local_scope)
1207 self.visitchildren(node)
1210 def visit_ScopedExprNode(self, node):
1211 if node.has_local_scope:
1212 node.expr_scope.infer_types()
1213 node.analyse_scoped_expressions(node.expr_scope)
1214 self.visitchildren(node)
1217 class ExpandInplaceOperators(EnvTransform):
1219 def visit_InPlaceAssignmentNode(self, node):
1222 if lhs.type.is_cpp_class:
1223 # No getting around this exact operator here.
1225 if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1226 # There is code to handle this case.
1229 env = self.current_env()
1230 def side_effect_free_reference(node, setting=False):
1231 if isinstance(node, ExprNodes.NameNode):
1233 elif node.type.is_pyobject and not setting:
1234 node = LetRefNode(node)
1236 elif isinstance(node, ExprNodes.IndexNode):
1237 if node.is_buffer_access:
1238 raise ValueError, "Buffer access"
1239 base, temps = side_effect_free_reference(node.base)
1240 index = LetRefNode(node.index)
1241 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1242 elif isinstance(node, ExprNodes.AttributeNode):
1243 obj, temps = side_effect_free_reference(node.obj)
1244 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1246 node = LetRefNode(node)
1249 lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1252 dup = lhs.__class__(**lhs.__dict__)
1253 binop = ExprNodes.binop_node(node.pos,
1254 operator = node.operator,
1258 # Manually analyse types for new node.
1259 lhs.analyse_target_types(env)
1260 dup.analyse_types(env)
1261 binop.analyse_operation(env)
1262 node = Nodes.SingleAssignmentNode(
1265 rhs=binop.coerce_to(lhs.type, env))
1266 # Use LetRefNode to avoid side effects.
1267 let_ref_nodes.reverse()
1268 for t in let_ref_nodes:
1269 node = LetNode(t, node)
1272 def visit_ExprNode(self, node):
1273 # In-place assignments can't happen within an expression.
1277 class AlignFunctionDefinitions(CythonTransform):
1279 This class takes the signatures from a .pxd file and applies them to
1280 the def methods in a .py file.
1283 def visit_ModuleNode(self, node):
1284 self.scope = node.scope
1285 self.directives = node.directives
1286 self.visitchildren(node)
1289 def visit_PyClassDefNode(self, node):
1290 pxd_def = self.scope.lookup(node.name)
1292 if pxd_def.is_cclass:
1293 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1295 error(node.pos, "'%s' redeclared" % node.name)
1296 error(pxd_def.pos, "previous declaration here")
1301 def visit_CClassDefNode(self, node, pxd_def=None):
1303 pxd_def = self.scope.lookup(node.class_name)
1305 outer_scope = self.scope
1306 self.scope = pxd_def.type.scope
1307 self.visitchildren(node)
1309 self.scope = outer_scope
1312 def visit_DefNode(self, node):
1313 pxd_def = self.scope.lookup(node.name)
1315 if not pxd_def.is_cfunction:
1316 error(node.pos, "'%s' redeclared" % node.name)
1317 error(pxd_def.pos, "previous declaration here")
1319 node = node.as_cfunction(pxd_def)
1320 elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1321 node = node.as_cfunction(scope=self.scope)
1322 # Enable this when internal def functions are allowed.
1323 # self.visitchildren(node)
1327 class YieldNodeCollector(TreeVisitor):
1330 super(YieldNodeCollector, self).__init__()
1333 self.has_return_value = False
1335 def visit_Node(self, node):
1336 return self.visitchildren(node)
1338 def visit_YieldExprNode(self, node):
1339 if self.has_return_value:
1340 error(node.pos, "'yield' outside function")
1341 self.yields.append(node)
1342 self.visitchildren(node)
1344 def visit_ReturnStatNode(self, node):
1346 self.has_return_value = True
1348 error(node.pos, "'return' with argument inside generator")
1349 self.returns.append(node)
1351 def visit_ClassDefNode(self, node):
1354 def visit_DefNode(self, node):
1357 def visit_LambdaNode(self, node):
1360 def visit_GeneratorExpressionNode(self, node):
1363 class MarkClosureVisitor(CythonTransform):
1365 def visit_ModuleNode(self, node):
1366 self.needs_closure = False
1367 self.visitchildren(node)
1370 def visit_FuncDefNode(self, node):
1371 self.needs_closure = False
1372 self.visitchildren(node)
1373 node.needs_closure = self.needs_closure
1374 self.needs_closure = True
1376 collector = YieldNodeCollector()
1377 collector.visitchildren(node)
1379 if collector.yields:
1380 for i, yield_expr in enumerate(collector.yields):
1381 yield_expr.label_num = i + 1
1383 gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1386 generator = Nodes.GeneratorDefNode(pos=node.pos,
1389 star_arg=node.star_arg,
1390 starstar_arg=node.starstar_arg,
1392 decorators=node.decorators,
1394 lambda_name=node.lambda_name)
1398 def visit_CFuncDefNode(self, node):
1399 self.visit_FuncDefNode(node)
1400 if node.needs_closure:
1401 error(node.pos, "closures inside cdef functions not yet supported")
1404 def visit_LambdaNode(self, node):
1405 self.needs_closure = False
1406 self.visitchildren(node)
1407 node.needs_closure = self.needs_closure
1408 self.needs_closure = True
1411 def visit_ClassDefNode(self, node):
1412 self.visitchildren(node)
1413 self.needs_closure = True
1416 class CreateClosureClasses(CythonTransform):
1417 # Output closure classes in module scope for all functions
1418 # that really need it.
1420 def __init__(self, context):
1421 super(CreateClosureClasses, self).__init__(context)
1423 self.in_lambda = False
1424 self.generator_class = None
1426 def visit_ModuleNode(self, node):
1427 self.module_scope = node.scope
1428 self.visitchildren(node)
1431 def create_generator_class(self, target_module_scope, pos):
1432 if self.generator_class:
1433 return self.generator_class
1434 # XXX: make generator class creation cleaner
1435 entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1436 objstruct_cname='__pyx_Generator_object',
1437 typeobj_cname='__pyx_Generator_type',
1438 pos=pos, defining=True, implementing=True)
1439 klass = entry.type.scope
1440 klass.is_internal = True
1441 klass.directives = {'final': True}
1443 body_type = PyrexTypes.create_typedef_type('generator_body',
1444 PyrexTypes.c_void_ptr_type,
1445 '__pyx_generator_body_t')
1446 klass.declare_var(pos=pos, name='body', cname='body',
1447 type=body_type, is_cdef=True)
1448 klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1450 klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1454 e = klass.declare_pyfunction('send', pos)
1455 e.func_cname = '__Pyx_Generator_Send'
1456 e.signature = TypeSlots.binaryfunc
1458 e = klass.declare_pyfunction('close', pos)
1459 e.func_cname = '__Pyx_Generator_Close'
1460 e.signature = TypeSlots.unaryfunc
1462 e = klass.declare_pyfunction('throw', pos)
1463 e.func_cname = '__Pyx_Generator_Throw'
1464 e.signature = TypeSlots.pyfunction_signature
1466 e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1467 e.func_cname = 'PyObject_SelfIter'
1469 e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1470 e.func_cname = '__Pyx_Generator_Next'
1472 self.generator_class = entry.type
1473 return self.generator_class
1475 def get_scope_use(self, node):
1478 for name, entry in node.local_scope.entries.items():
1479 if entry.from_closure:
1480 from_closure.append((name, entry))
1481 elif entry.in_closure and not entry.from_closure:
1482 in_closure.append((name, entry))
1483 return from_closure, in_closure
1485 def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1486 # skip generator body
1487 if node.is_generator_body:
1489 # move local variables into closure
1490 if node.is_generator:
1491 for entry in node.local_scope.entries.values():
1492 if not entry.from_closure:
1493 entry.in_closure = True
1495 from_closure, in_closure = self.get_scope_use(node)
1498 # Now from the begining
1499 node.needs_closure = False
1500 node.needs_outer_scope = False
1502 func_scope = node.local_scope
1503 cscope = node.entry.scope
1504 while cscope.is_py_class_scope or cscope.is_c_class_scope:
1505 cscope = cscope.outer_scope
1507 if not from_closure and (self.path or inner_node):
1510 raise InternalError, "DefNode does not have assignment node"
1511 inner_node = node.assmt.rhs
1512 inner_node.needs_self_code = False
1513 node.needs_outer_scope = False
1515 if node.is_generator:
1516 generator_class = self.create_generator_class(target_module_scope, node.pos)
1517 elif not in_closure and not from_closure:
1519 elif not in_closure:
1520 func_scope.is_passthrough = True
1521 func_scope.scope_class = cscope.scope_class
1522 node.needs_outer_scope = True
1525 as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1527 if node.is_generator:
1528 entry = target_module_scope.declare_c_class(name = as_name,
1529 pos = node.pos, defining = True, implementing = True, base_type=generator_class)
1531 entry = target_module_scope.declare_c_class(name = as_name,
1532 pos = node.pos, defining = True, implementing = True)
1533 func_scope.scope_class = entry
1534 class_scope = entry.type.scope
1535 class_scope.is_internal = True
1536 class_scope.directives = {'final': True}
1539 assert cscope.is_closure_scope
1540 class_scope.declare_var(pos=node.pos,
1541 name=Naming.outer_scope_cname,
1542 cname=Naming.outer_scope_cname,
1543 type=cscope.scope_class.type,
1545 node.needs_outer_scope = True
1546 for name, entry in in_closure:
1547 class_scope.declare_var(pos=entry.pos,
1552 node.needs_closure = True
1553 # Do it here because other classes are already checked
1554 target_module_scope.check_c_class(func_scope.scope_class)
1556 def visit_LambdaNode(self, node):
1557 was_in_lambda = self.in_lambda
1558 self.in_lambda = True
1559 self.create_class_from_scope(node.def_node, self.module_scope, node)
1560 self.visitchildren(node)
1561 self.in_lambda = was_in_lambda
1564 def visit_FuncDefNode(self, node):
1566 self.visitchildren(node)
1568 if node.needs_closure or self.path:
1569 self.create_class_from_scope(node, self.module_scope)
1570 self.path.append(node)
1571 self.visitchildren(node)
1576 class GilCheck(VisitorTransform):
1578 Call `node.gil_check(env)` on each node to make sure we hold the
1579 GIL when we need it. Raise an error when on Python operations
1580 inside a `nogil` environment.
1582 def __call__(self, root):
1583 self.env_stack = [root.scope]
1585 return super(GilCheck, self).__call__(root)
1587 def visit_FuncDefNode(self, node):
1588 self.env_stack.append(node.local_scope)
1589 was_nogil = self.nogil
1590 self.nogil = node.local_scope.nogil
1591 if self.nogil and node.nogil_check:
1592 node.nogil_check(node.local_scope)
1593 self.visitchildren(node)
1594 self.env_stack.pop()
1595 self.nogil = was_nogil
1598 def visit_GILStatNode(self, node):
1599 env = self.env_stack[-1]
1600 if self.nogil and node.nogil_check: node.nogil_check()
1601 was_nogil = self.nogil
1602 self.nogil = (node.state == 'nogil')
1603 self.visitchildren(node)
1604 self.nogil = was_nogil
1607 def visit_Node(self, node):
1608 if self.env_stack and self.nogil and node.nogil_check:
1609 node.nogil_check(self.env_stack[-1])
1610 self.visitchildren(node)
1614 class TransformBuiltinMethods(EnvTransform):
1616 def visit_SingleAssignmentNode(self, node):
1617 if node.declaration_only:
1620 self.visitchildren(node)
1623 def visit_AttributeNode(self, node):
1624 self.visitchildren(node)
1625 return self.visit_cython_attribute(node)
1627 def visit_NameNode(self, node):
1628 return self.visit_cython_attribute(node)
1630 def visit_cython_attribute(self, node):
1631 attribute = node.as_cython_attribute()
1633 if attribute == u'compiled':
1634 node = ExprNodes.BoolNode(node.pos, value=True)
1635 elif attribute == u'NULL':
1636 node = ExprNodes.NullNode(node.pos)
1637 elif attribute in (u'set', u'frozenset'):
1638 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1639 entry=self.current_env().builtin_scope().lookup_here(attribute))
1640 elif not PyrexTypes.parse_basic_type(attribute):
1641 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1644 def visit_SimpleCallNode(self, node):
1647 if isinstance(node.function, ExprNodes.NameNode):
1648 if node.function.name == 'locals':
1649 lenv = self.current_env()
1650 entry = lenv.lookup_here('locals')
1652 # not the builtin 'locals'
1654 if len(node.args) > 0:
1655 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1658 items = [ ExprNodes.DictItemNode(pos,
1659 key=ExprNodes.StringNode(pos, value=var),
1660 value=ExprNodes.NameNode(pos, name=var))
1661 for var in lenv.entries ]
1662 return ExprNodes.DictNode(pos, key_value_pairs=items)
1665 function = node.function.as_cython_attribute()
1667 if function in InterpretCompilerDirectives.unop_method_nodes:
1668 if len(node.args) != 1:
1669 error(node.function.pos, u"%s() takes exactly one argument" % function)
1671 node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1672 elif function in InterpretCompilerDirectives.binop_method_nodes:
1673 if len(node.args) != 2:
1674 error(node.function.pos, u"%s() takes exactly two arguments" % function)
1676 node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1677 elif function == u'cast':
1678 if len(node.args) != 2:
1679 error(node.function.pos, u"cast() takes exactly two arguments")
1681 type = node.args[0].analyse_as_type(self.current_env())
1683 node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1685 error(node.args[0].pos, "Not a type")
1686 elif function == u'sizeof':
1687 if len(node.args) != 1:
1688 error(node.function.pos, u"sizeof() takes exactly one argument")
1690 type = node.args[0].analyse_as_type(self.current_env())
1692 node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1694 node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1695 elif function == 'cmod':
1696 if len(node.args) != 2:
1697 error(node.function.pos, u"cmod() takes exactly two arguments")
1699 node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1700 node.cdivision = True
1701 elif function == 'cdiv':
1702 if len(node.args) != 2:
1703 error(node.function.pos, u"cdiv() takes exactly two arguments")
1705 node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1706 node.cdivision = True
1707 elif function == u'set':
1708 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1710 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1712 self.visitchildren(node)
1716 class DebugTransform(CythonTransform):
1718 Create debug information and all functions' visibility to extern in order
1719 to enable debugging.
1722 def __init__(self, context, options, result):
1723 super(DebugTransform, self).__init__(context)
1724 self.visited = cython.set()
1725 # our treebuilder and debug output writer
1726 # (see Cython.Debugger.debug_output.CythonDebugWriter)
1727 self.tb = self.context.gdb_debug_outputwriter
1728 #self.c_output_file = options.output_file
1729 self.c_output_file = result.c_file
1731 # Closure support, basically treat nested functions as if the AST were
1733 self.nested_funcdefs = []
1735 # tells visit_NameNode whether it should register step-into functions
1736 self.register_stepinto = False
1738 def visit_ModuleNode(self, node):
1739 self.tb.module_name = node.full_module_name
1741 module_name=node.full_module_name,
1742 filename=node.pos[0].filename,
1743 c_filename=self.c_output_file)
1745 self.tb.start('Module', attrs)
1747 # serialize functions
1748 self.tb.start('Functions')
1749 # First, serialize functions normally...
1750 self.visitchildren(node)
1752 # ... then, serialize nested functions
1753 for nested_funcdef in self.nested_funcdefs:
1754 self.visit_FuncDefNode(nested_funcdef)
1756 self.register_stepinto = True
1757 self.serialize_modulenode_as_function(node)
1758 self.register_stepinto = False
1759 self.tb.end('Functions')
1761 # 2.3 compatibility. Serialize global variables
1762 self.tb.start('Globals')
1765 for k, v in node.scope.entries.iteritems():
1766 if (v.qualified_name not in self.visited and not
1767 v.name.startswith('__pyx_') and not
1768 v.type.is_cfunction and not
1769 v.type.is_extension_type):
1772 self.serialize_local_variables(entries)
1773 self.tb.end('Globals')
1774 # self.tb.end('Module') # end Module after the line number mapping in
1775 # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1778 def visit_FuncDefNode(self, node):
1779 self.visited.add(node.local_scope.qualified_name)
1781 if getattr(node, 'is_wrapper', False):
1784 if self.register_stepinto:
1785 self.nested_funcdefs.append(node)
1788 # node.entry.visibility = 'extern'
1789 if node.py_func is None:
1792 pf_cname = node.py_func.entry.func_cname
1795 name=node.entry.name,
1796 cname=node.entry.func_cname,
1798 qualified_name=node.local_scope.qualified_name,
1799 lineno=str(node.pos[1]))
1801 self.tb.start('Function', attrs=attrs)
1803 self.tb.start('Locals')
1804 self.serialize_local_variables(node.local_scope.entries)
1805 self.tb.end('Locals')
1807 self.tb.start('Arguments')
1808 for arg in node.local_scope.arg_entries:
1809 self.tb.start(arg.name)
1810 self.tb.end(arg.name)
1811 self.tb.end('Arguments')
1813 self.tb.start('StepIntoFunctions')
1814 self.register_stepinto = True
1815 self.visitchildren(node)
1816 self.register_stepinto = False
1817 self.tb.end('StepIntoFunctions')
1818 self.tb.end('Function')
1822 def visit_NameNode(self, node):
1823 if (self.register_stepinto and
1824 node.type.is_cfunction and
1825 getattr(node, 'is_called', False) and
1826 node.entry.func_cname is not None):
1827 # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1828 # declared functions are not 'in_cinclude'.
1829 # This means we will list called 'cdef' functions as
1830 # "step into functions", but this is not an issue as they will be
1831 # recognized as Cython functions anyway.
1832 attrs = dict(name=node.entry.func_cname)
1833 self.tb.start('StepIntoFunction', attrs=attrs)
1834 self.tb.end('StepIntoFunction')
1836 self.visitchildren(node)
1839 def serialize_modulenode_as_function(self, node):
1841 Serialize the module-level code as a function so the debugger will know
1842 it's a "relevant frame" and it will know where to set the breakpoint
1843 for 'break modulename'.
1845 name = node.full_module_name.rpartition('.')[-1]
1847 cname_py2 = 'init' + name
1848 cname_py3 = 'PyInit_' + name
1854 # Ignore the qualified_name, breakpoints should be set using
1855 # `cy break modulename:lineno` for module-level breakpoints.
1858 is_initmodule_function="True",
1861 py3_attrs = dict(py2_attrs, cname=cname_py3)
1863 self._serialize_modulenode_as_function(node, py2_attrs)
1864 self._serialize_modulenode_as_function(node, py3_attrs)
1866 def _serialize_modulenode_as_function(self, node, attrs):
1867 self.tb.start('Function', attrs=attrs)
1869 self.tb.start('Locals')
1870 self.serialize_local_variables(node.scope.entries)
1871 self.tb.end('Locals')
1873 self.tb.start('Arguments')
1874 self.tb.end('Arguments')
1876 self.tb.start('StepIntoFunctions')
1877 self.register_stepinto = True
1878 self.visitchildren(node)
1879 self.register_stepinto = False
1880 self.tb.end('StepIntoFunctions')
1882 self.tb.end('Function')
1884 def serialize_local_variables(self, entries):
1885 for entry in entries.values():
1886 if entry.type.is_pyobject:
1887 vartype = 'PythonObject'
1891 if entry.from_closure:
1892 # We're dealing with a closure where a variable from an outer
1893 # scope is accessed, get it from the scope object.
1894 cname = '%s->%s' % (Naming.cur_scope_cname,
1895 entry.outer_entry.cname)
1897 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1900 elif entry.in_closure:
1901 cname = '%s->%s' % (Naming.cur_scope_cname,
1903 qname = entry.qualified_name
1906 qname = entry.qualified_name
1909 # this happens for variables that are not in the user's code,
1910 # e.g. for the global __builtins__, __doc__, etc. We can just
1911 # set the lineno to 0 for those.
1914 lineno = str(entry.pos[1])
1919 qualified_name=qname,
1923 self.tb.start('LocalVar', attrs)
1924 self.tb.end('LocalVar')