Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67 class NormalizeTree(CythonTransform):
  68     """
  69     This transform fixes up a few things after parsing
  70     in order to make the parse tree more suitable for
  71     transforms.
  72
  73     a) After parsing, blocks with only one statement will
  74     be represented by that statement, not by a StatListNode.
  75     When doing transforms this is annoying and inconsistent,
  76     as one cannot in general remove a statement in a consistent
  77     way and so on. This transform wraps any single statements
  78     in a StatListNode containing a single statement.
  79
  80     b) The PassStatNode is a noop and serves no purpose beyond
  81     plugging such one-statement blocks; i.e., once parsed a
  82 `    "pass" can just as well be represented using an empty
  83     StatListNode. This means less special cases to worry about
  84     in subsequent transforms (one always checks to see if a
  85     StatListNode has no children to see if the block is empty).
  86     """
  87
  88     def __init__(self, context):
  89         super(NormalizeTree, self).__init__(context)
  90         self.is_in_statlist = False
  91         self.is_in_expr = False
  92
  93     def visit_ExprNode(self, node):
  94         stacktmp = self.is_in_expr
  95         self.is_in_expr = True
  96         self.visitchildren(node)
  97         self.is_in_expr = stacktmp
  98         return node
  99
 100     def visit_StatNode(self, node, is_listcontainer=False):
 101         stacktmp = self.is_in_statlist
 102         self.is_in_statlist = is_listcontainer
 103         self.visitchildren(node)
 104         self.is_in_statlist = stacktmp
 105         if not self.is_in_statlist and not self.is_in_expr:
 106             return Nodes.StatListNode(pos=node.pos, stats=[node])
 107         else:
 108             return node
 109
 110     def visit_StatListNode(self, node):
 111         self.is_in_statlist = True
 112         self.visitchildren(node)
 113         self.is_in_statlist = False
 114         return node
 115
 116     def visit_ParallelAssignmentNode(self, node):
 117         return self.visit_StatNode(node, True)
 118
 119     def visit_CEnumDefNode(self, node):
 120         return self.visit_StatNode(node, True)
 121
 122     def visit_CStructOrUnionDefNode(self, node):
 123         return self.visit_StatNode(node, True)
 124
 125     # Eliminate PassStatNode
 126     def visit_PassStatNode(self, node):
 127         if not self.is_in_statlist:
 128             return Nodes.StatListNode(pos=node.pos, stats=[])
 129         else:
 130             return []
 131
 132     def visit_CDeclaratorNode(self, node):
 133         return node
 134
 135
 136 class PostParseError(CompileError): pass
 137
 138 # error strings checked by unit tests, so define them
 139 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 140 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 141 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 142 class PostParse(ScopeTrackingTransform):
 143     """
 144     Basic interpretation of the parse tree, as well as validity
 145     checking that can be done on a very basic level on the parse
 146     tree (while still not being a problem with the basic syntax,
 147     as such).
 148
 149     Specifically:
 150     - Default values to cdef assignments are turned into single
 151     assignments following the declaration (everywhere but in class
 152     bodies, where they raise a compile error)
 153
 154     - Interpret some node structures into Python runtime values.
 155     Some nodes take compile-time arguments (currently:
 156     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 157     which should be interpreted. This happens in a general way
 158     and other steps should be taken to ensure validity.
 159
 160     Type arguments cannot be interpreted in this way.
 161
 162     - For __cythonbufferdefaults__ the arguments are checked for
 163     validity.
 164
 165     TemplatedTypeNode has its directives interpreted:
 166     Any first positional argument goes into the "dtype" attribute,
 167     any "ndim" keyword argument goes into the "ndim" attribute and
 168     so on. Also it is checked that the directive combination is valid.
 169     - __cythonbufferdefaults__ attributes are parsed and put into the
 170     type information.
 171
 172     Note: Currently Parsing.py does a lot of interpretation and
 173     reorganization that can be refactored into this transform
 174     if a more pure Abstract Syntax Tree is wanted.
 175     """
 176
 177     def __init__(self, context):
 178         super(PostParse, self).__init__(context)
 179         self.specialattribute_handlers = {
 180             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 181         }
 182
 183     def visit_ModuleNode(self, node):
 184         self.lambda_counter = 1
 185         self.genexpr_counter = 1
 186         return super(PostParse, self).visit_ModuleNode(node)
 187
 188     def visit_LambdaNode(self, node):
 189         # unpack a lambda expression into the corresponding DefNode
 190         lambda_id = self.lambda_counter
 191         self.lambda_counter += 1
 192         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 193         collector = YieldNodeCollector()
 194         collector.visitchildren(node.result_expr)
 195         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 196             body = Nodes.ExprStatNode(
 197                 node.result_expr.pos, expr=node.result_expr)
 198         else:
 199             body = Nodes.ReturnStatNode(
 200                 node.result_expr.pos, value=node.result_expr)
 201         node.def_node = Nodes.DefNode(
 202             node.pos, name=node.name, lambda_name=node.lambda_name,
 203             args=node.args, star_arg=node.star_arg,
 204             starstar_arg=node.starstar_arg,
 205             body=body, doc=None)
 206         self.visitchildren(node)
 207         return node
 208
 209     def visit_GeneratorExpressionNode(self, node):
 210         # unpack a generator expression into the corresponding DefNode
 211         genexpr_id = self.genexpr_counter
 212         self.genexpr_counter += 1
 213         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 214
 215         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 216                                       doc=None,
 217                                       args=[], star_arg=None,
 218                                       starstar_arg=None,
 219                                       body=node.loop)
 220         self.visitchildren(node)
 221         return node
 222
 223     # cdef variables
 224     def handle_bufferdefaults(self, decl):
 225         if not isinstance(decl.default, ExprNodes.DictNode):
 226             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 227         self.scope_node.buffer_defaults_node = decl.default
 228         self.scope_node.buffer_defaults_pos = decl.pos
 229
 230     def visit_CVarDefNode(self, node):
 231         # This assumes only plain names and pointers are assignable on
 232         # declaration. Also, it makes use of the fact that a cdef decl
 233         # must appear before the first use, so we don't have to deal with
 234         # "i = 3; cdef int i = i" and can simply move the nodes around.
 235         try:
 236             self.visitchildren(node)
 237             stats = [node]
 238             newdecls = []
 239             for decl in node.declarators:
 240                 declbase = decl
 241                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 242                     declbase = declbase.base
 243                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 244                     if declbase.default is not None:
 245                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 246                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 247                                 handler = self.specialattribute_handlers.get(decl.name)
 248                                 if handler:
 249                                     if decl is not declbase:
 250                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 251                                     handler(decl)
 252                                     continue # Remove declaration
 253                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 254                         first_assignment = self.scope_type != 'module'
 255                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 256                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 257                             rhs=declbase.default, first=first_assignment))
 258                         declbase.default = None
 259                 newdecls.append(decl)
 260             node.declarators = newdecls
 261             return stats
 262         except PostParseError, e:
 263             # An error in a cdef clause is ok, simply remove the declaration
 264             # and try to move on to report more errors
 265             self.context.nonfatal_error(e)
 266             return None
 267
 268     # Split parallel assignments (a,b = b,a) into separate partial
 269     # assignments that are executed rhs-first using temps.  This
 270     # restructuring must be applied before type analysis so that known
 271     # types on rhs and lhs can be matched directly.  It is required in
 272     # the case that the types cannot be coerced to a Python type in
 273     # order to assign from a tuple.
 274
 275     def visit_SingleAssignmentNode(self, node):
 276         self.visitchildren(node)
 277         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 278
 279     def visit_CascadedAssignmentNode(self, node):
 280         self.visitchildren(node)
 281         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 282
 283     def _visit_assignment_node(self, node, expr_list):
 284         """Flatten parallel assignments into separate single
 285         assignments or cascaded assignments.
 286         """
 287         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 288             # no parallel assignments => nothing to do
 289             return node
 290
 291         expr_list_list = []
 292         flatten_parallel_assignments(expr_list, expr_list_list)
 293         temp_refs = []
 294         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 295
 296         nodes = []
 297         for expr_list in expr_list_list:
 298             lhs_list = expr_list[:-1]
 299             rhs = expr_list[-1]
 300             if len(lhs_list) == 1:
 301                 node = Nodes.SingleAssignmentNode(rhs.pos,
 302                     lhs = lhs_list[0], rhs = rhs)
 303             else:
 304                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 305                     lhs_list = lhs_list, rhs = rhs)
 306             nodes.append(node)
 307
 308         if len(nodes) == 1:
 309             assign_node = nodes[0]
 310         else:
 311             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 312
 313         if temp_refs:
 314             duplicates_and_temps = [ (temp.expression, temp)
 315                                      for temp in temp_refs ]
 316             sort_common_subsequences(duplicates_and_temps)
 317             for _, temp_ref in duplicates_and_temps[::-1]:
 318                 assign_node = LetNode(temp_ref, assign_node)
 319
 320         return assign_node
 321
 322     def _flatten_sequence(self, seq, result):
 323         for arg in seq.args:
 324             if arg.is_sequence_constructor:
 325                 self._flatten_sequence(arg, result)
 326             else:
 327                 result.append(arg)
 328         return result
 329
 330     def visit_DelStatNode(self, node):
 331         self.visitchildren(node)
 332         node.args = self._flatten_sequence(node, [])
 333         return node
 334
 335
 336 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 337     """Replace rhs items by LetRefNodes if they appear more than once.
 338     Creates a sequence of LetRefNodes that set up the required temps
 339     and appends them to ref_node_sequence.  The input list is modified
 340     in-place.
 341     """
 342     seen_nodes = cython.set()
 343     ref_nodes = {}
 344     def find_duplicates(node):
 345         if node.is_literal or node.is_name:
 346             # no need to replace those; can't include attributes here
 347             # as their access is not necessarily side-effect free
 348             return
 349         if node in seen_nodes:
 350             if node not in ref_nodes:
 351                 ref_node = LetRefNode(node)
 352                 ref_nodes[node] = ref_node
 353                 ref_node_sequence.append(ref_node)
 354         else:
 355             seen_nodes.add(node)
 356             if node.is_sequence_constructor:
 357                 for item in node.args:
 358                     find_duplicates(item)
 359
 360     for expr_list in expr_list_list:
 361         rhs = expr_list[-1]
 362         find_duplicates(rhs)
 363     if not ref_nodes:
 364         return
 365
 366     def substitute_nodes(node):
 367         if node in ref_nodes:
 368             return ref_nodes[node]
 369         elif node.is_sequence_constructor:
 370             node.args = list(map(substitute_nodes, node.args))
 371         return node
 372
 373     # replace nodes inside of the common subexpressions
 374     for node in ref_nodes:
 375         if node.is_sequence_constructor:
 376             node.args = list(map(substitute_nodes, node.args))
 377
 378     # replace common subexpressions on all rhs items
 379     for expr_list in expr_list_list:
 380         expr_list[-1] = substitute_nodes(expr_list[-1])
 381
 382 def sort_common_subsequences(items):
 383     """Sort items/subsequences so that all items and subsequences that
 384     an item contains appear before the item itself.  This is needed
 385     because each rhs item must only be evaluated once, so its value
 386     must be evaluated first and then reused when packing sequences
 387     that contain it.
 388
 389     This implies a partial order, and the sort must be stable to
 390     preserve the original order as much as possible, so we use a
 391     simple insertion sort (which is very fast for short sequences, the
 392     normal case in practice).
 393     """
 394     def contains(seq, x):
 395         for item in seq:
 396             if item is x:
 397                 return True
 398             elif item.is_sequence_constructor and contains(item.args, x):
 399                 return True
 400         return False
 401     def lower_than(a,b):
 402         return b.is_sequence_constructor and contains(b.args, a)
 403
 404     for pos, item in enumerate(items):
 405         key = item[1] # the ResultRefNode which has already been injected into the sequences
 406         new_pos = pos
 407         for i in xrange(pos-1, -1, -1):
 408             if lower_than(key, items[i][0]):
 409                 new_pos = i
 410         if new_pos != pos:
 411             for i in xrange(pos, new_pos, -1):
 412                 items[i] = items[i-1]
 413             items[new_pos] = item
 414
 415 def flatten_parallel_assignments(input, output):
 416     #  The input is a list of expression nodes, representing the LHSs
 417     #  and RHS of one (possibly cascaded) assignment statement.  For
 418     #  sequence constructors, rearranges the matching parts of both
 419     #  sides into a list of equivalent assignments between the
 420     #  individual elements.  This transformation is applied
 421     #  recursively, so that nested structures get matched as well.
 422     rhs = input[-1]
 423     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 424         output.append(input)
 425         return
 426
 427     complete_assignments = []
 428
 429     rhs_size = len(rhs.args)
 430     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 431     starred_assignments = []
 432     for lhs in input[:-1]:
 433         if not lhs.is_sequence_constructor:
 434             if lhs.is_starred:
 435                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 436             complete_assignments.append(lhs)
 437             continue
 438         lhs_size = len(lhs.args)
 439         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 440         if starred_targets > 1:
 441             error(lhs.pos, "more than 1 starred expression in assignment")
 442             output.append([lhs,rhs])
 443             continue
 444         elif lhs_size - starred_targets > rhs_size:
 445             error(lhs.pos, "need more than %d value%s to unpack"
 446                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 447             output.append([lhs,rhs])
 448             continue
 449         elif starred_targets:
 450             map_starred_assignment(lhs_targets, starred_assignments,
 451                                    lhs.args, rhs.args)
 452         elif lhs_size < rhs_size:
 453             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 454                   % (lhs_size, rhs_size))
 455             output.append([lhs,rhs])
 456             continue
 457         else:
 458             for targets, expr in zip(lhs_targets, lhs.args):
 459                 targets.append(expr)
 460
 461     if complete_assignments:
 462         complete_assignments.append(rhs)
 463         output.append(complete_assignments)
 464
 465     # recursively flatten partial assignments
 466     for cascade, rhs in zip(lhs_targets, rhs.args):
 467         if cascade:
 468             cascade.append(rhs)
 469             flatten_parallel_assignments(cascade, output)
 470
 471     # recursively flatten starred assignments
 472     for cascade in starred_assignments:
 473         if cascade[0].is_sequence_constructor:
 474             flatten_parallel_assignments(cascade, output)
 475         else:
 476             output.append(cascade)
 477
 478 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 479     # Appends the fixed-position LHS targets to the target list that
 480     # appear left and right of the starred argument.
 481     #
 482     # The starred_assignments list receives a new tuple
 483     # (lhs_target, rhs_values_list) that maps the remaining arguments
 484     # (those that match the starred target) to a list.
 485
 486     # left side of the starred target
 487     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 488         if expr.is_starred:
 489             starred = i
 490             lhs_remaining = len(lhs_args) - i - 1
 491             break
 492         targets.append(expr)
 493     else:
 494         raise InternalError("no starred arg found when splitting starred assignment")
 495
 496     # right side of the starred target
 497     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 498                                             lhs_args[starred + 1:])):
 499         targets.append(expr)
 500
 501     # the starred target itself, must be assigned a (potentially empty) list
 502     target = lhs_args[starred].target # unpack starred node
 503     starred_rhs = rhs_args[starred:]
 504     if lhs_remaining:
 505         starred_rhs = starred_rhs[:-lhs_remaining]
 506     if starred_rhs:
 507         pos = starred_rhs[0].pos
 508     else:
 509         pos = target.pos
 510     starred_assignments.append([
 511         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 512
 513
 514 class PxdPostParse(CythonTransform, SkipDeclarations):
 515     """
 516     Basic interpretation/validity checking that should only be
 517     done on pxd trees.
 518
 519     A lot of this checking currently happens in the parser; but
 520     what is listed below happens here.
 521
 522     - "def" functions are let through only if they fill the
 523     getbuffer/releasebuffer slots
 524
 525     - cdef functions are let through only if they are on the
 526     top level and are declared "inline"
 527     """
 528     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 529     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 530
 531     def __call__(self, node):
 532         self.scope_type = 'pxd'
 533         return super(PxdPostParse, self).__call__(node)
 534
 535     def visit_CClassDefNode(self, node):
 536         old = self.scope_type
 537         self.scope_type = 'cclass'
 538         self.visitchildren(node)
 539         self.scope_type = old
 540         return node
 541
 542     def visit_FuncDefNode(self, node):
 543         # FuncDefNode always come with an implementation (without
 544         # an imp they are CVarDefNodes..)
 545         err = self.ERR_INLINE_ONLY
 546
 547         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 548             and node.name in ('__getbuffer__', '__releasebuffer__')):
 549             err = None # allow these slots
 550
 551         if isinstance(node, Nodes.CFuncDefNode):
 552             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 553                 node.inline_in_pxd = True
 554                 if node.visibility != 'private':
 555                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 556                 elif node.api:
 557                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 558                 else:
 559                     err = None # allow inline function
 560             else:
 561                 err = self.ERR_INLINE_ONLY
 562
 563         if err:
 564             self.context.nonfatal_error(PostParseError(node.pos, err))
 565             return None
 566         else:
 567             return node
 568
 569 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 570     """
 571     After parsing, directives can be stored in a number of places:
 572     - #cython-comments at the top of the file (stored in ModuleNode)
 573     - Command-line arguments overriding these
 574     - @cython.directivename decorators
 575     - with cython.directivename: statements
 576
 577     This transform is responsible for interpreting these various sources
 578     and store the directive in two ways:
 579     - Set the directives attribute of the ModuleNode for global directives.
 580     - Use a CompilerDirectivesNode to override directives for a subtree.
 581
 582     (The first one is primarily to not have to modify with the tree
 583     structure, so that ModuleNode stay on top.)
 584
 585     The directives are stored in dictionaries from name to value in effect.
 586     Each such dictionary is always filled in for all possible directives,
 587     using default values where no value is given by the user.
 588
 589     The available directives are controlled in Options.py.
 590
 591     Note that we have to run this prior to analysis, and so some minor
 592     duplication of functionality has to occur: We manually track cimports
 593     and which names the "cython" module may have been imported to.
 594     """
 595     unop_method_nodes = {
 596         'typeof': ExprNodes.TypeofNode,
 597
 598         'operator.address': ExprNodes.AmpersandNode,
 599         'operator.dereference': ExprNodes.DereferenceNode,
 600         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 601         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 602         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 603         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 604
 605         # For backwards compatability.
 606         'address': ExprNodes.AmpersandNode,
 607     }
 608
 609     binop_method_nodes = {
 610         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 611     }
 612
 613     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 614                                   'cast', 'pointer', 'compiled', 'NULL'])
 615     special_methods.update(unop_method_nodes.keys())
 616
 617     def __init__(self, context, compilation_directive_defaults):
 618         super(InterpretCompilerDirectives, self).__init__(context)
 619         self.compilation_directive_defaults = {}
 620         for key, value in compilation_directive_defaults.items():
 621             self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
 622         self.cython_module_names = cython.set()
 623         self.directive_names = {}
 624
 625     def check_directive_scope(self, pos, directive, scope):
 626         legal_scopes = Options.directive_scopes.get(directive, None)
 627         if legal_scopes and scope not in legal_scopes:
 628             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 629                                         'is not allowed in %s scope' % (directive, scope)))
 630             return False
 631         else:
 632             return True
 633
 634     # Set up processing and handle the cython: comments.
 635     def visit_ModuleNode(self, node):
 636         for key, value in node.directive_comments.items():
 637             if not self.check_directive_scope(node.pos, key, 'module'):
 638                 self.wrong_scope_error(node.pos, key, 'module')
 639                 del node.directive_comments[key]
 640
 641         directives = copy.deepcopy(Options.directive_defaults)
 642         directives.update(copy.deepcopy(self.compilation_directive_defaults))
 643         directives.update(node.directive_comments)
 644         self.directives = directives
 645         node.directives = directives
 646         self.visitchildren(node)
 647         node.cython_module_names = self.cython_module_names
 648         return node
 649
 650     # The following four functions track imports and cimports that
 651     # begin with "cython"
 652     def is_cython_directive(self, name):
 653         return (name in Options.directive_types or
 654                 name in self.special_methods or
 655                 PyrexTypes.parse_basic_type(name))
 656
 657     def visit_CImportStatNode(self, node):
 658         if node.module_name == u"cython":
 659             self.cython_module_names.add(node.as_name or u"cython")
 660         elif node.module_name.startswith(u"cython."):
 661             if node.as_name:
 662                 self.directive_names[node.as_name] = node.module_name[7:]
 663             else:
 664                 self.cython_module_names.add(u"cython")
 665             # if this cimport was a compiler directive, we don't
 666             # want to leave the cimport node sitting in the tree
 667             return None
 668         return node
 669
 670     def visit_FromCImportStatNode(self, node):
 671         if (node.module_name == u"cython") or \
 672                node.module_name.startswith(u"cython."):
 673             submodule = (node.module_name + u".")[7:]
 674             newimp = []
 675             for pos, name, as_name, kind in node.imported_names:
 676                 full_name = submodule + name
 677                 if self.is_cython_directive(full_name):
 678                     if as_name is None:
 679                         as_name = full_name
 680                     self.directive_names[as_name] = full_name
 681                     if kind is not None:
 682                         self.context.nonfatal_error(PostParseError(pos,
 683                             "Compiler directive imports must be plain imports"))
 684                 else:
 685                     newimp.append((pos, name, as_name, kind))
 686             if not newimp:
 687                 return None
 688             node.imported_names = newimp
 689         return node
 690
 691     def visit_FromImportStatNode(self, node):
 692         if (node.module.module_name.value == u"cython") or \
 693                node.module.module_name.value.startswith(u"cython."):
 694             submodule = (node.module.module_name.value + u".")[7:]
 695             newimp = []
 696             for name, name_node in node.items:
 697                 full_name = submodule + name
 698                 if self.is_cython_directive(full_name):
 699                     self.directive_names[name_node.name] = full_name
 700                 else:
 701                     newimp.append((name, name_node))
 702             if not newimp:
 703                 return None
 704             node.items = newimp
 705         return node
 706
 707     def visit_SingleAssignmentNode(self, node):
 708         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 709                 node.rhs.module_name.value == u'cython'):
 710             node = Nodes.CImportStatNode(node.pos,
 711                                          module_name = u'cython',
 712                                          as_name = node.lhs.name)
 713             self.visit_CImportStatNode(node)
 714         else:
 715             self.visitchildren(node)
 716         return node
 717
 718     def visit_NameNode(self, node):
 719         if node.name in self.cython_module_names:
 720             node.is_cython_module = True
 721         else:
 722             node.cython_attribute = self.directive_names.get(node.name)
 723         return node
 724
 725     def try_to_parse_directives(self, node):
 726         # If node is the contents of an directive (in a with statement or
 727         # decorator), returns a list of (directivename, value) pairs.
 728         # Otherwise, returns None
 729         if isinstance(node, ExprNodes.CallNode):
 730             self.visit(node.function)
 731             optname = node.function.as_cython_attribute()
 732             if optname:
 733                 directivetype = Options.directive_types.get(optname)
 734                 if directivetype:
 735                     args, kwds = node.explicit_args_kwds()
 736                     directives = []
 737                     key_value_pairs = []
 738                     if kwds is not None and directivetype is not dict:
 739                         for keyvalue in kwds.key_value_pairs:
 740                             key, value = keyvalue
 741                             sub_optname = "%s.%s" % (optname, key.value)
 742                             if Options.directive_types.get(sub_optname):
 743                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 744                             else:
 745                                 key_value_pairs.append(keyvalue)
 746                         if not key_value_pairs:
 747                             kwds = None
 748                         else:
 749                             kwds.key_value_pairs = key_value_pairs
 750                         if directives and not kwds and not args:
 751                             return directives
 752                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 753                     return directives
 754         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 755             self.visit(node)
 756             optname = node.as_cython_attribute()
 757             if optname:
 758                 directivetype = Options.directive_types.get(optname)
 759                 if directivetype is bool:
 760                     return [(optname, True)]
 761                 elif directivetype is None:
 762                     return [(optname, None)]
 763                 else:
 764                     raise PostParseError(
 765                         node.pos, "The '%s' directive should be used as a function call." % optname)
 766         return None
 767
 768     def try_to_parse_directive(self, optname, args, kwds, pos):
 769         directivetype = Options.directive_types.get(optname)
 770         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 771             return optname, Options.directive_defaults[optname]
 772         elif directivetype is bool:
 773             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 774                 raise PostParseError(pos,
 775                     'The %s directive takes one compile-time boolean argument' % optname)
 776             return (optname, args[0].value)
 777         elif directivetype is str:
 778             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 779                                                                               ExprNodes.UnicodeNode)):
 780                 raise PostParseError(pos,
 781                     'The %s directive takes one compile-time string argument' % optname)
 782             return (optname, str(args[0].value))
 783         elif directivetype is dict:
 784             if len(args) != 0:
 785                 raise PostParseError(pos,
 786                     'The %s directive takes no prepositional arguments' % optname)
 787             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 788         elif directivetype is list:
 789             if kwds and len(kwds) != 0:
 790                 raise PostParseError(pos,
 791                     'The %s directive takes no keyword arguments' % optname)
 792             return optname, [ str(arg.value) for arg in args ]
 793         else:
 794             assert False
 795
 796     def visit_with_directives(self, body, directives):
 797         olddirectives = self.directives
 798         newdirectives = copy.copy(olddirectives)
 799         newdirectives.update(directives)
 800         self.directives = newdirectives
 801         assert isinstance(body, Nodes.StatListNode), body
 802         retbody = self.visit_Node(body)
 803         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 804                                                  directives=newdirectives)
 805         self.directives = olddirectives
 806         return directive
 807
 808     # Handle decorators
 809     def visit_FuncDefNode(self, node):
 810         directives = self._extract_directives(node, 'function')
 811         if not directives:
 812             return self.visit_Node(node)
 813         body = Nodes.StatListNode(node.pos, stats=[node])
 814         return self.visit_with_directives(body, directives)
 815
 816     def visit_CVarDefNode(self, node):
 817         if not node.decorators:
 818             return node
 819         for dec in node.decorators:
 820             for directive in self.try_to_parse_directives(dec.decorator) or ():
 821                 if directive is not None and directive[0] == u'locals':
 822                     node.directive_locals = directive[1]
 823                 else:
 824                     self.context.nonfatal_error(PostParseError(dec.pos,
 825                         "Cdef functions can only take cython.locals() decorator."))
 826         return node
 827
 828     def visit_CClassDefNode(self, node):
 829         directives = self._extract_directives(node, 'cclass')
 830         if not directives:
 831             return self.visit_Node(node)
 832         body = Nodes.StatListNode(node.pos, stats=[node])
 833         return self.visit_with_directives(body, directives)
 834
 835     def visit_PyClassDefNode(self, node):
 836         directives = self._extract_directives(node, 'class')
 837         if not directives:
 838             return self.visit_Node(node)
 839         body = Nodes.StatListNode(node.pos, stats=[node])
 840         return self.visit_with_directives(body, directives)
 841
 842     def _extract_directives(self, node, scope_name):
 843         if not node.decorators:
 844             return {}
 845         # Split the decorators into two lists -- real decorators and directives
 846         directives = []
 847         realdecs = []
 848         for dec in node.decorators:
 849             new_directives = self.try_to_parse_directives(dec.decorator)
 850             if new_directives is not None:
 851                 for directive in new_directives:
 852                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 853                         directives.append(directive)
 854             else:
 855                 realdecs.append(dec)
 856         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 857             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 858         else:
 859             node.decorators = realdecs
 860         # merge or override repeated directives
 861         optdict = {}
 862         directives.reverse() # Decorators coming first take precedence
 863         for directive in directives:
 864             name, value = directive
 865             if name in optdict:
 866                 old_value = optdict[name]
 867                 # keywords and arg lists can be merged, everything
 868                 # else overrides completely
 869                 if isinstance(old_value, dict):
 870                     old_value.update(value)
 871                 elif isinstance(old_value, list):
 872                     old_value.extend(value)
 873                 else:
 874                     optdict[name] = value
 875             else:
 876                 optdict[name] = value
 877         return optdict
 878
 879     # Handle with statements
 880     def visit_WithStatNode(self, node):
 881         directive_dict = {}
 882         for directive in self.try_to_parse_directives(node.manager) or []:
 883             if directive is not None:
 884                 if node.target is not None:
 885                     self.context.nonfatal_error(
 886                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 887                 else:
 888                     name, value = directive
 889                     if name == 'nogil':
 890                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 891                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 892                         return self.visit_Node(node)
 893                     if self.check_directive_scope(node.pos, name, 'with statement'):
 894                         directive_dict[name] = value
 895         if directive_dict:
 896             return self.visit_with_directives(node.body, directive_dict)
 897         return self.visit_Node(node)
 898
 899 class WithTransform(CythonTransform, SkipDeclarations):
 900
 901     # EXCINFO is manually set to a variable that contains
 902     # the exc_info() tuple that can be generated by the enclosing except
 903     # statement.
 904     template_without_target = TreeFragment(u"""
 905         MGR = EXPR
 906         EXIT = MGR.__exit__
 907         MGR.__enter__()
 908         EXC = True
 909         try:
 910             try:
 911                 EXCINFO = None
 912                 BODY
 913             except:
 914                 EXC = False
 915                 if not EXIT(*EXCINFO):
 916                     raise
 917         finally:
 918             if EXC:
 919                 EXIT(None, None, None)
 920     """, temps=[u'MGR', u'EXC', u"EXIT"],
 921     pipeline=[NormalizeTree(None)])
 922
 923     template_with_target = TreeFragment(u"""
 924         MGR = EXPR
 925         EXIT = MGR.__exit__
 926         VALUE = MGR.__enter__()
 927         EXC = True
 928         try:
 929             try:
 930                 EXCINFO = None
 931                 TARGET = VALUE
 932                 BODY
 933             except:
 934                 EXC = False
 935                 if not EXIT(*EXCINFO):
 936                     raise
 937         finally:
 938             if EXC:
 939                 EXIT(None, None, None)
 940             MGR = EXIT = VALUE = EXC = None
 941
 942     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 943     pipeline=[NormalizeTree(None)])
 944
 945     def visit_WithStatNode(self, node):
 946         # TODO: Cleanup badly needed
 947         TemplateTransform.temp_name_counter += 1
 948         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 949
 950         self.visitchildren(node, ['body'])
 951         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 952         if node.target is not None:
 953             result = self.template_with_target.substitute({
 954                 u'EXPR' : node.manager,
 955                 u'BODY' : node.body,
 956                 u'TARGET' : node.target,
 957                 u'EXCINFO' : excinfo_temp
 958                 }, pos=node.pos)
 959         else:
 960             result = self.template_without_target.substitute({
 961                 u'EXPR' : node.manager,
 962                 u'BODY' : node.body,
 963                 u'EXCINFO' : excinfo_temp
 964                 }, pos=node.pos)
 965
 966         # Set except excinfo target to EXCINFO
 967         try_except = result.stats[-1].body.stats[-1]
 968         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 969 #            excinfo_temp.ref(node.pos))
 970
 971 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 972 #            node.pos, temps=[excinfo_temp], body=try_except)
 973
 974         return result
 975
 976     def visit_ExprNode(self, node):
 977         # With statements are never inside expressions.
 978         return node
 979
 980
 981 class DecoratorTransform(CythonTransform, SkipDeclarations):
 982
 983     def visit_DefNode(self, func_node):
 984         self.visitchildren(func_node)
 985         if not func_node.decorators:
 986             return func_node
 987         return self._handle_decorators(
 988             func_node, func_node.name)
 989
 990     def visit_CClassDefNode(self, class_node):
 991         # This doesn't currently work, so it's disabled.
 992         #
 993         # Problem: assignments to cdef class names do not work.  They
 994         # would require an additional check anyway, as the extension
 995         # type must not change its C type, so decorators cannot
 996         # replace an extension type, just alter it and return it.
 997
 998         self.visitchildren(class_node)
 999         if not class_node.decorators:
1000             return class_node
1001         error(class_node.pos,
1002               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
1003         return class_node
1004         #return self._handle_decorators(
1005         #    class_node, class_node.class_name)
1006
1007     def visit_ClassDefNode(self, class_node):
1008         self.visitchildren(class_node)
1009         if not class_node.decorators:
1010             return class_node
1011         return self._handle_decorators(
1012             class_node, class_node.name)
1013
1014     def _handle_decorators(self, node, name):
1015         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1016         for decorator in node.decorators[::-1]:
1017             decorator_result = ExprNodes.SimpleCallNode(
1018                 decorator.pos,
1019                 function = decorator.decorator,
1020                 args = [decorator_result])
1021
1022         name_node = ExprNodes.NameNode(node.pos, name = name)
1023         reassignment = Nodes.SingleAssignmentNode(
1024             node.pos,
1025             lhs = name_node,
1026             rhs = decorator_result)
1027         return [node, reassignment]
1028
1029
1030 class AnalyseDeclarationsTransform(CythonTransform):
1031
1032     basic_property = TreeFragment(u"""
1033 property NAME:
1034     def __get__(self):
1035         return ATTR
1036     def __set__(self, value):
1037         ATTR = value
1038     """, level='c_class')
1039     basic_pyobject_property = TreeFragment(u"""
1040 property NAME:
1041     def __get__(self):
1042         return ATTR
1043     def __set__(self, value):
1044         ATTR = value
1045     def __del__(self):
1046         ATTR = None
1047     """, level='c_class')
1048     basic_property_ro = TreeFragment(u"""
1049 property NAME:
1050     def __get__(self):
1051         return ATTR
1052     """, level='c_class')
1053
1054     struct_or_union_wrapper = TreeFragment(u"""
1055 cdef class NAME:
1056     cdef TYPE value
1057     def __init__(self, MEMBER=None):
1058         cdef int count
1059         count = 0
1060         INIT_ASSIGNMENTS
1061         if IS_UNION and count > 1:
1062             raise ValueError, "At most one union member should be specified."
1063     def __str__(self):
1064         return STR_FORMAT % MEMBER_TUPLE
1065     def __repr__(self):
1066         return REPR_FORMAT % MEMBER_TUPLE
1067     """)
1068
1069     init_assignment = TreeFragment(u"""
1070 if VALUE is not None:
1071     ATTR = VALUE
1072     count += 1
1073     """)
1074
1075     def __call__(self, root):
1076         self.env_stack = [root.scope]
1077         # needed to determine if a cdef var is declared after it's used.
1078         self.seen_vars_stack = []
1079         return super(AnalyseDeclarationsTransform, self).__call__(root)
1080
1081     def visit_NameNode(self, node):
1082         self.seen_vars_stack[-1].add(node.name)
1083         return node
1084
1085     def visit_ModuleNode(self, node):
1086         self.seen_vars_stack.append(cython.set())
1087         node.analyse_declarations(self.env_stack[-1])
1088         self.visitchildren(node)
1089         self.seen_vars_stack.pop()
1090         return node
1091
1092     def visit_LambdaNode(self, node):
1093         node.analyse_declarations(self.env_stack[-1])
1094         self.visitchildren(node)
1095         return node
1096
1097     def visit_ClassDefNode(self, node):
1098         self.env_stack.append(node.scope)
1099         self.visitchildren(node)
1100         self.env_stack.pop()
1101         return node
1102
1103     def visit_CClassDefNode(self, node):
1104         node = self.visit_ClassDefNode(node)
1105         if node.scope and node.scope.implemented:
1106             stats = []
1107             for entry in node.scope.var_entries:
1108                 if entry.needs_property:
1109                     property = self.create_Property(entry)
1110                     property.analyse_declarations(node.scope)
1111                     self.visit(property)
1112                     stats.append(property)
1113             if stats:
1114                 node.body.stats += stats
1115         return node
1116
1117     def visit_FuncDefNode(self, node):
1118         self.seen_vars_stack.append(cython.set())
1119         lenv = node.local_scope
1120         node.body.analyse_control_flow(lenv) # this will be totally refactored
1121         node.declare_arguments(lenv)
1122         for var, type_node in node.directive_locals.items():
1123             if not lenv.lookup_here(var):   # don't redeclare args
1124                 type = type_node.analyse_as_type(lenv)
1125                 if type:
1126                     lenv.declare_var(var, type, type_node.pos)
1127                 else:
1128                     error(type_node.pos, "Not a type")
1129         node.body.analyse_declarations(lenv)
1130         self.env_stack.append(lenv)
1131         self.visitchildren(node)
1132         self.env_stack.pop()
1133         self.seen_vars_stack.pop()
1134         return node
1135
1136     def visit_ScopedExprNode(self, node):
1137         env = self.env_stack[-1]
1138         node.analyse_declarations(env)
1139         # the node may or may not have a local scope
1140         if node.has_local_scope:
1141             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1142             self.env_stack.append(node.expr_scope)
1143             node.analyse_scoped_declarations(node.expr_scope)
1144             self.visitchildren(node)
1145             self.env_stack.pop()
1146             self.seen_vars_stack.pop()
1147         else:
1148             node.analyse_scoped_declarations(env)
1149             self.visitchildren(node)
1150         return node
1151
1152     def visit_TempResultFromStatNode(self, node):
1153         self.visitchildren(node)
1154         node.analyse_declarations(self.env_stack[-1])
1155         return node
1156
1157     def visit_CStructOrUnionDefNode(self, node):
1158         # Create a wrapper node if needed.
1159         # We want to use the struct type information (so it can't happen
1160         # before this phase) but also create new objects to be declared
1161         # (so it can't happen later).
1162         # Note that we don't return the original node, as it is
1163         # never used after this phase.
1164         if True: # private (default)
1165             return None
1166
1167         self_value = ExprNodes.AttributeNode(
1168             pos = node.pos,
1169             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1170             attribute = EncodedString(u"value"))
1171         var_entries = node.entry.type.scope.var_entries
1172         attributes = []
1173         for entry in var_entries:
1174             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1175                                                       obj = self_value,
1176                                                       attribute = entry.name))
1177         # __init__ assignments
1178         init_assignments = []
1179         for entry, attr in zip(var_entries, attributes):
1180             # TODO: branch on visibility
1181             init_assignments.append(self.init_assignment.substitute({
1182                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1183                     u"ATTR": attr,
1184                 }, pos = entry.pos))
1185
1186         # create the class
1187         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1188         wrapper_class = self.struct_or_union_wrapper.substitute({
1189             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1190             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1191             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1192             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1193             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1194         }, pos = node.pos).stats[0]
1195         wrapper_class.class_name = node.name
1196         wrapper_class.shadow = True
1197         class_body = wrapper_class.body.stats
1198
1199         # fix value type
1200         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1201         class_body[0].base_type.name = node.name
1202
1203         # fix __init__ arguments
1204         init_method = class_body[1]
1205         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1206         arg_template = init_method.args[1]
1207         if not node.entry.type.is_struct:
1208             arg_template.kw_only = True
1209         del init_method.args[1]
1210         for entry, attr in zip(var_entries, attributes):
1211             arg = copy.deepcopy(arg_template)
1212             arg.declarator.name = entry.name
1213             init_method.args.append(arg)
1214
1215         # setters/getters
1216         for entry, attr in zip(var_entries, attributes):
1217             # TODO: branch on visibility
1218             if entry.type.is_pyobject:
1219                 template = self.basic_pyobject_property
1220             else:
1221                 template = self.basic_property
1222             property = template.substitute({
1223                     u"ATTR": attr,
1224                 }, pos = entry.pos).stats[0]
1225             property.name = entry.name
1226             wrapper_class.body.stats.append(property)
1227
1228         wrapper_class.analyse_declarations(self.env_stack[-1])
1229         return self.visit_CClassDefNode(wrapper_class)
1230
1231     # Some nodes are no longer needed after declaration
1232     # analysis and can be dropped. The analysis was performed
1233     # on these nodes in a seperate recursive process from the
1234     # enclosing function or module, so we can simply drop them.
1235     def visit_CDeclaratorNode(self, node):
1236         # necessary to ensure that all CNameDeclaratorNodes are visited.
1237         self.visitchildren(node)
1238         return node
1239
1240     def visit_CTypeDefNode(self, node):
1241         return node
1242
1243     def visit_CBaseTypeNode(self, node):
1244         return None
1245
1246     def visit_CEnumDefNode(self, node):
1247         if node.visibility == 'public':
1248             return node
1249         else:
1250             return None
1251
1252     def visit_CNameDeclaratorNode(self, node):
1253         if node.name in self.seen_vars_stack[-1]:
1254             entry = self.env_stack[-1].lookup(node.name)
1255             if (entry is None or entry.visibility != 'extern'
1256                 and not entry.scope.is_c_class_scope):
1257                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1258         self.visitchildren(node)
1259         return node
1260
1261     def visit_CVarDefNode(self, node):
1262         # to ensure all CNameDeclaratorNodes are visited.
1263         self.visitchildren(node)
1264         return None
1265
1266     def create_Property(self, entry):
1267         if entry.visibility == 'public':
1268             if entry.type.is_pyobject:
1269                 template = self.basic_pyobject_property
1270             else:
1271                 template = self.basic_property
1272         elif entry.visibility == 'readonly':
1273             template = self.basic_property_ro
1274         property = template.substitute({
1275                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1276                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1277                                                  attribute=entry.name),
1278             }, pos=entry.pos).stats[0]
1279         property.name = entry.name
1280         # ---------------------------------------
1281         # XXX This should go to AutoDocTransforms
1282         # ---------------------------------------
1283         if (Options.docstrings and
1284             self.current_directives['embedsignature']):
1285             attr_name = entry.name
1286             type_name = entry.type.declaration_code("", for_display=1)
1287             default_value = ''
1288             if not entry.type.is_pyobject:
1289                 type_name = "'%s'" % type_name
1290             elif entry.type.is_extension_type:
1291                 type_name = entry.type.module_name + '.' + type_name
1292             if entry.init is not None:
1293                 default_value = ' = ' + entry.init
1294             elif entry.init_to_none:
1295                 default_value = ' = ' + repr(None)
1296             docstring = attr_name + ': ' + type_name + default_value
1297             property.doc = EncodedString(docstring)
1298         # ---------------------------------------
1299         return property
1300
1301 class AnalyseExpressionsTransform(CythonTransform):
1302
1303     def visit_ModuleNode(self, node):
1304         node.scope.infer_types()
1305         node.body.analyse_expressions(node.scope)
1306         self.visitchildren(node)
1307         return node
1308
1309     def visit_FuncDefNode(self, node):
1310         node.local_scope.infer_types()
1311         node.body.analyse_expressions(node.local_scope)
1312         self.visitchildren(node)
1313         return node
1314
1315     def visit_ScopedExprNode(self, node):
1316         if node.has_local_scope:
1317             node.expr_scope.infer_types()
1318             node.analyse_scoped_expressions(node.expr_scope)
1319         self.visitchildren(node)
1320         return node
1321
1322 class ExpandInplaceOperators(EnvTransform):
1323
1324     def visit_InPlaceAssignmentNode(self, node):
1325         lhs = node.lhs
1326         rhs = node.rhs
1327         if lhs.type.is_cpp_class:
1328             # No getting around this exact operator here.
1329             return node
1330         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1331             # There is code to handle this case.
1332             return node
1333
1334         env = self.current_env()
1335         def side_effect_free_reference(node, setting=False):
1336             if isinstance(node, ExprNodes.NameNode):
1337                 return node, []
1338             elif node.type.is_pyobject and not setting:
1339                 node = LetRefNode(node)
1340                 return node, [node]
1341             elif isinstance(node, ExprNodes.IndexNode):
1342                 if node.is_buffer_access:
1343                     raise ValueError, "Buffer access"
1344                 base, temps = side_effect_free_reference(node.base)
1345                 index = LetRefNode(node.index)
1346                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1347             elif isinstance(node, ExprNodes.AttributeNode):
1348                 obj, temps = side_effect_free_reference(node.obj)
1349                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1350             else:
1351                 node = LetRefNode(node)
1352                 return node, [node]
1353         try:
1354             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1355         except ValueError:
1356             return node
1357         dup = lhs.__class__(**lhs.__dict__)
1358         binop = ExprNodes.binop_node(node.pos,
1359                                      operator = node.operator,
1360                                      operand1 = dup,
1361                                      operand2 = rhs,
1362                                      inplace=True)
1363         # Manually analyse types for new node.
1364         lhs.analyse_target_types(env)
1365         dup.analyse_types(env)
1366         binop.analyse_operation(env)
1367         node = Nodes.SingleAssignmentNode(
1368             node.pos,
1369             lhs = lhs,
1370             rhs=binop.coerce_to(lhs.type, env))
1371         # Use LetRefNode to avoid side effects.
1372         let_ref_nodes.reverse()
1373         for t in let_ref_nodes:
1374             node = LetNode(t, node)
1375         return node
1376
1377     def visit_ExprNode(self, node):
1378         # In-place assignments can't happen within an expression.
1379         return node
1380
1381
1382 class AlignFunctionDefinitions(CythonTransform):
1383     """
1384     This class takes the signatures from a .pxd file and applies them to
1385     the def methods in a .py file.
1386     """
1387
1388     def visit_ModuleNode(self, node):
1389         self.scope = node.scope
1390         self.directives = node.directives
1391         self.visitchildren(node)
1392         return node
1393
1394     def visit_PyClassDefNode(self, node):
1395         pxd_def = self.scope.lookup(node.name)
1396         if pxd_def:
1397             if pxd_def.is_cclass:
1398                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1399             else:
1400                 error(node.pos, "'%s' redeclared" % node.name)
1401                 if pxd_def.pos:
1402                     error(pxd_def.pos, "previous declaration here")
1403                 return None
1404         else:
1405             return node
1406
1407     def visit_CClassDefNode(self, node, pxd_def=None):
1408         if pxd_def is None:
1409             pxd_def = self.scope.lookup(node.class_name)
1410         if pxd_def:
1411             outer_scope = self.scope
1412             self.scope = pxd_def.type.scope
1413         self.visitchildren(node)
1414         if pxd_def:
1415             self.scope = outer_scope
1416         return node
1417
1418     def visit_DefNode(self, node):
1419         pxd_def = self.scope.lookup(node.name)
1420         if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
1421             if not pxd_def.is_cfunction:
1422                 error(node.pos, "'%s' redeclared" % node.name)
1423                 if pxd_def.pos:
1424                     error(pxd_def.pos, "previous declaration here")
1425                 return None
1426             node = node.as_cfunction(pxd_def)
1427         elif (self.scope.is_module_scope and self.directives['auto_cpdef']
1428               and node.is_cdef_func_compatible()):
1429             node = node.as_cfunction(scope=self.scope)
1430         # Enable this when nested cdef functions are allowed.
1431         # self.visitchildren(node)
1432         return node
1433
1434
1435 class YieldNodeCollector(TreeVisitor):
1436
1437     def __init__(self):
1438         super(YieldNodeCollector, self).__init__()
1439         self.yields = []
1440         self.returns = []
1441         self.has_return_value = False
1442
1443     def visit_Node(self, node):
1444         return self.visitchildren(node)
1445
1446     def visit_YieldExprNode(self, node):
1447         if self.has_return_value:
1448             error(node.pos, "'yield' outside function")
1449         self.yields.append(node)
1450         self.visitchildren(node)
1451
1452     def visit_ReturnStatNode(self, node):
1453         if node.value:
1454             self.has_return_value = True
1455             if self.yields:
1456                 error(node.pos, "'return' with argument inside generator")
1457         self.returns.append(node)
1458
1459     def visit_ClassDefNode(self, node):
1460         pass
1461
1462     def visit_FuncDefNode(self, node):
1463         pass
1464
1465     def visit_LambdaNode(self, node):
1466         pass
1467
1468     def visit_GeneratorExpressionNode(self, node):
1469         pass
1470
1471 class MarkClosureVisitor(CythonTransform):
1472
1473     def visit_ModuleNode(self, node):
1474         self.needs_closure = False
1475         self.visitchildren(node)
1476         return node
1477
1478     def visit_FuncDefNode(self, node):
1479         self.needs_closure = False
1480         self.visitchildren(node)
1481         node.needs_closure = self.needs_closure
1482         self.needs_closure = True
1483
1484         collector = YieldNodeCollector()
1485         collector.visitchildren(node)
1486
1487         if collector.yields:
1488             for i, yield_expr in enumerate(collector.yields):
1489                 yield_expr.label_num = i + 1
1490
1491             gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1492                                                name=node.name,
1493                                                body=node.body)
1494             generator = Nodes.GeneratorDefNode(pos=node.pos,
1495                                                name=node.name,
1496                                                args=node.args,
1497                                                star_arg=node.star_arg,
1498                                                starstar_arg=node.starstar_arg,
1499                                                doc=node.doc,
1500                                                decorators=node.decorators,
1501                                                gbody=gbody,
1502                                                lambda_name=node.lambda_name)
1503             return generator
1504         return node
1505
1506     def visit_CFuncDefNode(self, node):
1507         self.visit_FuncDefNode(node)
1508         if node.needs_closure:
1509             error(node.pos, "closures inside cdef functions not yet supported")
1510         return node
1511
1512     def visit_LambdaNode(self, node):
1513         self.needs_closure = False
1514         self.visitchildren(node)
1515         node.needs_closure = self.needs_closure
1516         self.needs_closure = True
1517         return node
1518
1519     def visit_ClassDefNode(self, node):
1520         self.visitchildren(node)
1521         self.needs_closure = True
1522         return node
1523
1524 class CreateClosureClasses(CythonTransform):
1525     # Output closure classes in module scope for all functions
1526     # that really need it.
1527
1528     def __init__(self, context):
1529         super(CreateClosureClasses, self).__init__(context)
1530         self.path = []
1531         self.in_lambda = False
1532         self.generator_class = None
1533
1534     def visit_ModuleNode(self, node):
1535         self.module_scope = node.scope
1536         self.visitchildren(node)
1537         return node
1538
1539     def create_generator_class(self, target_module_scope, pos):
1540         if self.generator_class:
1541             return self.generator_class
1542         # XXX: make generator class creation cleaner
1543         entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1544                     objstruct_cname='__pyx_Generator_object',
1545                     typeobj_cname='__pyx_Generator_type',
1546                     pos=pos, defining=True, implementing=True)
1547         klass = entry.type.scope
1548         klass.is_internal = True
1549         klass.directives = {'final': True}
1550
1551         body_type = PyrexTypes.create_typedef_type('generator_body',
1552                                                    PyrexTypes.c_void_ptr_type,
1553                                                    '__pyx_generator_body_t')
1554         klass.declare_var(pos=pos, name='body', cname='body',
1555                           type=body_type, is_cdef=True)
1556         klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1557                           is_cdef=True)
1558         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1559                           is_cdef=True)
1560         klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
1561                           type=PyrexTypes.py_object_type, is_cdef=True)
1562         klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
1563                           type=PyrexTypes.py_object_type, is_cdef=True)
1564         klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
1565                           type=PyrexTypes.py_object_type, is_cdef=True)
1566
1567         import TypeSlots
1568         e = klass.declare_pyfunction('send', pos)
1569         e.func_cname = '__Pyx_Generator_Send'
1570         e.signature = TypeSlots.binaryfunc
1571
1572         e = klass.declare_pyfunction('close', pos)
1573         e.func_cname = '__Pyx_Generator_Close'
1574         e.signature = TypeSlots.unaryfunc
1575
1576         e = klass.declare_pyfunction('throw', pos)
1577         e.func_cname = '__Pyx_Generator_Throw'
1578         e.signature = TypeSlots.pyfunction_signature
1579
1580         e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1581         e.func_cname = 'PyObject_SelfIter'
1582
1583         e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1584         e.func_cname = '__Pyx_Generator_Next'
1585
1586         self.generator_class = entry.type
1587         return self.generator_class
1588
1589     def find_entries_used_in_closures(self, node):
1590         from_closure = []
1591         in_closure = []
1592         for name, entry in node.local_scope.entries.items():
1593             if entry.from_closure:
1594                 from_closure.append((name, entry))
1595             elif entry.in_closure:
1596                 in_closure.append((name, entry))
1597         return from_closure, in_closure
1598
1599     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1600         # skip generator body
1601         if node.is_generator_body:
1602             return
1603         # move local variables into closure
1604         if node.is_generator:
1605             for entry in node.local_scope.entries.values():
1606                 if not entry.from_closure:
1607                     entry.in_closure = True
1608
1609         from_closure, in_closure = self.find_entries_used_in_closures(node)
1610         in_closure.sort()
1611
1612         # Now from the begining
1613         node.needs_closure = False
1614         node.needs_outer_scope = False
1615
1616         func_scope = node.local_scope
1617         cscope = node.entry.scope
1618         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1619             cscope = cscope.outer_scope
1620
1621         if not from_closure and (self.path or inner_node):
1622             if not inner_node:
1623                 if not node.assmt:
1624                     raise InternalError, "DefNode does not have assignment node"
1625                 inner_node = node.assmt.rhs
1626             inner_node.needs_self_code = False
1627             node.needs_outer_scope = False
1628
1629         base_type = None
1630         if node.is_generator:
1631             base_type = self.create_generator_class(target_module_scope, node.pos)
1632         elif not in_closure and not from_closure:
1633             return
1634         elif not in_closure:
1635             func_scope.is_passthrough = True
1636             func_scope.scope_class = cscope.scope_class
1637             node.needs_outer_scope = True
1638             return
1639
1640         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1641
1642         entry = target_module_scope.declare_c_class(
1643             name=as_name, pos=node.pos, defining=True,
1644             implementing=True, base_type=base_type)
1645
1646         func_scope.scope_class = entry
1647         class_scope = entry.type.scope
1648         class_scope.is_internal = True
1649         class_scope.directives = {'final': True}
1650
1651         if from_closure:
1652             assert cscope.is_closure_scope
1653             class_scope.declare_var(pos=node.pos,
1654                                     name=Naming.outer_scope_cname,
1655                                     cname=Naming.outer_scope_cname,
1656                                     type=cscope.scope_class.type,
1657                                     is_cdef=True)
1658             node.needs_outer_scope = True
1659         for name, entry in in_closure:
1660             closure_entry = class_scope.declare_var(pos=entry.pos,
1661                                     name=entry.name,
1662                                     cname=entry.cname,
1663                                     type=entry.type,
1664                                     is_cdef=True)
1665             if entry.is_declared_generic:
1666                 closure_entry.is_declared_generic = 1
1667         node.needs_closure = True
1668         # Do it here because other classes are already checked
1669         target_module_scope.check_c_class(func_scope.scope_class)
1670
1671     def visit_LambdaNode(self, node):
1672         was_in_lambda = self.in_lambda
1673         self.in_lambda = True
1674         self.create_class_from_scope(node.def_node, self.module_scope, node)
1675         self.visitchildren(node)
1676         self.in_lambda = was_in_lambda
1677         return node
1678
1679     def visit_FuncDefNode(self, node):
1680         if self.in_lambda:
1681             self.visitchildren(node)
1682             return node
1683         if node.needs_closure or self.path:
1684             self.create_class_from_scope(node, self.module_scope)
1685             self.path.append(node)
1686             self.visitchildren(node)
1687             self.path.pop()
1688         return node
1689
1690
1691 class GilCheck(VisitorTransform):
1692     """
1693     Call `node.gil_check(env)` on each node to make sure we hold the
1694     GIL when we need it.  Raise an error when on Python operations
1695     inside a `nogil` environment.
1696     """
1697     def __call__(self, root):
1698         self.env_stack = [root.scope]
1699         self.nogil = False
1700         return super(GilCheck, self).__call__(root)
1701
1702     def visit_FuncDefNode(self, node):
1703         self.env_stack.append(node.local_scope)
1704         was_nogil = self.nogil
1705         self.nogil = node.local_scope.nogil
1706         if self.nogil and node.nogil_check:
1707             node.nogil_check(node.local_scope)
1708         self.visitchildren(node)
1709         self.env_stack.pop()
1710         self.nogil = was_nogil
1711         return node
1712
1713     def visit_GILStatNode(self, node):
1714         env = self.env_stack[-1]
1715         if self.nogil and node.nogil_check: node.nogil_check()
1716         was_nogil = self.nogil
1717         self.nogil = (node.state == 'nogil')
1718         self.visitchildren(node)
1719         self.nogil = was_nogil
1720         return node
1721
1722     def visit_Node(self, node):
1723         if self.env_stack and self.nogil and node.nogil_check:
1724             node.nogil_check(self.env_stack[-1])
1725         self.visitchildren(node)
1726         return node
1727
1728
1729 class TransformBuiltinMethods(EnvTransform):
1730
1731     def visit_SingleAssignmentNode(self, node):
1732         if node.declaration_only:
1733             return None
1734         else:
1735             self.visitchildren(node)
1736             return node
1737
1738     def visit_AttributeNode(self, node):
1739         self.visitchildren(node)
1740         return self.visit_cython_attribute(node)
1741
1742     def visit_NameNode(self, node):
1743         return self.visit_cython_attribute(node)
1744
1745     def visit_cython_attribute(self, node):
1746         attribute = node.as_cython_attribute()
1747         if attribute:
1748             if attribute == u'compiled':
1749                 node = ExprNodes.BoolNode(node.pos, value=True)
1750             elif attribute == u'NULL':
1751                 node = ExprNodes.NullNode(node.pos)
1752             elif attribute in (u'set', u'frozenset'):
1753                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1754                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1755             elif not PyrexTypes.parse_basic_type(attribute):
1756                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1757         return node
1758
1759     def _inject_locals(self, node, func_name):
1760         # locals()/dir() builtins
1761         lenv = self.current_env()
1762         entry = lenv.lookup_here(func_name)
1763         if entry:
1764             # not the builtin
1765             return node
1766         pos = node.pos
1767         if func_name == 'locals':
1768             if len(node.args) > 0:
1769                 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
1770                       % len(node.args))
1771                 return node
1772             items = [ ExprNodes.DictItemNode(pos,
1773                                              key=ExprNodes.StringNode(pos, value=var),
1774                                              value=ExprNodes.NameNode(pos, name=var))
1775                       for var in lenv.entries ]
1776             return ExprNodes.DictNode(pos, key_value_pairs=items)
1777         else:
1778             if len(node.args) > 1:
1779                 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
1780                       % len(node.args))
1781                 return node
1782             elif len(node.args) == 1:
1783                 # optimised in Builtin.py
1784                 return node
1785             items = [ ExprNodes.StringNode(pos, value=var) for var in lenv.entries ]
1786             return ExprNodes.ListNode(pos, args=items)
1787
1788     def visit_SimpleCallNode(self, node):
1789         if isinstance(node.function, ExprNodes.NameNode):
1790             func_name = node.function.name
1791             if func_name in ('dir', 'locals'):
1792                 return self._inject_locals(node, func_name)
1793
1794         # cython.foo
1795         function = node.function.as_cython_attribute()
1796         if function:
1797             if function in InterpretCompilerDirectives.unop_method_nodes:
1798                 if len(node.args) != 1:
1799                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1800                 else:
1801                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1802             elif function in InterpretCompilerDirectives.binop_method_nodes:
1803                 if len(node.args) != 2:
1804                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1805                 else:
1806                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1807             elif function == u'cast':
1808                 if len(node.args) != 2:
1809                     error(node.function.pos, u"cast() takes exactly two arguments")
1810                 else:
1811                     type = node.args[0].analyse_as_type(self.current_env())
1812                     if type:
1813                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1814                     else:
1815                         error(node.args[0].pos, "Not a type")
1816             elif function == u'sizeof':
1817                 if len(node.args) != 1:
1818                     error(node.function.pos, u"sizeof() takes exactly one argument")
1819                 else:
1820                     type = node.args[0].analyse_as_type(self.current_env())
1821                     if type:
1822                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1823                     else:
1824                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1825             elif function == 'cmod':
1826                 if len(node.args) != 2:
1827                     error(node.function.pos, u"cmod() takes exactly two arguments")
1828                 else:
1829                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1830                     node.cdivision = True
1831             elif function == 'cdiv':
1832                 if len(node.args) != 2:
1833                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1834                 else:
1835                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1836                     node.cdivision = True
1837             elif function == u'set':
1838                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1839             else:
1840                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1841
1842         self.visitchildren(node)
1843         return node
1844
1845
1846 class DebugTransform(CythonTransform):
1847     """
1848     Create debug information and all functions' visibility to extern in order
1849     to enable debugging.
1850     """
1851
1852     def __init__(self, context, options, result):
1853         super(DebugTransform, self).__init__(context)
1854         self.visited = cython.set()
1855         # our treebuilder and debug output writer
1856         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1857         self.tb = self.context.gdb_debug_outputwriter
1858         #self.c_output_file = options.output_file
1859         self.c_output_file = result.c_file
1860
1861         # Closure support, basically treat nested functions as if the AST were
1862         # never nested
1863         self.nested_funcdefs = []
1864
1865         # tells visit_NameNode whether it should register step-into functions
1866         self.register_stepinto = False
1867
1868     def visit_ModuleNode(self, node):
1869         self.tb.module_name = node.full_module_name
1870         attrs = dict(
1871             module_name=node.full_module_name,
1872             filename=node.pos[0].filename,
1873             c_filename=self.c_output_file)
1874
1875         self.tb.start('Module', attrs)
1876
1877         # serialize functions
1878         self.tb.start('Functions')
1879         # First, serialize functions normally...
1880         self.visitchildren(node)
1881
1882         # ... then, serialize nested functions
1883         for nested_funcdef in self.nested_funcdefs:
1884             self.visit_FuncDefNode(nested_funcdef)
1885
1886         self.register_stepinto = True
1887         self.serialize_modulenode_as_function(node)
1888         self.register_stepinto = False
1889         self.tb.end('Functions')
1890
1891         # 2.3 compatibility. Serialize global variables
1892         self.tb.start('Globals')
1893         entries = {}
1894
1895         for k, v in node.scope.entries.iteritems():
1896             if (v.qualified_name not in self.visited and not
1897                 v.name.startswith('__pyx_') and not
1898                 v.type.is_cfunction and not
1899                 v.type.is_extension_type):
1900                 entries[k]= v
1901
1902         self.serialize_local_variables(entries)
1903         self.tb.end('Globals')
1904         # self.tb.end('Module') # end Module after the line number mapping in
1905         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1906         return node
1907
1908     def visit_FuncDefNode(self, node):
1909         self.visited.add(node.local_scope.qualified_name)
1910
1911         if getattr(node, 'is_wrapper', False):
1912             return node
1913
1914         if self.register_stepinto:
1915             self.nested_funcdefs.append(node)
1916             return node
1917
1918         # node.entry.visibility = 'extern'
1919         if node.py_func is None:
1920             pf_cname = ''
1921         else:
1922             pf_cname = node.py_func.entry.func_cname
1923
1924         attrs = dict(
1925             name=node.entry.name,
1926             cname=node.entry.func_cname,
1927             pf_cname=pf_cname,
1928             qualified_name=node.local_scope.qualified_name,
1929             lineno=str(node.pos[1]))
1930
1931         self.tb.start('Function', attrs=attrs)
1932
1933         self.tb.start('Locals')
1934         self.serialize_local_variables(node.local_scope.entries)
1935         self.tb.end('Locals')
1936
1937         self.tb.start('Arguments')
1938         for arg in node.local_scope.arg_entries:
1939             self.tb.start(arg.name)
1940             self.tb.end(arg.name)
1941         self.tb.end('Arguments')
1942
1943         self.tb.start('StepIntoFunctions')
1944         self.register_stepinto = True
1945         self.visitchildren(node)
1946         self.register_stepinto = False
1947         self.tb.end('StepIntoFunctions')
1948         self.tb.end('Function')
1949
1950         return node
1951
1952     def visit_NameNode(self, node):
1953         if (self.register_stepinto and
1954             node.type.is_cfunction and
1955             getattr(node, 'is_called', False) and
1956             node.entry.func_cname is not None):
1957             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1958             # declared functions are not 'in_cinclude'.
1959             # This means we will list called 'cdef' functions as
1960             # "step into functions", but this is not an issue as they will be
1961             # recognized as Cython functions anyway.
1962             attrs = dict(name=node.entry.func_cname)
1963             self.tb.start('StepIntoFunction', attrs=attrs)
1964             self.tb.end('StepIntoFunction')
1965
1966         self.visitchildren(node)
1967         return node
1968
1969     def serialize_modulenode_as_function(self, node):
1970         """
1971         Serialize the module-level code as a function so the debugger will know
1972         it's a "relevant frame" and it will know where to set the breakpoint
1973         for 'break modulename'.
1974         """
1975         name = node.full_module_name.rpartition('.')[-1]
1976
1977         cname_py2 = 'init' + name
1978         cname_py3 = 'PyInit_' + name
1979
1980         py2_attrs = dict(
1981             name=name,
1982             cname=cname_py2,
1983             pf_cname='',
1984             # Ignore the qualified_name, breakpoints should be set using
1985             # `cy break modulename:lineno` for module-level breakpoints.
1986             qualified_name='',
1987             lineno='1',
1988             is_initmodule_function="True",
1989         )
1990
1991         py3_attrs = dict(py2_attrs, cname=cname_py3)
1992
1993         self._serialize_modulenode_as_function(node, py2_attrs)
1994         self._serialize_modulenode_as_function(node, py3_attrs)
1995
1996     def _serialize_modulenode_as_function(self, node, attrs):
1997         self.tb.start('Function', attrs=attrs)
1998
1999         self.tb.start('Locals')
2000         self.serialize_local_variables(node.scope.entries)
2001         self.tb.end('Locals')
2002
2003         self.tb.start('Arguments')
2004         self.tb.end('Arguments')
2005
2006         self.tb.start('StepIntoFunctions')
2007         self.register_stepinto = True
2008         self.visitchildren(node)
2009         self.register_stepinto = False
2010         self.tb.end('StepIntoFunctions')
2011
2012         self.tb.end('Function')
2013
2014     def serialize_local_variables(self, entries):
2015         for entry in entries.values():
2016             if entry.type.is_pyobject:
2017                 vartype = 'PythonObject'
2018             else:
2019                 vartype = 'CObject'
2020
2021             if entry.from_closure:
2022                 # We're dealing with a closure where a variable from an outer
2023                 # scope is accessed, get it from the scope object.
2024                 cname = '%s->%s' % (Naming.cur_scope_cname,
2025                                     entry.outer_entry.cname)
2026
2027                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2028                                       entry.scope.name,
2029                                       entry.name)
2030             elif entry.in_closure:
2031                 cname = '%s->%s' % (Naming.cur_scope_cname,
2032                                     entry.cname)
2033                 qname = entry.qualified_name
2034             else:
2035                 cname = entry.cname
2036                 qname = entry.qualified_name
2037
2038             if not entry.pos:
2039                 # this happens for variables that are not in the user's code,
2040                 # e.g. for the global __builtins__, __doc__, etc. We can just
2041                 # set the lineno to 0 for those.
2042                 lineno = '0'
2043             else:
2044                 lineno = str(entry.pos[1])
2045
2046             attrs = dict(
2047                 name=entry.name,
2048                 cname=cname,
2049                 qualified_name=qname,
2050                 type=vartype,
2051                 lineno=lineno)
2052
2053             self.tb.start('LocalVar', attrs)
2054             self.tb.end('LocalVar')
2055