Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67
  68 class NormalizeTree(CythonTransform):
  69     """
  70     This transform fixes up a few things after parsing
  71     in order to make the parse tree more suitable for
  72     transforms.
  73
  74     a) After parsing, blocks with only one statement will
  75     be represented by that statement, not by a StatListNode.
  76     When doing transforms this is annoying and inconsistent,
  77     as one cannot in general remove a statement in a consistent
  78     way and so on. This transform wraps any single statements
  79     in a StatListNode containing a single statement.
  80
  81     b) The PassStatNode is a noop and serves no purpose beyond
  82     plugging such one-statement blocks; i.e., once parsed a
  83 `    "pass" can just as well be represented using an empty
  84     StatListNode. This means less special cases to worry about
  85     in subsequent transforms (one always checks to see if a
  86     StatListNode has no children to see if the block is empty).
  87     """
  88
  89     def __init__(self, context):
  90         super(NormalizeTree, self).__init__(context)
  91         self.is_in_statlist = False
  92         self.is_in_expr = False
  93
  94     def visit_ExprNode(self, node):
  95         stacktmp = self.is_in_expr
  96         self.is_in_expr = True
  97         self.visitchildren(node)
  98         self.is_in_expr = stacktmp
  99         return node
 100
 101     def visit_StatNode(self, node, is_listcontainer=False):
 102         stacktmp = self.is_in_statlist
 103         self.is_in_statlist = is_listcontainer
 104         self.visitchildren(node)
 105         self.is_in_statlist = stacktmp
 106         if not self.is_in_statlist and not self.is_in_expr:
 107             return Nodes.StatListNode(pos=node.pos, stats=[node])
 108         else:
 109             return node
 110
 111     def visit_StatListNode(self, node):
 112         self.is_in_statlist = True
 113         self.visitchildren(node)
 114         self.is_in_statlist = False
 115         return node
 116
 117     def visit_ParallelAssignmentNode(self, node):
 118         return self.visit_StatNode(node, True)
 119
 120     def visit_CEnumDefNode(self, node):
 121         return self.visit_StatNode(node, True)
 122
 123     def visit_CStructOrUnionDefNode(self, node):
 124         return self.visit_StatNode(node, True)
 125
 126     # Eliminate PassStatNode
 127     def visit_PassStatNode(self, node):
 128         if not self.is_in_statlist:
 129             return Nodes.StatListNode(pos=node.pos, stats=[])
 130         else:
 131             return []
 132
 133     def visit_CDeclaratorNode(self, node):
 134         return node
 135
 136
 137 class PostParseError(CompileError): pass
 138
 139 # error strings checked by unit tests, so define them
 140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 143 class PostParse(ScopeTrackingTransform):
 144     """
 145     Basic interpretation of the parse tree, as well as validity
 146     checking that can be done on a very basic level on the parse
 147     tree (while still not being a problem with the basic syntax,
 148     as such).
 149
 150     Specifically:
 151     - Default values to cdef assignments are turned into single
 152     assignments following the declaration (everywhere but in class
 153     bodies, where they raise a compile error)
 154
 155     - Interpret some node structures into Python runtime values.
 156     Some nodes take compile-time arguments (currently:
 157     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 158     which should be interpreted. This happens in a general way
 159     and other steps should be taken to ensure validity.
 160
 161     Type arguments cannot be interpreted in this way.
 162
 163     - For __cythonbufferdefaults__ the arguments are checked for
 164     validity.
 165
 166     TemplatedTypeNode has its directives interpreted:
 167     Any first positional argument goes into the "dtype" attribute,
 168     any "ndim" keyword argument goes into the "ndim" attribute and
 169     so on. Also it is checked that the directive combination is valid.
 170     - __cythonbufferdefaults__ attributes are parsed and put into the
 171     type information.
 172
 173     Note: Currently Parsing.py does a lot of interpretation and
 174     reorganization that can be refactored into this transform
 175     if a more pure Abstract Syntax Tree is wanted.
 176     """
 177
 178     def __init__(self, context):
 179         super(PostParse, self).__init__(context)
 180         self.specialattribute_handlers = {
 181             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 182         }
 183
 184     def visit_ModuleNode(self, node):
 185         self.lambda_counter = 1
 186         self.genexpr_counter = 1
 187         return super(PostParse, self).visit_ModuleNode(node)
 188
 189     def visit_LambdaNode(self, node):
 190         # unpack a lambda expression into the corresponding DefNode
 191         lambda_id = self.lambda_counter
 192         self.lambda_counter += 1
 193         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 194         collector = YieldNodeCollector()
 195         collector.visitchildren(node.result_expr)
 196         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 197             body = ExprNodes.YieldExprNode(
 198                 node.result_expr.pos, arg=node.result_expr)
 199             body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
 200         else:
 201             body = Nodes.ReturnStatNode(
 202                 node.result_expr.pos, value=node.result_expr)
 203         node.def_node = Nodes.DefNode(
 204             node.pos, name=node.name, lambda_name=node.lambda_name,
 205             args=node.args, star_arg=node.star_arg,
 206             starstar_arg=node.starstar_arg,
 207             body=body, doc=None)
 208         self.visitchildren(node)
 209         return node
 210
 211     def visit_GeneratorExpressionNode(self, node):
 212         # unpack a generator expression into the corresponding DefNode
 213         genexpr_id = self.genexpr_counter
 214         self.genexpr_counter += 1
 215         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 216
 217         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 218                                       doc=None,
 219                                       args=[], star_arg=None,
 220                                       starstar_arg=None,
 221                                       body=node.loop)
 222         self.visitchildren(node)
 223         return node
 224
 225     # cdef variables
 226     def handle_bufferdefaults(self, decl):
 227         if not isinstance(decl.default, ExprNodes.DictNode):
 228             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 229         self.scope_node.buffer_defaults_node = decl.default
 230         self.scope_node.buffer_defaults_pos = decl.pos
 231
 232     def visit_CVarDefNode(self, node):
 233         # This assumes only plain names and pointers are assignable on
 234         # declaration. Also, it makes use of the fact that a cdef decl
 235         # must appear before the first use, so we don't have to deal with
 236         # "i = 3; cdef int i = i" and can simply move the nodes around.
 237         try:
 238             self.visitchildren(node)
 239             stats = [node]
 240             newdecls = []
 241             for decl in node.declarators:
 242                 declbase = decl
 243                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 244                     declbase = declbase.base
 245                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 246                     if declbase.default is not None:
 247                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 248                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 249                                 handler = self.specialattribute_handlers.get(decl.name)
 250                                 if handler:
 251                                     if decl is not declbase:
 252                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 253                                     handler(decl)
 254                                     continue # Remove declaration
 255                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 256                         first_assignment = self.scope_type != 'module'
 257                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 258                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 259                             rhs=declbase.default, first=first_assignment))
 260                         declbase.default = None
 261                 newdecls.append(decl)
 262             node.declarators = newdecls
 263             return stats
 264         except PostParseError, e:
 265             # An error in a cdef clause is ok, simply remove the declaration
 266             # and try to move on to report more errors
 267             self.context.nonfatal_error(e)
 268             return None
 269
 270     # Split parallel assignments (a,b = b,a) into separate partial
 271     # assignments that are executed rhs-first using temps.  This
 272     # restructuring must be applied before type analysis so that known
 273     # types on rhs and lhs can be matched directly.  It is required in
 274     # the case that the types cannot be coerced to a Python type in
 275     # order to assign from a tuple.
 276
 277     def visit_SingleAssignmentNode(self, node):
 278         self.visitchildren(node)
 279         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 280
 281     def visit_CascadedAssignmentNode(self, node):
 282         self.visitchildren(node)
 283         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 284
 285     def _visit_assignment_node(self, node, expr_list):
 286         """Flatten parallel assignments into separate single
 287         assignments or cascaded assignments.
 288         """
 289         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 290             # no parallel assignments => nothing to do
 291             return node
 292
 293         expr_list_list = []
 294         flatten_parallel_assignments(expr_list, expr_list_list)
 295         temp_refs = []
 296         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 297
 298         nodes = []
 299         for expr_list in expr_list_list:
 300             lhs_list = expr_list[:-1]
 301             rhs = expr_list[-1]
 302             if len(lhs_list) == 1:
 303                 node = Nodes.SingleAssignmentNode(rhs.pos,
 304                     lhs = lhs_list[0], rhs = rhs)
 305             else:
 306                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 307                     lhs_list = lhs_list, rhs = rhs)
 308             nodes.append(node)
 309
 310         if len(nodes) == 1:
 311             assign_node = nodes[0]
 312         else:
 313             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 314
 315         if temp_refs:
 316             duplicates_and_temps = [ (temp.expression, temp)
 317                                      for temp in temp_refs ]
 318             sort_common_subsequences(duplicates_and_temps)
 319             for _, temp_ref in duplicates_and_temps[::-1]:
 320                 assign_node = LetNode(temp_ref, assign_node)
 321
 322         return assign_node
 323
 324 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 325     """Replace rhs items by LetRefNodes if they appear more than once.
 326     Creates a sequence of LetRefNodes that set up the required temps
 327     and appends them to ref_node_sequence.  The input list is modified
 328     in-place.
 329     """
 330     seen_nodes = cython.set()
 331     ref_nodes = {}
 332     def find_duplicates(node):
 333         if node.is_literal or node.is_name:
 334             # no need to replace those; can't include attributes here
 335             # as their access is not necessarily side-effect free
 336             return
 337         if node in seen_nodes:
 338             if node not in ref_nodes:
 339                 ref_node = LetRefNode(node)
 340                 ref_nodes[node] = ref_node
 341                 ref_node_sequence.append(ref_node)
 342         else:
 343             seen_nodes.add(node)
 344             if node.is_sequence_constructor:
 345                 for item in node.args:
 346                     find_duplicates(item)
 347
 348     for expr_list in expr_list_list:
 349         rhs = expr_list[-1]
 350         find_duplicates(rhs)
 351     if not ref_nodes:
 352         return
 353
 354     def substitute_nodes(node):
 355         if node in ref_nodes:
 356             return ref_nodes[node]
 357         elif node.is_sequence_constructor:
 358             node.args = list(map(substitute_nodes, node.args))
 359         return node
 360
 361     # replace nodes inside of the common subexpressions
 362     for node in ref_nodes:
 363         if node.is_sequence_constructor:
 364             node.args = list(map(substitute_nodes, node.args))
 365
 366     # replace common subexpressions on all rhs items
 367     for expr_list in expr_list_list:
 368         expr_list[-1] = substitute_nodes(expr_list[-1])
 369
 370 def sort_common_subsequences(items):
 371     """Sort items/subsequences so that all items and subsequences that
 372     an item contains appear before the item itself.  This is needed
 373     because each rhs item must only be evaluated once, so its value
 374     must be evaluated first and then reused when packing sequences
 375     that contain it.
 376
 377     This implies a partial order, and the sort must be stable to
 378     preserve the original order as much as possible, so we use a
 379     simple insertion sort (which is very fast for short sequences, the
 380     normal case in practice).
 381     """
 382     def contains(seq, x):
 383         for item in seq:
 384             if item is x:
 385                 return True
 386             elif item.is_sequence_constructor and contains(item.args, x):
 387                 return True
 388         return False
 389     def lower_than(a,b):
 390         return b.is_sequence_constructor and contains(b.args, a)
 391
 392     for pos, item in enumerate(items):
 393         key = item[1] # the ResultRefNode which has already been injected into the sequences
 394         new_pos = pos
 395         for i in xrange(pos-1, -1, -1):
 396             if lower_than(key, items[i][0]):
 397                 new_pos = i
 398         if new_pos != pos:
 399             for i in xrange(pos, new_pos, -1):
 400                 items[i] = items[i-1]
 401             items[new_pos] = item
 402
 403 def flatten_parallel_assignments(input, output):
 404     #  The input is a list of expression nodes, representing the LHSs
 405     #  and RHS of one (possibly cascaded) assignment statement.  For
 406     #  sequence constructors, rearranges the matching parts of both
 407     #  sides into a list of equivalent assignments between the
 408     #  individual elements.  This transformation is applied
 409     #  recursively, so that nested structures get matched as well.
 410     rhs = input[-1]
 411     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 412         output.append(input)
 413         return
 414
 415     complete_assignments = []
 416
 417     rhs_size = len(rhs.args)
 418     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 419     starred_assignments = []
 420     for lhs in input[:-1]:
 421         if not lhs.is_sequence_constructor:
 422             if lhs.is_starred:
 423                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 424             complete_assignments.append(lhs)
 425             continue
 426         lhs_size = len(lhs.args)
 427         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 428         if starred_targets > 1:
 429             error(lhs.pos, "more than 1 starred expression in assignment")
 430             output.append([lhs,rhs])
 431             continue
 432         elif lhs_size - starred_targets > rhs_size:
 433             error(lhs.pos, "need more than %d value%s to unpack"
 434                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 435             output.append([lhs,rhs])
 436             continue
 437         elif starred_targets:
 438             map_starred_assignment(lhs_targets, starred_assignments,
 439                                    lhs.args, rhs.args)
 440         elif lhs_size < rhs_size:
 441             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 442                   % (lhs_size, rhs_size))
 443             output.append([lhs,rhs])
 444             continue
 445         else:
 446             for targets, expr in zip(lhs_targets, lhs.args):
 447                 targets.append(expr)
 448
 449     if complete_assignments:
 450         complete_assignments.append(rhs)
 451         output.append(complete_assignments)
 452
 453     # recursively flatten partial assignments
 454     for cascade, rhs in zip(lhs_targets, rhs.args):
 455         if cascade:
 456             cascade.append(rhs)
 457             flatten_parallel_assignments(cascade, output)
 458
 459     # recursively flatten starred assignments
 460     for cascade in starred_assignments:
 461         if cascade[0].is_sequence_constructor:
 462             flatten_parallel_assignments(cascade, output)
 463         else:
 464             output.append(cascade)
 465
 466 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 467     # Appends the fixed-position LHS targets to the target list that
 468     # appear left and right of the starred argument.
 469     #
 470     # The starred_assignments list receives a new tuple
 471     # (lhs_target, rhs_values_list) that maps the remaining arguments
 472     # (those that match the starred target) to a list.
 473
 474     # left side of the starred target
 475     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 476         if expr.is_starred:
 477             starred = i
 478             lhs_remaining = len(lhs_args) - i - 1
 479             break
 480         targets.append(expr)
 481     else:
 482         raise InternalError("no starred arg found when splitting starred assignment")
 483
 484     # right side of the starred target
 485     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 486                                             lhs_args[starred + 1:])):
 487         targets.append(expr)
 488
 489     # the starred target itself, must be assigned a (potentially empty) list
 490     target = lhs_args[starred].target # unpack starred node
 491     starred_rhs = rhs_args[starred:]
 492     if lhs_remaining:
 493         starred_rhs = starred_rhs[:-lhs_remaining]
 494     if starred_rhs:
 495         pos = starred_rhs[0].pos
 496     else:
 497         pos = target.pos
 498     starred_assignments.append([
 499         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 500
 501
 502 class PxdPostParse(CythonTransform, SkipDeclarations):
 503     """
 504     Basic interpretation/validity checking that should only be
 505     done on pxd trees.
 506
 507     A lot of this checking currently happens in the parser; but
 508     what is listed below happens here.
 509
 510     - "def" functions are let through only if they fill the
 511     getbuffer/releasebuffer slots
 512
 513     - cdef functions are let through only if they are on the
 514     top level and are declared "inline"
 515     """
 516     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 517     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 518
 519     def __call__(self, node):
 520         self.scope_type = 'pxd'
 521         return super(PxdPostParse, self).__call__(node)
 522
 523     def visit_CClassDefNode(self, node):
 524         old = self.scope_type
 525         self.scope_type = 'cclass'
 526         self.visitchildren(node)
 527         self.scope_type = old
 528         return node
 529
 530     def visit_FuncDefNode(self, node):
 531         # FuncDefNode always come with an implementation (without
 532         # an imp they are CVarDefNodes..)
 533         err = self.ERR_INLINE_ONLY
 534
 535         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 536             and node.name in ('__getbuffer__', '__releasebuffer__')):
 537             err = None # allow these slots
 538
 539         if isinstance(node, Nodes.CFuncDefNode):
 540             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 541                 node.inline_in_pxd = True
 542                 if node.visibility != 'private':
 543                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 544                 elif node.api:
 545                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 546                 else:
 547                     err = None # allow inline function
 548             else:
 549                 err = self.ERR_INLINE_ONLY
 550
 551         if err:
 552             self.context.nonfatal_error(PostParseError(node.pos, err))
 553             return None
 554         else:
 555             return node
 556
 557 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 558     """
 559     After parsing, directives can be stored in a number of places:
 560     - #cython-comments at the top of the file (stored in ModuleNode)
 561     - Command-line arguments overriding these
 562     - @cython.directivename decorators
 563     - with cython.directivename: statements
 564
 565     This transform is responsible for interpreting these various sources
 566     and store the directive in two ways:
 567     - Set the directives attribute of the ModuleNode for global directives.
 568     - Use a CompilerDirectivesNode to override directives for a subtree.
 569
 570     (The first one is primarily to not have to modify with the tree
 571     structure, so that ModuleNode stay on top.)
 572
 573     The directives are stored in dictionaries from name to value in effect.
 574     Each such dictionary is always filled in for all possible directives,
 575     using default values where no value is given by the user.
 576
 577     The available directives are controlled in Options.py.
 578
 579     Note that we have to run this prior to analysis, and so some minor
 580     duplication of functionality has to occur: We manually track cimports
 581     and which names the "cython" module may have been imported to.
 582     """
 583     unop_method_nodes = {
 584         'typeof': ExprNodes.TypeofNode,
 585
 586         'operator.address': ExprNodes.AmpersandNode,
 587         'operator.dereference': ExprNodes.DereferenceNode,
 588         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 589         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 590         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 591         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 592
 593         # For backwards compatability.
 594         'address': ExprNodes.AmpersandNode,
 595     }
 596
 597     binop_method_nodes = {
 598         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 599     }
 600
 601     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 602                                   'cast', 'pointer', 'compiled', 'NULL'])
 603     special_methods.update(unop_method_nodes.keys())
 604
 605     def __init__(self, context, compilation_directive_defaults):
 606         super(InterpretCompilerDirectives, self).__init__(context)
 607         self.compilation_directive_defaults = {}
 608         for key, value in compilation_directive_defaults.items():
 609             self.compilation_directive_defaults[unicode(key)] = value
 610         self.cython_module_names = cython.set()
 611         self.directive_names = {}
 612
 613     def check_directive_scope(self, pos, directive, scope):
 614         legal_scopes = Options.directive_scopes.get(directive, None)
 615         if legal_scopes and scope not in legal_scopes:
 616             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 617                                         'is not allowed in %s scope' % (directive, scope)))
 618             return False
 619         else:
 620             return True
 621
 622     # Set up processing and handle the cython: comments.
 623     def visit_ModuleNode(self, node):
 624         for key, value in node.directive_comments.items():
 625             if not self.check_directive_scope(node.pos, key, 'module'):
 626                 self.wrong_scope_error(node.pos, key, 'module')
 627                 del node.directive_comments[key]
 628
 629         directives = copy.copy(Options.directive_defaults)
 630         directives.update(self.compilation_directive_defaults)
 631         directives.update(node.directive_comments)
 632         self.directives = directives
 633         node.directives = directives
 634         self.visitchildren(node)
 635         node.cython_module_names = self.cython_module_names
 636         return node
 637
 638     # The following four functions track imports and cimports that
 639     # begin with "cython"
 640     def is_cython_directive(self, name):
 641         return (name in Options.directive_types or
 642                 name in self.special_methods or
 643                 PyrexTypes.parse_basic_type(name))
 644
 645     def visit_CImportStatNode(self, node):
 646         if node.module_name == u"cython":
 647             self.cython_module_names.add(node.as_name or u"cython")
 648         elif node.module_name.startswith(u"cython."):
 649             if node.as_name:
 650                 self.directive_names[node.as_name] = node.module_name[7:]
 651             else:
 652                 self.cython_module_names.add(u"cython")
 653             # if this cimport was a compiler directive, we don't
 654             # want to leave the cimport node sitting in the tree
 655             return None
 656         return node
 657
 658     def visit_FromCImportStatNode(self, node):
 659         if (node.module_name == u"cython") or \
 660                node.module_name.startswith(u"cython."):
 661             submodule = (node.module_name + u".")[7:]
 662             newimp = []
 663             for pos, name, as_name, kind in node.imported_names:
 664                 full_name = submodule + name
 665                 if self.is_cython_directive(full_name):
 666                     if as_name is None:
 667                         as_name = full_name
 668                     self.directive_names[as_name] = full_name
 669                     if kind is not None:
 670                         self.context.nonfatal_error(PostParseError(pos,
 671                             "Compiler directive imports must be plain imports"))
 672                 else:
 673                     newimp.append((pos, name, as_name, kind))
 674             if not newimp:
 675                 return None
 676             node.imported_names = newimp
 677         return node
 678
 679     def visit_FromImportStatNode(self, node):
 680         if (node.module.module_name.value == u"cython") or \
 681                node.module.module_name.value.startswith(u"cython."):
 682             submodule = (node.module.module_name.value + u".")[7:]
 683             newimp = []
 684             for name, name_node in node.items:
 685                 full_name = submodule + name
 686                 if self.is_cython_directive(full_name):
 687                     self.directive_names[name_node.name] = full_name
 688                 else:
 689                     newimp.append((name, name_node))
 690             if not newimp:
 691                 return None
 692             node.items = newimp
 693         return node
 694
 695     def visit_SingleAssignmentNode(self, node):
 696         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 697                 node.rhs.module_name.value == u'cython'):
 698             node = Nodes.CImportStatNode(node.pos,
 699                                          module_name = u'cython',
 700                                          as_name = node.lhs.name)
 701             self.visit_CImportStatNode(node)
 702         else:
 703             self.visitchildren(node)
 704         return node
 705
 706     def visit_NameNode(self, node):
 707         if node.name in self.cython_module_names:
 708             node.is_cython_module = True
 709         else:
 710             node.cython_attribute = self.directive_names.get(node.name)
 711         return node
 712
 713     def try_to_parse_directives(self, node):
 714         # If node is the contents of an directive (in a with statement or
 715         # decorator), returns a list of (directivename, value) pairs.
 716         # Otherwise, returns None
 717         if isinstance(node, ExprNodes.CallNode):
 718             self.visit(node.function)
 719             optname = node.function.as_cython_attribute()
 720             if optname:
 721                 directivetype = Options.directive_types.get(optname)
 722                 if directivetype:
 723                     args, kwds = node.explicit_args_kwds()
 724                     directives = []
 725                     key_value_pairs = []
 726                     if kwds is not None and directivetype is not dict:
 727                         for keyvalue in kwds.key_value_pairs:
 728                             key, value = keyvalue
 729                             sub_optname = "%s.%s" % (optname, key.value)
 730                             if Options.directive_types.get(sub_optname):
 731                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 732                             else:
 733                                 key_value_pairs.append(keyvalue)
 734                         if not key_value_pairs:
 735                             kwds = None
 736                         else:
 737                             kwds.key_value_pairs = key_value_pairs
 738                         if directives and not kwds and not args:
 739                             return directives
 740                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 741                     return directives
 742         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 743             self.visit(node)
 744             optname = node.as_cython_attribute()
 745             if optname:
 746                 directivetype = Options.directive_types.get(optname)
 747                 if directivetype is bool:
 748                     return [(optname, True)]
 749                 elif directivetype is None:
 750                     return [(optname, None)]
 751                 else:
 752                     raise PostParseError(
 753                         node.pos, "The '%s' directive should be used as a function call." % optname)
 754         return None
 755
 756     def try_to_parse_directive(self, optname, args, kwds, pos):
 757         directivetype = Options.directive_types.get(optname)
 758         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 759             return optname, Options.directive_defaults[optname]
 760         elif directivetype is bool:
 761             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 762                 raise PostParseError(pos,
 763                     'The %s directive takes one compile-time boolean argument' % optname)
 764             return (optname, args[0].value)
 765         elif directivetype is str:
 766             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 767                                                                               ExprNodes.UnicodeNode)):
 768                 raise PostParseError(pos,
 769                     'The %s directive takes one compile-time string argument' % optname)
 770             return (optname, str(args[0].value))
 771         elif directivetype is dict:
 772             if len(args) != 0:
 773                 raise PostParseError(pos,
 774                     'The %s directive takes no prepositional arguments' % optname)
 775             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 776         elif directivetype is list:
 777             if kwds and len(kwds) != 0:
 778                 raise PostParseError(pos,
 779                     'The %s directive takes no keyword arguments' % optname)
 780             return optname, [ str(arg.value) for arg in args ]
 781         else:
 782             assert False
 783
 784     def visit_with_directives(self, body, directives):
 785         olddirectives = self.directives
 786         newdirectives = copy.copy(olddirectives)
 787         newdirectives.update(directives)
 788         self.directives = newdirectives
 789         assert isinstance(body, Nodes.StatListNode), body
 790         retbody = self.visit_Node(body)
 791         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 792                                                  directives=newdirectives)
 793         self.directives = olddirectives
 794         return directive
 795
 796     # Handle decorators
 797     def visit_FuncDefNode(self, node):
 798         directives = self._extract_directives(node, 'function')
 799         if not directives:
 800             return self.visit_Node(node)
 801         body = Nodes.StatListNode(node.pos, stats=[node])
 802         return self.visit_with_directives(body, directives)
 803
 804     def visit_CVarDefNode(self, node):
 805         if not node.decorators:
 806             return node
 807         for dec in node.decorators:
 808             for directive in self.try_to_parse_directives(dec.decorator) or ():
 809                 if directive is not None and directive[0] == u'locals':
 810                     node.directive_locals = directive[1]
 811                 else:
 812                     self.context.nonfatal_error(PostParseError(dec.pos,
 813                         "Cdef functions can only take cython.locals() decorator."))
 814         return node
 815
 816     def visit_CClassDefNode(self, node):
 817         directives = self._extract_directives(node, 'cclass')
 818         if not directives:
 819             return self.visit_Node(node)
 820         body = Nodes.StatListNode(node.pos, stats=[node])
 821         return self.visit_with_directives(body, directives)
 822
 823     def visit_PyClassDefNode(self, node):
 824         directives = self._extract_directives(node, 'class')
 825         if not directives:
 826             return self.visit_Node(node)
 827         body = Nodes.StatListNode(node.pos, stats=[node])
 828         return self.visit_with_directives(body, directives)
 829
 830     def _extract_directives(self, node, scope_name):
 831         if not node.decorators:
 832             return {}
 833         # Split the decorators into two lists -- real decorators and directives
 834         directives = []
 835         realdecs = []
 836         for dec in node.decorators:
 837             new_directives = self.try_to_parse_directives(dec.decorator)
 838             if new_directives is not None:
 839                 for directive in new_directives:
 840                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 841                         directives.append(directive)
 842             else:
 843                 realdecs.append(dec)
 844         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 845             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 846         else:
 847             node.decorators = realdecs
 848         # merge or override repeated directives
 849         optdict = {}
 850         directives.reverse() # Decorators coming first take precedence
 851         for directive in directives:
 852             name, value = directive
 853             if name in optdict:
 854                 old_value = optdict[name]
 855                 # keywords and arg lists can be merged, everything
 856                 # else overrides completely
 857                 if isinstance(old_value, dict):
 858                     old_value.update(value)
 859                 elif isinstance(old_value, list):
 860                     old_value.extend(value)
 861                 else:
 862                     optdict[name] = value
 863             else:
 864                 optdict[name] = value
 865         return optdict
 866
 867     # Handle with statements
 868     def visit_WithStatNode(self, node):
 869         directive_dict = {}
 870         for directive in self.try_to_parse_directives(node.manager) or []:
 871             if directive is not None:
 872                 if node.target is not None:
 873                     self.context.nonfatal_error(
 874                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 875                 else:
 876                     name, value = directive
 877                     if name == 'nogil':
 878                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 879                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 880                         return self.visit_Node(node)
 881                     if self.check_directive_scope(node.pos, name, 'with statement'):
 882                         directive_dict[name] = value
 883         if directive_dict:
 884             return self.visit_with_directives(node.body, directive_dict)
 885         return self.visit_Node(node)
 886
 887 class WithTransform(CythonTransform, SkipDeclarations):
 888
 889     # EXCINFO is manually set to a variable that contains
 890     # the exc_info() tuple that can be generated by the enclosing except
 891     # statement.
 892     template_without_target = TreeFragment(u"""
 893         MGR = EXPR
 894         EXIT = MGR.__exit__
 895         MGR.__enter__()
 896         EXC = True
 897         try:
 898             try:
 899                 EXCINFO = None
 900                 BODY
 901             except:
 902                 EXC = False
 903                 if not EXIT(*EXCINFO):
 904                     raise
 905         finally:
 906             if EXC:
 907                 EXIT(None, None, None)
 908     """, temps=[u'MGR', u'EXC', u"EXIT"],
 909     pipeline=[NormalizeTree(None)])
 910
 911     template_with_target = TreeFragment(u"""
 912         MGR = EXPR
 913         EXIT = MGR.__exit__
 914         VALUE = MGR.__enter__()
 915         EXC = True
 916         try:
 917             try:
 918                 EXCINFO = None
 919                 TARGET = VALUE
 920                 BODY
 921             except:
 922                 EXC = False
 923                 if not EXIT(*EXCINFO):
 924                     raise
 925         finally:
 926             if EXC:
 927                 EXIT(None, None, None)
 928             MGR = EXIT = VALUE = EXC = None
 929
 930     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 931     pipeline=[NormalizeTree(None)])
 932
 933     def visit_WithStatNode(self, node):
 934         # TODO: Cleanup badly needed
 935         TemplateTransform.temp_name_counter += 1
 936         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 937
 938         self.visitchildren(node, ['body'])
 939         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 940         if node.target is not None:
 941             result = self.template_with_target.substitute({
 942                 u'EXPR' : node.manager,
 943                 u'BODY' : node.body,
 944                 u'TARGET' : node.target,
 945                 u'EXCINFO' : excinfo_temp
 946                 }, pos=node.pos)
 947         else:
 948             result = self.template_without_target.substitute({
 949                 u'EXPR' : node.manager,
 950                 u'BODY' : node.body,
 951                 u'EXCINFO' : excinfo_temp
 952                 }, pos=node.pos)
 953
 954         # Set except excinfo target to EXCINFO
 955         try_except = result.stats[-1].body.stats[-1]
 956         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 957 #            excinfo_temp.ref(node.pos))
 958
 959 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 960 #            node.pos, temps=[excinfo_temp], body=try_except)
 961
 962         return result
 963
 964     def visit_ExprNode(self, node):
 965         # With statements are never inside expressions.
 966         return node
 967
 968
 969 class DecoratorTransform(CythonTransform, SkipDeclarations):
 970
 971     def visit_DefNode(self, func_node):
 972         self.visitchildren(func_node)
 973         if not func_node.decorators:
 974             return func_node
 975         return self._handle_decorators(
 976             func_node, func_node.name)
 977
 978     def visit_CClassDefNode(self, class_node):
 979         # This doesn't currently work, so it's disabled.
 980         #
 981         # Problem: assignments to cdef class names do not work.  They
 982         # would require an additional check anyway, as the extension
 983         # type must not change its C type, so decorators cannot
 984         # replace an extension type, just alter it and return it.
 985
 986         self.visitchildren(class_node)
 987         if not class_node.decorators:
 988             return class_node
 989         error(class_node.pos,
 990               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 991         return class_node
 992         #return self._handle_decorators(
 993         #    class_node, class_node.class_name)
 994
 995     def visit_ClassDefNode(self, class_node):
 996         self.visitchildren(class_node)
 997         if not class_node.decorators:
 998             return class_node
 999         return self._handle_decorators(
1000             class_node, class_node.name)
1001
1002     def _handle_decorators(self, node, name):
1003         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1004         for decorator in node.decorators[::-1]:
1005             decorator_result = ExprNodes.SimpleCallNode(
1006                 decorator.pos,
1007                 function = decorator.decorator,
1008                 args = [decorator_result])
1009
1010         name_node = ExprNodes.NameNode(node.pos, name = name)
1011         reassignment = Nodes.SingleAssignmentNode(
1012             node.pos,
1013             lhs = name_node,
1014             rhs = decorator_result)
1015         return [node, reassignment]
1016
1017
1018 class AnalyseDeclarationsTransform(CythonTransform):
1019
1020     basic_property = TreeFragment(u"""
1021 property NAME:
1022     def __get__(self):
1023         return ATTR
1024     def __set__(self, value):
1025         ATTR = value
1026     """, level='c_class')
1027     basic_pyobject_property = TreeFragment(u"""
1028 property NAME:
1029     def __get__(self):
1030         return ATTR
1031     def __set__(self, value):
1032         ATTR = value
1033     def __del__(self):
1034         ATTR = None
1035     """, level='c_class')
1036     basic_property_ro = TreeFragment(u"""
1037 property NAME:
1038     def __get__(self):
1039         return ATTR
1040     """, level='c_class')
1041
1042     struct_or_union_wrapper = TreeFragment(u"""
1043 cdef class NAME:
1044     cdef TYPE value
1045     def __init__(self, MEMBER=None):
1046         cdef int count
1047         count = 0
1048         INIT_ASSIGNMENTS
1049         if IS_UNION and count > 1:
1050             raise ValueError, "At most one union member should be specified."
1051     def __str__(self):
1052         return STR_FORMAT % MEMBER_TUPLE
1053     def __repr__(self):
1054         return REPR_FORMAT % MEMBER_TUPLE
1055     """)
1056
1057     init_assignment = TreeFragment(u"""
1058 if VALUE is not None:
1059     ATTR = VALUE
1060     count += 1
1061     """)
1062
1063     def __call__(self, root):
1064         self.env_stack = [root.scope]
1065         # needed to determine if a cdef var is declared after it's used.
1066         self.seen_vars_stack = []
1067         return super(AnalyseDeclarationsTransform, self).__call__(root)
1068
1069     def visit_NameNode(self, node):
1070         self.seen_vars_stack[-1].add(node.name)
1071         return node
1072
1073     def visit_ModuleNode(self, node):
1074         self.seen_vars_stack.append(cython.set())
1075         node.analyse_declarations(self.env_stack[-1])
1076         self.visitchildren(node)
1077         self.seen_vars_stack.pop()
1078         return node
1079
1080     def visit_LambdaNode(self, node):
1081         node.analyse_declarations(self.env_stack[-1])
1082         self.visitchildren(node)
1083         return node
1084
1085     def visit_ClassDefNode(self, node):
1086         self.env_stack.append(node.scope)
1087         self.visitchildren(node)
1088         self.env_stack.pop()
1089         return node
1090
1091     def visit_CClassDefNode(self, node):
1092         node = self.visit_ClassDefNode(node)
1093         if node.scope and node.scope.implemented:
1094             stats = []
1095             for entry in node.scope.var_entries:
1096                 if entry.needs_property:
1097                     property = self.create_Property(entry)
1098                     property.analyse_declarations(node.scope)
1099                     self.visit(property)
1100                     stats.append(property)
1101             if stats:
1102                 node.body.stats += stats
1103         return node
1104
1105     def visit_FuncDefNode(self, node):
1106         self.seen_vars_stack.append(cython.set())
1107         lenv = node.local_scope
1108         node.body.analyse_control_flow(lenv) # this will be totally refactored
1109         node.declare_arguments(lenv)
1110         for var, type_node in node.directive_locals.items():
1111             if not lenv.lookup_here(var):   # don't redeclare args
1112                 type = type_node.analyse_as_type(lenv)
1113                 if type:
1114                     lenv.declare_var(var, type, type_node.pos)
1115                 else:
1116                     error(type_node.pos, "Not a type")
1117         node.body.analyse_declarations(lenv)
1118         self.env_stack.append(lenv)
1119         self.visitchildren(node)
1120         self.env_stack.pop()
1121         self.seen_vars_stack.pop()
1122         return node
1123
1124     def visit_ScopedExprNode(self, node):
1125         env = self.env_stack[-1]
1126         node.analyse_declarations(env)
1127         # the node may or may not have a local scope
1128         if node.has_local_scope:
1129             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1130             self.env_stack.append(node.expr_scope)
1131             node.analyse_scoped_declarations(node.expr_scope)
1132             self.visitchildren(node)
1133             self.env_stack.pop()
1134             self.seen_vars_stack.pop()
1135         else:
1136             node.analyse_scoped_declarations(env)
1137             self.visitchildren(node)
1138         return node
1139
1140     def visit_TempResultFromStatNode(self, node):
1141         self.visitchildren(node)
1142         node.analyse_declarations(self.env_stack[-1])
1143         return node
1144
1145     def visit_CStructOrUnionDefNode(self, node):
1146         # Create a wrapper node if needed.
1147         # We want to use the struct type information (so it can't happen
1148         # before this phase) but also create new objects to be declared
1149         # (so it can't happen later).
1150         # Note that we don't return the original node, as it is
1151         # never used after this phase.
1152         if True: # private (default)
1153             return None
1154
1155         self_value = ExprNodes.AttributeNode(
1156             pos = node.pos,
1157             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1158             attribute = EncodedString(u"value"))
1159         var_entries = node.entry.type.scope.var_entries
1160         attributes = []
1161         for entry in var_entries:
1162             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1163                                                       obj = self_value,
1164                                                       attribute = entry.name))
1165         # __init__ assignments
1166         init_assignments = []
1167         for entry, attr in zip(var_entries, attributes):
1168             # TODO: branch on visibility
1169             init_assignments.append(self.init_assignment.substitute({
1170                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1171                     u"ATTR": attr,
1172                 }, pos = entry.pos))
1173
1174         # create the class
1175         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1176         wrapper_class = self.struct_or_union_wrapper.substitute({
1177             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1178             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1179             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1180             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1181             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1182         }, pos = node.pos).stats[0]
1183         wrapper_class.class_name = node.name
1184         wrapper_class.shadow = True
1185         class_body = wrapper_class.body.stats
1186
1187         # fix value type
1188         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1189         class_body[0].base_type.name = node.name
1190
1191         # fix __init__ arguments
1192         init_method = class_body[1]
1193         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1194         arg_template = init_method.args[1]
1195         if not node.entry.type.is_struct:
1196             arg_template.kw_only = True
1197         del init_method.args[1]
1198         for entry, attr in zip(var_entries, attributes):
1199             arg = copy.deepcopy(arg_template)
1200             arg.declarator.name = entry.name
1201             init_method.args.append(arg)
1202
1203         # setters/getters
1204         for entry, attr in zip(var_entries, attributes):
1205             # TODO: branch on visibility
1206             if entry.type.is_pyobject:
1207                 template = self.basic_pyobject_property
1208             else:
1209                 template = self.basic_property
1210             property = template.substitute({
1211                     u"ATTR": attr,
1212                 }, pos = entry.pos).stats[0]
1213             property.name = entry.name
1214             wrapper_class.body.stats.append(property)
1215
1216         wrapper_class.analyse_declarations(self.env_stack[-1])
1217         return self.visit_CClassDefNode(wrapper_class)
1218
1219     # Some nodes are no longer needed after declaration
1220     # analysis and can be dropped. The analysis was performed
1221     # on these nodes in a seperate recursive process from the
1222     # enclosing function or module, so we can simply drop them.
1223     def visit_CDeclaratorNode(self, node):
1224         # necessary to ensure that all CNameDeclaratorNodes are visited.
1225         self.visitchildren(node)
1226         return node
1227
1228     def visit_CTypeDefNode(self, node):
1229         return node
1230
1231     def visit_CBaseTypeNode(self, node):
1232         return None
1233
1234     def visit_CEnumDefNode(self, node):
1235         if node.visibility == 'public':
1236             return node
1237         else:
1238             return None
1239
1240     def visit_CNameDeclaratorNode(self, node):
1241         if node.name in self.seen_vars_stack[-1]:
1242             entry = self.env_stack[-1].lookup(node.name)
1243             if entry is None or entry.visibility != 'extern':
1244                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1245         self.visitchildren(node)
1246         return node
1247
1248     def visit_CVarDefNode(self, node):
1249         # to ensure all CNameDeclaratorNodes are visited.
1250         self.visitchildren(node)
1251         return None
1252
1253     def create_Property(self, entry):
1254         if entry.visibility == 'public':
1255             if entry.type.is_pyobject:
1256                 template = self.basic_pyobject_property
1257             else:
1258                 template = self.basic_property
1259         elif entry.visibility == 'readonly':
1260             template = self.basic_property_ro
1261         property = template.substitute({
1262                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1263                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1264                                                  attribute=entry.name),
1265             }, pos=entry.pos).stats[0]
1266         property.name = entry.name
1267         # ---------------------------------------
1268         # XXX This should go to AutoDocTransforms
1269         # ---------------------------------------
1270         if (Options.docstrings and
1271             self.current_directives['embedsignature']):
1272             attr_name = entry.name
1273             type_name = entry.type.declaration_code("", for_display=1)
1274             default_value = ''
1275             if not entry.type.is_pyobject:
1276                 type_name = "'%s'" % type_name
1277             elif entry.type.is_extension_type:
1278                 type_name = entry.type.module_name + '.' + type_name
1279             if entry.init is not None:
1280                 default_value = ' = ' + entry.init
1281             elif entry.init_to_none:
1282                 default_value = ' = ' + repr(None)
1283             docstring = attr_name + ': ' + type_name + default_value
1284             property.doc = EncodedString(docstring)
1285         # ---------------------------------------
1286         return property
1287
1288 class AnalyseExpressionsTransform(CythonTransform):
1289
1290     def visit_ModuleNode(self, node):
1291         node.scope.infer_types()
1292         node.body.analyse_expressions(node.scope)
1293         self.visitchildren(node)
1294         return node
1295
1296     def visit_FuncDefNode(self, node):
1297         node.local_scope.infer_types()
1298         node.body.analyse_expressions(node.local_scope)
1299         self.visitchildren(node)
1300         return node
1301
1302     def visit_ScopedExprNode(self, node):
1303         if node.has_local_scope:
1304             node.expr_scope.infer_types()
1305             node.analyse_scoped_expressions(node.expr_scope)
1306         self.visitchildren(node)
1307         return node
1308
1309 class ExpandInplaceOperators(EnvTransform):
1310
1311     def visit_InPlaceAssignmentNode(self, node):
1312         lhs = node.lhs
1313         rhs = node.rhs
1314         if lhs.type.is_cpp_class:
1315             # No getting around this exact operator here.
1316             return node
1317         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1318             # There is code to handle this case.
1319             return node
1320
1321         env = self.current_env()
1322         def side_effect_free_reference(node, setting=False):
1323             if isinstance(node, ExprNodes.NameNode):
1324                 return node, []
1325             elif node.type.is_pyobject and not setting:
1326                 node = LetRefNode(node)
1327                 return node, [node]
1328             elif isinstance(node, ExprNodes.IndexNode):
1329                 if node.is_buffer_access:
1330                     raise ValueError, "Buffer access"
1331                 base, temps = side_effect_free_reference(node.base)
1332                 index = LetRefNode(node.index)
1333                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1334             elif isinstance(node, ExprNodes.AttributeNode):
1335                 obj, temps = side_effect_free_reference(node.obj)
1336                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1337             else:
1338                 node = LetRefNode(node)
1339                 return node, [node]
1340         try:
1341             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1342         except ValueError:
1343             return node
1344         dup = lhs.__class__(**lhs.__dict__)
1345         binop = ExprNodes.binop_node(node.pos,
1346                                      operator = node.operator,
1347                                      operand1 = dup,
1348                                      operand2 = rhs,
1349                                      inplace=True)
1350         # Manually analyse types for new node.
1351         lhs.analyse_target_types(env)
1352         dup.analyse_types(env)
1353         binop.analyse_operation(env)
1354         node = Nodes.SingleAssignmentNode(
1355             node.pos,
1356             lhs = lhs,
1357             rhs=binop.coerce_to(lhs.type, env))
1358         # Use LetRefNode to avoid side effects.
1359         let_ref_nodes.reverse()
1360         for t in let_ref_nodes:
1361             node = LetNode(t, node)
1362         return node
1363
1364     def visit_ExprNode(self, node):
1365         # In-place assignments can't happen within an expression.
1366         return node
1367
1368
1369 class AlignFunctionDefinitions(CythonTransform):
1370     """
1371     This class takes the signatures from a .pxd file and applies them to
1372     the def methods in a .py file.
1373     """
1374
1375     def visit_ModuleNode(self, node):
1376         self.scope = node.scope
1377         self.directives = node.directives
1378         self.visitchildren(node)
1379         return node
1380
1381     def visit_PyClassDefNode(self, node):
1382         pxd_def = self.scope.lookup(node.name)
1383         if pxd_def:
1384             if pxd_def.is_cclass:
1385                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1386             else:
1387                 error(node.pos, "'%s' redeclared" % node.name)
1388                 error(pxd_def.pos, "previous declaration here")
1389                 return None
1390         else:
1391             return node
1392
1393     def visit_CClassDefNode(self, node, pxd_def=None):
1394         if pxd_def is None:
1395             pxd_def = self.scope.lookup(node.class_name)
1396         if pxd_def:
1397             outer_scope = self.scope
1398             self.scope = pxd_def.type.scope
1399         self.visitchildren(node)
1400         if pxd_def:
1401             self.scope = outer_scope
1402         return node
1403
1404     def visit_DefNode(self, node):
1405         pxd_def = self.scope.lookup(node.name)
1406         if pxd_def:
1407             if not pxd_def.is_cfunction:
1408                 error(node.pos, "'%s' redeclared" % node.name)
1409                 error(pxd_def.pos, "previous declaration here")
1410                 return None
1411             node = node.as_cfunction(pxd_def)
1412         elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1413             node = node.as_cfunction(scope=self.scope)
1414         # Enable this when internal def functions are allowed.
1415         # self.visitchildren(node)
1416         return node
1417
1418
1419 class YieldNodeCollector(TreeVisitor):
1420
1421     def __init__(self):
1422         super(YieldNodeCollector, self).__init__()
1423         self.yields = []
1424         self.returns = []
1425         self.has_return_value = False
1426
1427     def visit_Node(self, node):
1428         return self.visitchildren(node)
1429
1430     def visit_YieldExprNode(self, node):
1431         if self.has_return_value:
1432             error(node.pos, "'yield' outside function")
1433         self.yields.append(node)
1434         self.visitchildren(node)
1435
1436     def visit_ReturnStatNode(self, node):
1437         if node.value:
1438             self.has_return_value = True
1439             if self.yields:
1440                 error(node.pos, "'return' with argument inside generator")
1441         self.returns.append(node)
1442
1443     def visit_ClassDefNode(self, node):
1444         pass
1445
1446     def visit_DefNode(self, node):
1447         pass
1448
1449     def visit_LambdaNode(self, node):
1450         pass
1451
1452     def visit_GeneratorExpressionNode(self, node):
1453         pass
1454
1455 class MarkClosureVisitor(CythonTransform):
1456
1457     def visit_ModuleNode(self, node):
1458         self.needs_closure = False
1459         self.visitchildren(node)
1460         return node
1461
1462     def visit_FuncDefNode(self, node):
1463         self.needs_closure = False
1464         self.visitchildren(node)
1465         node.needs_closure = self.needs_closure
1466         self.needs_closure = True
1467
1468         collector = YieldNodeCollector()
1469         collector.visitchildren(node)
1470
1471         if collector.yields:
1472             for i, yield_expr in enumerate(collector.yields):
1473                 yield_expr.label_num = i + 1
1474
1475             gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1476                                                name=node.name,
1477                                                body=node.body)
1478             generator = Nodes.GeneratorDefNode(pos=node.pos,
1479                                                name=node.name,
1480                                                args=node.args,
1481                                                star_arg=node.star_arg,
1482                                                starstar_arg=node.starstar_arg,
1483                                                doc=node.doc,
1484                                                decorators=node.decorators,
1485                                                gbody=gbody,
1486                                                lambda_name=node.lambda_name)
1487             return generator
1488         return node
1489
1490     def visit_CFuncDefNode(self, node):
1491         self.visit_FuncDefNode(node)
1492         if node.needs_closure:
1493             error(node.pos, "closures inside cdef functions not yet supported")
1494         return node
1495
1496     def visit_LambdaNode(self, node):
1497         self.needs_closure = False
1498         self.visitchildren(node)
1499         node.needs_closure = self.needs_closure
1500         self.needs_closure = True
1501         return node
1502
1503     def visit_ClassDefNode(self, node):
1504         self.visitchildren(node)
1505         self.needs_closure = True
1506         return node
1507
1508 class CreateClosureClasses(CythonTransform):
1509     # Output closure classes in module scope for all functions
1510     # that really need it.
1511
1512     def __init__(self, context):
1513         super(CreateClosureClasses, self).__init__(context)
1514         self.path = []
1515         self.in_lambda = False
1516         self.generator_class = None
1517
1518     def visit_ModuleNode(self, node):
1519         self.module_scope = node.scope
1520         self.visitchildren(node)
1521         return node
1522
1523     def create_generator_class(self, target_module_scope, pos):
1524         if self.generator_class:
1525             return self.generator_class
1526         # XXX: make generator class creation cleaner
1527         entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1528                     objstruct_cname='__pyx_Generator_object',
1529                     typeobj_cname='__pyx_Generator_type',
1530                     pos=pos, defining=True, implementing=True)
1531         klass = entry.type.scope
1532         klass.is_internal = True
1533         klass.directives = {'final': True}
1534
1535         body_type = PyrexTypes.create_typedef_type('generator_body',
1536                                                    PyrexTypes.c_void_ptr_type,
1537                                                    '__pyx_generator_body_t')
1538         klass.declare_var(pos=pos, name='body', cname='body',
1539                           type=body_type, is_cdef=True)
1540         klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1541                           is_cdef=True)
1542         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1543                           is_cdef=True)
1544
1545         import TypeSlots
1546         e = klass.declare_pyfunction('send', pos)
1547         e.func_cname = '__Pyx_Generator_Send'
1548         e.signature = TypeSlots.binaryfunc
1549
1550         e = klass.declare_pyfunction('close', pos)
1551         e.func_cname = '__Pyx_Generator_Close'
1552         e.signature = TypeSlots.unaryfunc
1553
1554         e = klass.declare_pyfunction('throw', pos)
1555         e.func_cname = '__Pyx_Generator_Throw'
1556         e.signature = TypeSlots.pyfunction_signature
1557
1558         e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1559         e.func_cname = 'PyObject_SelfIter'
1560
1561         e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1562         e.func_cname = '__Pyx_Generator_Next'
1563
1564         self.generator_class = entry.type
1565         return self.generator_class
1566
1567     def find_entries_used_in_closures(self, node):
1568         from_closure = []
1569         in_closure = []
1570         for name, entry in node.local_scope.entries.items():
1571             if entry.from_closure:
1572                 from_closure.append((name, entry))
1573             elif entry.in_closure:
1574                 in_closure.append((name, entry))
1575         return from_closure, in_closure
1576
1577     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1578         # skip generator body
1579         if node.is_generator_body:
1580             return
1581         # move local variables into closure
1582         if node.is_generator:
1583             for entry in node.local_scope.entries.values():
1584                 if not entry.from_closure:
1585                     entry.in_closure = True
1586
1587         from_closure, in_closure = self.find_entries_used_in_closures(node)
1588         in_closure.sort()
1589
1590         # Now from the begining
1591         node.needs_closure = False
1592         node.needs_outer_scope = False
1593
1594         func_scope = node.local_scope
1595         cscope = node.entry.scope
1596         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1597             cscope = cscope.outer_scope
1598
1599         if not from_closure and (self.path or inner_node):
1600             if not inner_node:
1601                 if not node.assmt:
1602                     raise InternalError, "DefNode does not have assignment node"
1603                 inner_node = node.assmt.rhs
1604             inner_node.needs_self_code = False
1605             node.needs_outer_scope = False
1606
1607         if node.is_generator:
1608             generator_class = self.create_generator_class(target_module_scope, node.pos)
1609         elif not in_closure and not from_closure:
1610             return
1611         elif not in_closure:
1612             func_scope.is_passthrough = True
1613             func_scope.scope_class = cscope.scope_class
1614             node.needs_outer_scope = True
1615             return
1616
1617         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1618
1619         if node.is_generator:
1620             entry = target_module_scope.declare_c_class(name = as_name,
1621                         pos = node.pos, defining = True, implementing = True, base_type=generator_class)
1622         else:
1623             entry = target_module_scope.declare_c_class(name = as_name,
1624                         pos = node.pos, defining = True, implementing = True)
1625         func_scope.scope_class = entry
1626         class_scope = entry.type.scope
1627         class_scope.is_internal = True
1628         class_scope.directives = {'final': True}
1629
1630         if from_closure:
1631             assert cscope.is_closure_scope
1632             class_scope.declare_var(pos=node.pos,
1633                                     name=Naming.outer_scope_cname,
1634                                     cname=Naming.outer_scope_cname,
1635                                     type=cscope.scope_class.type,
1636                                     is_cdef=True)
1637             node.needs_outer_scope = True
1638         for name, entry in in_closure:
1639             class_scope.declare_var(pos=entry.pos,
1640                                     name=entry.name,
1641                                     cname=entry.cname,
1642                                     type=entry.type,
1643                                     is_cdef=True)
1644         node.needs_closure = True
1645         # Do it here because other classes are already checked
1646         target_module_scope.check_c_class(func_scope.scope_class)
1647
1648     def visit_LambdaNode(self, node):
1649         was_in_lambda = self.in_lambda
1650         self.in_lambda = True
1651         self.create_class_from_scope(node.def_node, self.module_scope, node)
1652         self.visitchildren(node)
1653         self.in_lambda = was_in_lambda
1654         return node
1655
1656     def visit_FuncDefNode(self, node):
1657         if self.in_lambda:
1658             self.visitchildren(node)
1659             return node
1660         if node.needs_closure or self.path:
1661             self.create_class_from_scope(node, self.module_scope)
1662             self.path.append(node)
1663             self.visitchildren(node)
1664             self.path.pop()
1665         return node
1666
1667
1668 class GilCheck(VisitorTransform):
1669     """
1670     Call `node.gil_check(env)` on each node to make sure we hold the
1671     GIL when we need it.  Raise an error when on Python operations
1672     inside a `nogil` environment.
1673     """
1674     def __call__(self, root):
1675         self.env_stack = [root.scope]
1676         self.nogil = False
1677         return super(GilCheck, self).__call__(root)
1678
1679     def visit_FuncDefNode(self, node):
1680         self.env_stack.append(node.local_scope)
1681         was_nogil = self.nogil
1682         self.nogil = node.local_scope.nogil
1683         if self.nogil and node.nogil_check:
1684             node.nogil_check(node.local_scope)
1685         self.visitchildren(node)
1686         self.env_stack.pop()
1687         self.nogil = was_nogil
1688         return node
1689
1690     def visit_GILStatNode(self, node):
1691         env = self.env_stack[-1]
1692         if self.nogil and node.nogil_check: node.nogil_check()
1693         was_nogil = self.nogil
1694         self.nogil = (node.state == 'nogil')
1695         self.visitchildren(node)
1696         self.nogil = was_nogil
1697         return node
1698
1699     def visit_Node(self, node):
1700         if self.env_stack and self.nogil and node.nogil_check:
1701             node.nogil_check(self.env_stack[-1])
1702         self.visitchildren(node)
1703         return node
1704
1705
1706 class TransformBuiltinMethods(EnvTransform):
1707
1708     def visit_SingleAssignmentNode(self, node):
1709         if node.declaration_only:
1710             return None
1711         else:
1712             self.visitchildren(node)
1713             return node
1714
1715     def visit_AttributeNode(self, node):
1716         self.visitchildren(node)
1717         return self.visit_cython_attribute(node)
1718
1719     def visit_NameNode(self, node):
1720         return self.visit_cython_attribute(node)
1721
1722     def visit_cython_attribute(self, node):
1723         attribute = node.as_cython_attribute()
1724         if attribute:
1725             if attribute == u'compiled':
1726                 node = ExprNodes.BoolNode(node.pos, value=True)
1727             elif attribute == u'NULL':
1728                 node = ExprNodes.NullNode(node.pos)
1729             elif attribute in (u'set', u'frozenset'):
1730                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1731                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1732             elif not PyrexTypes.parse_basic_type(attribute):
1733                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1734         return node
1735
1736     def visit_SimpleCallNode(self, node):
1737
1738         # locals builtin
1739         if isinstance(node.function, ExprNodes.NameNode):
1740             if node.function.name == 'locals':
1741                 lenv = self.current_env()
1742                 entry = lenv.lookup_here('locals')
1743                 if entry:
1744                     # not the builtin 'locals'
1745                     return node
1746                 if len(node.args) > 0:
1747                     error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1748                     return node
1749                 pos = node.pos
1750                 items = [ ExprNodes.DictItemNode(pos,
1751                                                  key=ExprNodes.StringNode(pos, value=var),
1752                                                  value=ExprNodes.NameNode(pos, name=var))
1753                           for var in lenv.entries ]
1754                 return ExprNodes.DictNode(pos, key_value_pairs=items)
1755
1756         # cython.foo
1757         function = node.function.as_cython_attribute()
1758         if function:
1759             if function in InterpretCompilerDirectives.unop_method_nodes:
1760                 if len(node.args) != 1:
1761                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1762                 else:
1763                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1764             elif function in InterpretCompilerDirectives.binop_method_nodes:
1765                 if len(node.args) != 2:
1766                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1767                 else:
1768                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1769             elif function == u'cast':
1770                 if len(node.args) != 2:
1771                     error(node.function.pos, u"cast() takes exactly two arguments")
1772                 else:
1773                     type = node.args[0].analyse_as_type(self.current_env())
1774                     if type:
1775                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1776                     else:
1777                         error(node.args[0].pos, "Not a type")
1778             elif function == u'sizeof':
1779                 if len(node.args) != 1:
1780                     error(node.function.pos, u"sizeof() takes exactly one argument")
1781                 else:
1782                     type = node.args[0].analyse_as_type(self.current_env())
1783                     if type:
1784                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1785                     else:
1786                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1787             elif function == 'cmod':
1788                 if len(node.args) != 2:
1789                     error(node.function.pos, u"cmod() takes exactly two arguments")
1790                 else:
1791                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1792                     node.cdivision = True
1793             elif function == 'cdiv':
1794                 if len(node.args) != 2:
1795                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1796                 else:
1797                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1798                     node.cdivision = True
1799             elif function == u'set':
1800                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1801             else:
1802                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1803
1804         self.visitchildren(node)
1805         return node
1806
1807
1808 class DebugTransform(CythonTransform):
1809     """
1810     Create debug information and all functions' visibility to extern in order
1811     to enable debugging.
1812     """
1813
1814     def __init__(self, context, options, result):
1815         super(DebugTransform, self).__init__(context)
1816         self.visited = cython.set()
1817         # our treebuilder and debug output writer
1818         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1819         self.tb = self.context.gdb_debug_outputwriter
1820         #self.c_output_file = options.output_file
1821         self.c_output_file = result.c_file
1822
1823         # Closure support, basically treat nested functions as if the AST were
1824         # never nested
1825         self.nested_funcdefs = []
1826
1827         # tells visit_NameNode whether it should register step-into functions
1828         self.register_stepinto = False
1829
1830     def visit_ModuleNode(self, node):
1831         self.tb.module_name = node.full_module_name
1832         attrs = dict(
1833             module_name=node.full_module_name,
1834             filename=node.pos[0].filename,
1835             c_filename=self.c_output_file)
1836
1837         self.tb.start('Module', attrs)
1838
1839         # serialize functions
1840         self.tb.start('Functions')
1841         # First, serialize functions normally...
1842         self.visitchildren(node)
1843
1844         # ... then, serialize nested functions
1845         for nested_funcdef in self.nested_funcdefs:
1846             self.visit_FuncDefNode(nested_funcdef)
1847
1848         self.register_stepinto = True
1849         self.serialize_modulenode_as_function(node)
1850         self.register_stepinto = False
1851         self.tb.end('Functions')
1852
1853         # 2.3 compatibility. Serialize global variables
1854         self.tb.start('Globals')
1855         entries = {}
1856
1857         for k, v in node.scope.entries.iteritems():
1858             if (v.qualified_name not in self.visited and not
1859                 v.name.startswith('__pyx_') and not
1860                 v.type.is_cfunction and not
1861                 v.type.is_extension_type):
1862                 entries[k]= v
1863
1864         self.serialize_local_variables(entries)
1865         self.tb.end('Globals')
1866         # self.tb.end('Module') # end Module after the line number mapping in
1867         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1868         return node
1869
1870     def visit_FuncDefNode(self, node):
1871         self.visited.add(node.local_scope.qualified_name)
1872
1873         if getattr(node, 'is_wrapper', False):
1874             return node
1875
1876         if self.register_stepinto:
1877             self.nested_funcdefs.append(node)
1878             return node
1879
1880         # node.entry.visibility = 'extern'
1881         if node.py_func is None:
1882             pf_cname = ''
1883         else:
1884             pf_cname = node.py_func.entry.func_cname
1885
1886         attrs = dict(
1887             name=node.entry.name,
1888             cname=node.entry.func_cname,
1889             pf_cname=pf_cname,
1890             qualified_name=node.local_scope.qualified_name,
1891             lineno=str(node.pos[1]))
1892
1893         self.tb.start('Function', attrs=attrs)
1894
1895         self.tb.start('Locals')
1896         self.serialize_local_variables(node.local_scope.entries)
1897         self.tb.end('Locals')
1898
1899         self.tb.start('Arguments')
1900         for arg in node.local_scope.arg_entries:
1901             self.tb.start(arg.name)
1902             self.tb.end(arg.name)
1903         self.tb.end('Arguments')
1904
1905         self.tb.start('StepIntoFunctions')
1906         self.register_stepinto = True
1907         self.visitchildren(node)
1908         self.register_stepinto = False
1909         self.tb.end('StepIntoFunctions')
1910         self.tb.end('Function')
1911
1912         return node
1913
1914     def visit_NameNode(self, node):
1915         if (self.register_stepinto and
1916             node.type.is_cfunction and
1917             getattr(node, 'is_called', False) and
1918             node.entry.func_cname is not None):
1919             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1920             # declared functions are not 'in_cinclude'.
1921             # This means we will list called 'cdef' functions as
1922             # "step into functions", but this is not an issue as they will be
1923             # recognized as Cython functions anyway.
1924             attrs = dict(name=node.entry.func_cname)
1925             self.tb.start('StepIntoFunction', attrs=attrs)
1926             self.tb.end('StepIntoFunction')
1927
1928         self.visitchildren(node)
1929         return node
1930
1931     def serialize_modulenode_as_function(self, node):
1932         """
1933         Serialize the module-level code as a function so the debugger will know
1934         it's a "relevant frame" and it will know where to set the breakpoint
1935         for 'break modulename'.
1936         """
1937         name = node.full_module_name.rpartition('.')[-1]
1938
1939         cname_py2 = 'init' + name
1940         cname_py3 = 'PyInit_' + name
1941
1942         py2_attrs = dict(
1943             name=name,
1944             cname=cname_py2,
1945             pf_cname='',
1946             # Ignore the qualified_name, breakpoints should be set using
1947             # `cy break modulename:lineno` for module-level breakpoints.
1948             qualified_name='',
1949             lineno='1',
1950             is_initmodule_function="True",
1951         )
1952
1953         py3_attrs = dict(py2_attrs, cname=cname_py3)
1954
1955         self._serialize_modulenode_as_function(node, py2_attrs)
1956         self._serialize_modulenode_as_function(node, py3_attrs)
1957
1958     def _serialize_modulenode_as_function(self, node, attrs):
1959         self.tb.start('Function', attrs=attrs)
1960
1961         self.tb.start('Locals')
1962         self.serialize_local_variables(node.scope.entries)
1963         self.tb.end('Locals')
1964
1965         self.tb.start('Arguments')
1966         self.tb.end('Arguments')
1967
1968         self.tb.start('StepIntoFunctions')
1969         self.register_stepinto = True
1970         self.visitchildren(node)
1971         self.register_stepinto = False
1972         self.tb.end('StepIntoFunctions')
1973
1974         self.tb.end('Function')
1975
1976     def serialize_local_variables(self, entries):
1977         for entry in entries.values():
1978             if entry.type.is_pyobject:
1979                 vartype = 'PythonObject'
1980             else:
1981                 vartype = 'CObject'
1982
1983             if entry.from_closure:
1984                 # We're dealing with a closure where a variable from an outer
1985                 # scope is accessed, get it from the scope object.
1986                 cname = '%s->%s' % (Naming.cur_scope_cname,
1987                                     entry.outer_entry.cname)
1988
1989                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1990                                       entry.scope.name,
1991                                       entry.name)
1992             elif entry.in_closure:
1993                 cname = '%s->%s' % (Naming.cur_scope_cname,
1994                                     entry.cname)
1995                 qname = entry.qualified_name
1996             else:
1997                 cname = entry.cname
1998                 qname = entry.qualified_name
1999
2000             if not entry.pos:
2001                 # this happens for variables that are not in the user's code,
2002                 # e.g. for the global __builtins__, __doc__, etc. We can just
2003                 # set the lineno to 0 for those.
2004                 lineno = '0'
2005             else:
2006                 lineno = str(entry.pos[1])
2007
2008             attrs = dict(
2009                 name=entry.name,
2010                 cname=cname,
2011                 qualified_name=qname,
2012                 type=vartype,
2013                 lineno=lineno)
2014
2015             self.tb.start('LocalVar', attrs)
2016             self.tb.end('LocalVar')
2017