Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14 import Builtin
  15
  16 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  17 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  18 from Cython.Compiler.ModuleNode import ModuleNode
  19 from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
  20 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  21 from Cython.Compiler.StringEncoding import EncodedString
  22 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  23
  24 import copy
  25
  26
  27 class NameNodeCollector(TreeVisitor):
  28     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  29     attribute.
  30     """
  31     def __init__(self):
  32         super(NameNodeCollector, self).__init__()
  33         self.name_nodes = []
  34
  35     def visit_NameNode(self, node):
  36         self.name_nodes.append(node)
  37
  38     def visit_Node(self, node):
  39         self._visitchildren(node, None)
  40
  41
  42 class SkipDeclarations(object):
  43     """
  44     Variable and function declarations can often have a deep tree structure,
  45     and yet most transformations don't need to descend to this depth.
  46
  47     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  48     is no need to use this for transformations after that point.
  49     """
  50     def visit_CTypeDefNode(self, node):
  51         return node
  52
  53     def visit_CVarDefNode(self, node):
  54         return node
  55
  56     def visit_CDeclaratorNode(self, node):
  57         return node
  58
  59     def visit_CBaseTypeNode(self, node):
  60         return node
  61
  62     def visit_CEnumDefNode(self, node):
  63         return node
  64
  65     def visit_CStructOrUnionDefNode(self, node):
  66         return node
  67
  68 class NormalizeTree(CythonTransform):
  69     """
  70     This transform fixes up a few things after parsing
  71     in order to make the parse tree more suitable for
  72     transforms.
  73
  74     a) After parsing, blocks with only one statement will
  75     be represented by that statement, not by a StatListNode.
  76     When doing transforms this is annoying and inconsistent,
  77     as one cannot in general remove a statement in a consistent
  78     way and so on. This transform wraps any single statements
  79     in a StatListNode containing a single statement.
  80
  81     b) The PassStatNode is a noop and serves no purpose beyond
  82     plugging such one-statement blocks; i.e., once parsed a
  83 `    "pass" can just as well be represented using an empty
  84     StatListNode. This means less special cases to worry about
  85     in subsequent transforms (one always checks to see if a
  86     StatListNode has no children to see if the block is empty).
  87     """
  88
  89     def __init__(self, context):
  90         super(NormalizeTree, self).__init__(context)
  91         self.is_in_statlist = False
  92         self.is_in_expr = False
  93
  94     def visit_ExprNode(self, node):
  95         stacktmp = self.is_in_expr
  96         self.is_in_expr = True
  97         self.visitchildren(node)
  98         self.is_in_expr = stacktmp
  99         return node
 100
 101     def visit_StatNode(self, node, is_listcontainer=False):
 102         stacktmp = self.is_in_statlist
 103         self.is_in_statlist = is_listcontainer
 104         self.visitchildren(node)
 105         self.is_in_statlist = stacktmp
 106         if not self.is_in_statlist and not self.is_in_expr:
 107             return Nodes.StatListNode(pos=node.pos, stats=[node])
 108         else:
 109             return node
 110
 111     def visit_StatListNode(self, node):
 112         self.is_in_statlist = True
 113         self.visitchildren(node)
 114         self.is_in_statlist = False
 115         return node
 116
 117     def visit_ParallelAssignmentNode(self, node):
 118         return self.visit_StatNode(node, True)
 119
 120     def visit_CEnumDefNode(self, node):
 121         return self.visit_StatNode(node, True)
 122
 123     def visit_CStructOrUnionDefNode(self, node):
 124         return self.visit_StatNode(node, True)
 125
 126     # Eliminate PassStatNode
 127     def visit_PassStatNode(self, node):
 128         if not self.is_in_statlist:
 129             return Nodes.StatListNode(pos=node.pos, stats=[])
 130         else:
 131             return []
 132
 133     def visit_CDeclaratorNode(self, node):
 134         return node
 135
 136
 137 class PostParseError(CompileError): pass
 138
 139 # error strings checked by unit tests, so define them
 140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 143 class PostParse(ScopeTrackingTransform):
 144     """
 145     Basic interpretation of the parse tree, as well as validity
 146     checking that can be done on a very basic level on the parse
 147     tree (while still not being a problem with the basic syntax,
 148     as such).
 149
 150     Specifically:
 151     - Default values to cdef assignments are turned into single
 152     assignments following the declaration (everywhere but in class
 153     bodies, where they raise a compile error)
 154
 155     - Interpret some node structures into Python runtime values.
 156     Some nodes take compile-time arguments (currently:
 157     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 158     which should be interpreted. This happens in a general way
 159     and other steps should be taken to ensure validity.
 160
 161     Type arguments cannot be interpreted in this way.
 162
 163     - For __cythonbufferdefaults__ the arguments are checked for
 164     validity.
 165
 166     TemplatedTypeNode has its directives interpreted:
 167     Any first positional argument goes into the "dtype" attribute,
 168     any "ndim" keyword argument goes into the "ndim" attribute and
 169     so on. Also it is checked that the directive combination is valid.
 170     - __cythonbufferdefaults__ attributes are parsed and put into the
 171     type information.
 172
 173     Note: Currently Parsing.py does a lot of interpretation and
 174     reorganization that can be refactored into this transform
 175     if a more pure Abstract Syntax Tree is wanted.
 176     """
 177
 178     def __init__(self, context):
 179         super(PostParse, self).__init__(context)
 180         self.specialattribute_handlers = {
 181             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 182         }
 183
 184     def visit_ModuleNode(self, node):
 185         self.lambda_counter = 1
 186         self.genexpr_counter = 1
 187         return super(PostParse, self).visit_ModuleNode(node)
 188
 189     def visit_LambdaNode(self, node):
 190         # unpack a lambda expression into the corresponding DefNode
 191         lambda_id = self.lambda_counter
 192         self.lambda_counter += 1
 193         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 194         collector = YieldNodeCollector()
 195         collector.visitchildren(node.result_expr)
 196         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 197             body = Nodes.ExprStatNode(
 198                 node.result_expr.pos, expr=node.result_expr)
 199         else:
 200             body = Nodes.ReturnStatNode(
 201                 node.result_expr.pos, value=node.result_expr)
 202         node.def_node = Nodes.DefNode(
 203             node.pos, name=node.name, lambda_name=node.lambda_name,
 204             args=node.args, star_arg=node.star_arg,
 205             starstar_arg=node.starstar_arg,
 206             body=body, doc=None)
 207         self.visitchildren(node)
 208         return node
 209
 210     def visit_GeneratorExpressionNode(self, node):
 211         # unpack a generator expression into the corresponding DefNode
 212         genexpr_id = self.genexpr_counter
 213         self.genexpr_counter += 1
 214         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 215
 216         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 217                                       doc=None,
 218                                       args=[], star_arg=None,
 219                                       starstar_arg=None,
 220                                       body=node.loop)
 221         self.visitchildren(node)
 222         return node
 223
 224     # cdef variables
 225     def handle_bufferdefaults(self, decl):
 226         if not isinstance(decl.default, ExprNodes.DictNode):
 227             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 228         self.scope_node.buffer_defaults_node = decl.default
 229         self.scope_node.buffer_defaults_pos = decl.pos
 230
 231     def visit_CVarDefNode(self, node):
 232         # This assumes only plain names and pointers are assignable on
 233         # declaration. Also, it makes use of the fact that a cdef decl
 234         # must appear before the first use, so we don't have to deal with
 235         # "i = 3; cdef int i = i" and can simply move the nodes around.
 236         try:
 237             self.visitchildren(node)
 238             stats = [node]
 239             newdecls = []
 240             for decl in node.declarators:
 241                 declbase = decl
 242                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 243                     declbase = declbase.base
 244                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 245                     if declbase.default is not None:
 246                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 247                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 248                                 handler = self.specialattribute_handlers.get(decl.name)
 249                                 if handler:
 250                                     if decl is not declbase:
 251                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 252                                     handler(decl)
 253                                     continue # Remove declaration
 254                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 255                         first_assignment = self.scope_type != 'module'
 256                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 257                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 258                             rhs=declbase.default, first=first_assignment))
 259                         declbase.default = None
 260                 newdecls.append(decl)
 261             node.declarators = newdecls
 262             return stats
 263         except PostParseError, e:
 264             # An error in a cdef clause is ok, simply remove the declaration
 265             # and try to move on to report more errors
 266             self.context.nonfatal_error(e)
 267             return None
 268
 269     # Split parallel assignments (a,b = b,a) into separate partial
 270     # assignments that are executed rhs-first using temps.  This
 271     # restructuring must be applied before type analysis so that known
 272     # types on rhs and lhs can be matched directly.  It is required in
 273     # the case that the types cannot be coerced to a Python type in
 274     # order to assign from a tuple.
 275
 276     def visit_SingleAssignmentNode(self, node):
 277         self.visitchildren(node)
 278         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 279
 280     def visit_CascadedAssignmentNode(self, node):
 281         self.visitchildren(node)
 282         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 283
 284     def _visit_assignment_node(self, node, expr_list):
 285         """Flatten parallel assignments into separate single
 286         assignments or cascaded assignments.
 287         """
 288         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 289             # no parallel assignments => nothing to do
 290             return node
 291
 292         expr_list_list = []
 293         flatten_parallel_assignments(expr_list, expr_list_list)
 294         temp_refs = []
 295         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 296
 297         nodes = []
 298         for expr_list in expr_list_list:
 299             lhs_list = expr_list[:-1]
 300             rhs = expr_list[-1]
 301             if len(lhs_list) == 1:
 302                 node = Nodes.SingleAssignmentNode(rhs.pos,
 303                     lhs = lhs_list[0], rhs = rhs)
 304             else:
 305                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 306                     lhs_list = lhs_list, rhs = rhs)
 307             nodes.append(node)
 308
 309         if len(nodes) == 1:
 310             assign_node = nodes[0]
 311         else:
 312             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 313
 314         if temp_refs:
 315             duplicates_and_temps = [ (temp.expression, temp)
 316                                      for temp in temp_refs ]
 317             sort_common_subsequences(duplicates_and_temps)
 318             for _, temp_ref in duplicates_and_temps[::-1]:
 319                 assign_node = LetNode(temp_ref, assign_node)
 320
 321         return assign_node
 322
 323     def _flatten_sequence(self, seq, result):
 324         for arg in seq.args:
 325             if arg.is_sequence_constructor:
 326                 self._flatten_sequence(arg, result)
 327             else:
 328                 result.append(arg)
 329         return result
 330
 331     def visit_DelStatNode(self, node):
 332         self.visitchildren(node)
 333         node.args = self._flatten_sequence(node, [])
 334         return node
 335
 336
 337 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 338     """Replace rhs items by LetRefNodes if they appear more than once.
 339     Creates a sequence of LetRefNodes that set up the required temps
 340     and appends them to ref_node_sequence.  The input list is modified
 341     in-place.
 342     """
 343     seen_nodes = cython.set()
 344     ref_nodes = {}
 345     def find_duplicates(node):
 346         if node.is_literal or node.is_name:
 347             # no need to replace those; can't include attributes here
 348             # as their access is not necessarily side-effect free
 349             return
 350         if node in seen_nodes:
 351             if node not in ref_nodes:
 352                 ref_node = LetRefNode(node)
 353                 ref_nodes[node] = ref_node
 354                 ref_node_sequence.append(ref_node)
 355         else:
 356             seen_nodes.add(node)
 357             if node.is_sequence_constructor:
 358                 for item in node.args:
 359                     find_duplicates(item)
 360
 361     for expr_list in expr_list_list:
 362         rhs = expr_list[-1]
 363         find_duplicates(rhs)
 364     if not ref_nodes:
 365         return
 366
 367     def substitute_nodes(node):
 368         if node in ref_nodes:
 369             return ref_nodes[node]
 370         elif node.is_sequence_constructor:
 371             node.args = list(map(substitute_nodes, node.args))
 372         return node
 373
 374     # replace nodes inside of the common subexpressions
 375     for node in ref_nodes:
 376         if node.is_sequence_constructor:
 377             node.args = list(map(substitute_nodes, node.args))
 378
 379     # replace common subexpressions on all rhs items
 380     for expr_list in expr_list_list:
 381         expr_list[-1] = substitute_nodes(expr_list[-1])
 382
 383 def sort_common_subsequences(items):
 384     """Sort items/subsequences so that all items and subsequences that
 385     an item contains appear before the item itself.  This is needed
 386     because each rhs item must only be evaluated once, so its value
 387     must be evaluated first and then reused when packing sequences
 388     that contain it.
 389
 390     This implies a partial order, and the sort must be stable to
 391     preserve the original order as much as possible, so we use a
 392     simple insertion sort (which is very fast for short sequences, the
 393     normal case in practice).
 394     """
 395     def contains(seq, x):
 396         for item in seq:
 397             if item is x:
 398                 return True
 399             elif item.is_sequence_constructor and contains(item.args, x):
 400                 return True
 401         return False
 402     def lower_than(a,b):
 403         return b.is_sequence_constructor and contains(b.args, a)
 404
 405     for pos, item in enumerate(items):
 406         key = item[1] # the ResultRefNode which has already been injected into the sequences
 407         new_pos = pos
 408         for i in xrange(pos-1, -1, -1):
 409             if lower_than(key, items[i][0]):
 410                 new_pos = i
 411         if new_pos != pos:
 412             for i in xrange(pos, new_pos, -1):
 413                 items[i] = items[i-1]
 414             items[new_pos] = item
 415
 416 def flatten_parallel_assignments(input, output):
 417     #  The input is a list of expression nodes, representing the LHSs
 418     #  and RHS of one (possibly cascaded) assignment statement.  For
 419     #  sequence constructors, rearranges the matching parts of both
 420     #  sides into a list of equivalent assignments between the
 421     #  individual elements.  This transformation is applied
 422     #  recursively, so that nested structures get matched as well.
 423     rhs = input[-1]
 424     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 425         output.append(input)
 426         return
 427
 428     complete_assignments = []
 429
 430     rhs_size = len(rhs.args)
 431     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 432     starred_assignments = []
 433     for lhs in input[:-1]:
 434         if not lhs.is_sequence_constructor:
 435             if lhs.is_starred:
 436                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 437             complete_assignments.append(lhs)
 438             continue
 439         lhs_size = len(lhs.args)
 440         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 441         if starred_targets > 1:
 442             error(lhs.pos, "more than 1 starred expression in assignment")
 443             output.append([lhs,rhs])
 444             continue
 445         elif lhs_size - starred_targets > rhs_size:
 446             error(lhs.pos, "need more than %d value%s to unpack"
 447                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 448             output.append([lhs,rhs])
 449             continue
 450         elif starred_targets:
 451             map_starred_assignment(lhs_targets, starred_assignments,
 452                                    lhs.args, rhs.args)
 453         elif lhs_size < rhs_size:
 454             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 455                   % (lhs_size, rhs_size))
 456             output.append([lhs,rhs])
 457             continue
 458         else:
 459             for targets, expr in zip(lhs_targets, lhs.args):
 460                 targets.append(expr)
 461
 462     if complete_assignments:
 463         complete_assignments.append(rhs)
 464         output.append(complete_assignments)
 465
 466     # recursively flatten partial assignments
 467     for cascade, rhs in zip(lhs_targets, rhs.args):
 468         if cascade:
 469             cascade.append(rhs)
 470             flatten_parallel_assignments(cascade, output)
 471
 472     # recursively flatten starred assignments
 473     for cascade in starred_assignments:
 474         if cascade[0].is_sequence_constructor:
 475             flatten_parallel_assignments(cascade, output)
 476         else:
 477             output.append(cascade)
 478
 479 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 480     # Appends the fixed-position LHS targets to the target list that
 481     # appear left and right of the starred argument.
 482     #
 483     # The starred_assignments list receives a new tuple
 484     # (lhs_target, rhs_values_list) that maps the remaining arguments
 485     # (those that match the starred target) to a list.
 486
 487     # left side of the starred target
 488     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 489         if expr.is_starred:
 490             starred = i
 491             lhs_remaining = len(lhs_args) - i - 1
 492             break
 493         targets.append(expr)
 494     else:
 495         raise InternalError("no starred arg found when splitting starred assignment")
 496
 497     # right side of the starred target
 498     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 499                                             lhs_args[starred + 1:])):
 500         targets.append(expr)
 501
 502     # the starred target itself, must be assigned a (potentially empty) list
 503     target = lhs_args[starred].target # unpack starred node
 504     starred_rhs = rhs_args[starred:]
 505     if lhs_remaining:
 506         starred_rhs = starred_rhs[:-lhs_remaining]
 507     if starred_rhs:
 508         pos = starred_rhs[0].pos
 509     else:
 510         pos = target.pos
 511     starred_assignments.append([
 512         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 513
 514
 515 class PxdPostParse(CythonTransform, SkipDeclarations):
 516     """
 517     Basic interpretation/validity checking that should only be
 518     done on pxd trees.
 519
 520     A lot of this checking currently happens in the parser; but
 521     what is listed below happens here.
 522
 523     - "def" functions are let through only if they fill the
 524     getbuffer/releasebuffer slots
 525
 526     - cdef functions are let through only if they are on the
 527     top level and are declared "inline"
 528     """
 529     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 530     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 531
 532     def __call__(self, node):
 533         self.scope_type = 'pxd'
 534         return super(PxdPostParse, self).__call__(node)
 535
 536     def visit_CClassDefNode(self, node):
 537         old = self.scope_type
 538         self.scope_type = 'cclass'
 539         self.visitchildren(node)
 540         self.scope_type = old
 541         return node
 542
 543     def visit_FuncDefNode(self, node):
 544         # FuncDefNode always come with an implementation (without
 545         # an imp they are CVarDefNodes..)
 546         err = self.ERR_INLINE_ONLY
 547
 548         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 549             and node.name in ('__getbuffer__', '__releasebuffer__')):
 550             err = None # allow these slots
 551
 552         if isinstance(node, Nodes.CFuncDefNode):
 553             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 554                 node.inline_in_pxd = True
 555                 if node.visibility != 'private':
 556                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 557                 elif node.api:
 558                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 559                 else:
 560                     err = None # allow inline function
 561             else:
 562                 err = self.ERR_INLINE_ONLY
 563
 564         if err:
 565             self.context.nonfatal_error(PostParseError(node.pos, err))
 566             return None
 567         else:
 568             return node
 569
 570 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 571     """
 572     After parsing, directives can be stored in a number of places:
 573     - #cython-comments at the top of the file (stored in ModuleNode)
 574     - Command-line arguments overriding these
 575     - @cython.directivename decorators
 576     - with cython.directivename: statements
 577
 578     This transform is responsible for interpreting these various sources
 579     and store the directive in two ways:
 580     - Set the directives attribute of the ModuleNode for global directives.
 581     - Use a CompilerDirectivesNode to override directives for a subtree.
 582
 583     (The first one is primarily to not have to modify with the tree
 584     structure, so that ModuleNode stay on top.)
 585
 586     The directives are stored in dictionaries from name to value in effect.
 587     Each such dictionary is always filled in for all possible directives,
 588     using default values where no value is given by the user.
 589
 590     The available directives are controlled in Options.py.
 591
 592     Note that we have to run this prior to analysis, and so some minor
 593     duplication of functionality has to occur: We manually track cimports
 594     and which names the "cython" module may have been imported to.
 595     """
 596     unop_method_nodes = {
 597         'typeof': ExprNodes.TypeofNode,
 598
 599         'operator.address': ExprNodes.AmpersandNode,
 600         'operator.dereference': ExprNodes.DereferenceNode,
 601         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 602         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 603         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 604         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 605
 606         # For backwards compatability.
 607         'address': ExprNodes.AmpersandNode,
 608     }
 609
 610     binop_method_nodes = {
 611         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 612     }
 613
 614     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 615                                   'cast', 'pointer', 'compiled', 'NULL'])
 616     special_methods.update(unop_method_nodes.keys())
 617
 618     def __init__(self, context, compilation_directive_defaults):
 619         super(InterpretCompilerDirectives, self).__init__(context)
 620         self.compilation_directive_defaults = {}
 621         for key, value in compilation_directive_defaults.items():
 622             self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
 623         self.cython_module_names = cython.set()
 624         self.directive_names = {}
 625
 626     def check_directive_scope(self, pos, directive, scope):
 627         legal_scopes = Options.directive_scopes.get(directive, None)
 628         if legal_scopes and scope not in legal_scopes:
 629             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 630                                         'is not allowed in %s scope' % (directive, scope)))
 631             return False
 632         else:
 633             return True
 634
 635     # Set up processing and handle the cython: comments.
 636     def visit_ModuleNode(self, node):
 637         for key, value in node.directive_comments.items():
 638             if not self.check_directive_scope(node.pos, key, 'module'):
 639                 self.wrong_scope_error(node.pos, key, 'module')
 640                 del node.directive_comments[key]
 641
 642         directives = copy.deepcopy(Options.directive_defaults)
 643         directives.update(copy.deepcopy(self.compilation_directive_defaults))
 644         directives.update(node.directive_comments)
 645         self.directives = directives
 646         node.directives = directives
 647         self.visitchildren(node)
 648         node.cython_module_names = self.cython_module_names
 649         return node
 650
 651     # The following four functions track imports and cimports that
 652     # begin with "cython"
 653     def is_cython_directive(self, name):
 654         return (name in Options.directive_types or
 655                 name in self.special_methods or
 656                 PyrexTypes.parse_basic_type(name))
 657
 658     def visit_CImportStatNode(self, node):
 659         if node.module_name == u"cython":
 660             self.cython_module_names.add(node.as_name or u"cython")
 661         elif node.module_name.startswith(u"cython."):
 662             if node.as_name:
 663                 self.directive_names[node.as_name] = node.module_name[7:]
 664             else:
 665                 self.cython_module_names.add(u"cython")
 666             # if this cimport was a compiler directive, we don't
 667             # want to leave the cimport node sitting in the tree
 668             return None
 669         return node
 670
 671     def visit_FromCImportStatNode(self, node):
 672         if (node.module_name == u"cython") or \
 673                node.module_name.startswith(u"cython."):
 674             submodule = (node.module_name + u".")[7:]
 675             newimp = []
 676             for pos, name, as_name, kind in node.imported_names:
 677                 full_name = submodule + name
 678                 if self.is_cython_directive(full_name):
 679                     if as_name is None:
 680                         as_name = full_name
 681                     self.directive_names[as_name] = full_name
 682                     if kind is not None:
 683                         self.context.nonfatal_error(PostParseError(pos,
 684                             "Compiler directive imports must be plain imports"))
 685                 else:
 686                     newimp.append((pos, name, as_name, kind))
 687             if not newimp:
 688                 return None
 689             node.imported_names = newimp
 690         return node
 691
 692     def visit_FromImportStatNode(self, node):
 693         if (node.module.module_name.value == u"cython") or \
 694                node.module.module_name.value.startswith(u"cython."):
 695             submodule = (node.module.module_name.value + u".")[7:]
 696             newimp = []
 697             for name, name_node in node.items:
 698                 full_name = submodule + name
 699                 if self.is_cython_directive(full_name):
 700                     self.directive_names[name_node.name] = full_name
 701                 else:
 702                     newimp.append((name, name_node))
 703             if not newimp:
 704                 return None
 705             node.items = newimp
 706         return node
 707
 708     def visit_SingleAssignmentNode(self, node):
 709         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 710                 node.rhs.module_name.value == u'cython'):
 711             node = Nodes.CImportStatNode(node.pos,
 712                                          module_name = u'cython',
 713                                          as_name = node.lhs.name)
 714             self.visit_CImportStatNode(node)
 715         else:
 716             self.visitchildren(node)
 717         return node
 718
 719     def visit_NameNode(self, node):
 720         if node.name in self.cython_module_names:
 721             node.is_cython_module = True
 722         else:
 723             node.cython_attribute = self.directive_names.get(node.name)
 724         return node
 725
 726     def try_to_parse_directives(self, node):
 727         # If node is the contents of an directive (in a with statement or
 728         # decorator), returns a list of (directivename, value) pairs.
 729         # Otherwise, returns None
 730         if isinstance(node, ExprNodes.CallNode):
 731             self.visit(node.function)
 732             optname = node.function.as_cython_attribute()
 733             if optname:
 734                 directivetype = Options.directive_types.get(optname)
 735                 if directivetype:
 736                     args, kwds = node.explicit_args_kwds()
 737                     directives = []
 738                     key_value_pairs = []
 739                     if kwds is not None and directivetype is not dict:
 740                         for keyvalue in kwds.key_value_pairs:
 741                             key, value = keyvalue
 742                             sub_optname = "%s.%s" % (optname, key.value)
 743                             if Options.directive_types.get(sub_optname):
 744                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 745                             else:
 746                                 key_value_pairs.append(keyvalue)
 747                         if not key_value_pairs:
 748                             kwds = None
 749                         else:
 750                             kwds.key_value_pairs = key_value_pairs
 751                         if directives and not kwds and not args:
 752                             return directives
 753                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 754                     return directives
 755         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 756             self.visit(node)
 757             optname = node.as_cython_attribute()
 758             if optname:
 759                 directivetype = Options.directive_types.get(optname)
 760                 if directivetype is bool:
 761                     return [(optname, True)]
 762                 elif directivetype is None:
 763                     return [(optname, None)]
 764                 else:
 765                     raise PostParseError(
 766                         node.pos, "The '%s' directive should be used as a function call." % optname)
 767         return None
 768
 769     def try_to_parse_directive(self, optname, args, kwds, pos):
 770         directivetype = Options.directive_types.get(optname)
 771         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 772             return optname, Options.directive_defaults[optname]
 773         elif directivetype is bool:
 774             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 775                 raise PostParseError(pos,
 776                     'The %s directive takes one compile-time boolean argument' % optname)
 777             return (optname, args[0].value)
 778         elif directivetype is str:
 779             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 780                                                                               ExprNodes.UnicodeNode)):
 781                 raise PostParseError(pos,
 782                     'The %s directive takes one compile-time string argument' % optname)
 783             return (optname, str(args[0].value))
 784         elif directivetype is dict:
 785             if len(args) != 0:
 786                 raise PostParseError(pos,
 787                     'The %s directive takes no prepositional arguments' % optname)
 788             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 789         elif directivetype is list:
 790             if kwds and len(kwds) != 0:
 791                 raise PostParseError(pos,
 792                     'The %s directive takes no keyword arguments' % optname)
 793             return optname, [ str(arg.value) for arg in args ]
 794         else:
 795             assert False
 796
 797     def visit_with_directives(self, body, directives):
 798         olddirectives = self.directives
 799         newdirectives = copy.copy(olddirectives)
 800         newdirectives.update(directives)
 801         self.directives = newdirectives
 802         assert isinstance(body, Nodes.StatListNode), body
 803         retbody = self.visit_Node(body)
 804         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 805                                                  directives=newdirectives)
 806         self.directives = olddirectives
 807         return directive
 808
 809     # Handle decorators
 810     def visit_FuncDefNode(self, node):
 811         directives = self._extract_directives(node, 'function')
 812         if not directives:
 813             return self.visit_Node(node)
 814         body = Nodes.StatListNode(node.pos, stats=[node])
 815         return self.visit_with_directives(body, directives)
 816
 817     def visit_CVarDefNode(self, node):
 818         if not node.decorators:
 819             return node
 820         for dec in node.decorators:
 821             for directive in self.try_to_parse_directives(dec.decorator) or ():
 822                 if directive is not None and directive[0] == u'locals':
 823                     node.directive_locals = directive[1]
 824                 else:
 825                     self.context.nonfatal_error(PostParseError(dec.pos,
 826                         "Cdef functions can only take cython.locals() decorator."))
 827         return node
 828
 829     def visit_CClassDefNode(self, node):
 830         directives = self._extract_directives(node, 'cclass')
 831         if not directives:
 832             return self.visit_Node(node)
 833         body = Nodes.StatListNode(node.pos, stats=[node])
 834         return self.visit_with_directives(body, directives)
 835
 836     def visit_PyClassDefNode(self, node):
 837         directives = self._extract_directives(node, 'class')
 838         if not directives:
 839             return self.visit_Node(node)
 840         body = Nodes.StatListNode(node.pos, stats=[node])
 841         return self.visit_with_directives(body, directives)
 842
 843     def _extract_directives(self, node, scope_name):
 844         if not node.decorators:
 845             return {}
 846         # Split the decorators into two lists -- real decorators and directives
 847         directives = []
 848         realdecs = []
 849         for dec in node.decorators:
 850             new_directives = self.try_to_parse_directives(dec.decorator)
 851             if new_directives is not None:
 852                 for directive in new_directives:
 853                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 854                         directives.append(directive)
 855             else:
 856                 realdecs.append(dec)
 857         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 858             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 859         else:
 860             node.decorators = realdecs
 861         # merge or override repeated directives
 862         optdict = {}
 863         directives.reverse() # Decorators coming first take precedence
 864         for directive in directives:
 865             name, value = directive
 866             if name in optdict:
 867                 old_value = optdict[name]
 868                 # keywords and arg lists can be merged, everything
 869                 # else overrides completely
 870                 if isinstance(old_value, dict):
 871                     old_value.update(value)
 872                 elif isinstance(old_value, list):
 873                     old_value.extend(value)
 874                 else:
 875                     optdict[name] = value
 876             else:
 877                 optdict[name] = value
 878         return optdict
 879
 880     # Handle with statements
 881     def visit_WithStatNode(self, node):
 882         directive_dict = {}
 883         for directive in self.try_to_parse_directives(node.manager) or []:
 884             if directive is not None:
 885                 if node.target is not None:
 886                     self.context.nonfatal_error(
 887                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 888                 else:
 889                     name, value = directive
 890                     if name == 'nogil':
 891                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 892                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 893                         return self.visit_Node(node)
 894                     if self.check_directive_scope(node.pos, name, 'with statement'):
 895                         directive_dict[name] = value
 896         if directive_dict:
 897             return self.visit_with_directives(node.body, directive_dict)
 898         return self.visit_Node(node)
 899
 900 class WithTransform(CythonTransform, SkipDeclarations):
 901
 902     # EXCINFO is manually set to a variable that contains
 903     # the exc_info() tuple that can be generated by the enclosing except
 904     # statement.
 905     template_without_target = TreeFragment(u"""
 906         MGR = EXPR
 907         EXIT = MGR.__exit__
 908         MGR.__enter__()
 909         EXC = True
 910         try:
 911             try:
 912                 BODY
 913             except:
 914                 EXC = False
 915                 if not EXIT(*EXCINFO):
 916                     raise
 917         finally:
 918             if EXC:
 919                 EXIT(None, None, None)
 920     """, temps=[u'MGR', u'EXC', u"EXIT"],
 921     pipeline=[NormalizeTree(None)])
 922
 923     template_with_target = TreeFragment(u"""
 924         MGR = EXPR
 925         EXIT = MGR.__exit__
 926         VALUE = MGR.__enter__()
 927         EXC = True
 928         try:
 929             try:
 930                 TARGET = VALUE
 931                 BODY
 932             except:
 933                 EXC = False
 934                 if not EXIT(*EXCINFO):
 935                     raise
 936         finally:
 937             if EXC:
 938                 EXIT(None, None, None)
 939             MGR = EXIT = VALUE = None
 940     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 941     pipeline=[NormalizeTree(None)])
 942
 943     def visit_WithStatNode(self, node):
 944         exc_info = ResultRefNode(pos=node.pos, type=Builtin.tuple_type, may_hold_none=False)
 945         self.visitchildren(node, ['body'])
 946         if node.target is not None:
 947             result = self.template_with_target.substitute({
 948                 u'EXPR' : node.manager,
 949                 u'BODY' : node.body,
 950                 u'TARGET' : node.target,
 951                 u'EXCINFO' : exc_info,
 952                 }, pos=node.pos)
 953         else:
 954             result = self.template_without_target.substitute({
 955                 u'EXPR' : node.manager,
 956                 u'BODY' : node.body,
 957                 u'EXCINFO' : exc_info,
 958                 }, pos=node.pos)
 959
 960         # Set except excinfo target to EXCINFO
 961         try_except = result.body.stats[-1].body.stats[-1]
 962         try_except.except_clauses[0].excinfo_target = exc_info
 963
 964         return result
 965
 966     def visit_ExprNode(self, node):
 967         # With statements are never inside expressions.
 968         return node
 969
 970
 971 class DecoratorTransform(CythonTransform, SkipDeclarations):
 972
 973     def visit_DefNode(self, func_node):
 974         self.visitchildren(func_node)
 975         if not func_node.decorators:
 976             return func_node
 977         return self._handle_decorators(
 978             func_node, func_node.name)
 979
 980     def visit_CClassDefNode(self, class_node):
 981         # This doesn't currently work, so it's disabled.
 982         #
 983         # Problem: assignments to cdef class names do not work.  They
 984         # would require an additional check anyway, as the extension
 985         # type must not change its C type, so decorators cannot
 986         # replace an extension type, just alter it and return it.
 987
 988         self.visitchildren(class_node)
 989         if not class_node.decorators:
 990             return class_node
 991         error(class_node.pos,
 992               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 993         return class_node
 994         #return self._handle_decorators(
 995         #    class_node, class_node.class_name)
 996
 997     def visit_ClassDefNode(self, class_node):
 998         self.visitchildren(class_node)
 999         if not class_node.decorators:
1000             return class_node
1001         return self._handle_decorators(
1002             class_node, class_node.name)
1003
1004     def _handle_decorators(self, node, name):
1005         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1006         for decorator in node.decorators[::-1]:
1007             decorator_result = ExprNodes.SimpleCallNode(
1008                 decorator.pos,
1009                 function = decorator.decorator,
1010                 args = [decorator_result])
1011
1012         name_node = ExprNodes.NameNode(node.pos, name = name)
1013         reassignment = Nodes.SingleAssignmentNode(
1014             node.pos,
1015             lhs = name_node,
1016             rhs = decorator_result)
1017         return [node, reassignment]
1018
1019
1020 class AnalyseDeclarationsTransform(CythonTransform):
1021
1022     basic_property = TreeFragment(u"""
1023 property NAME:
1024     def __get__(self):
1025         return ATTR
1026     def __set__(self, value):
1027         ATTR = value
1028     """, level='c_class')
1029     basic_pyobject_property = TreeFragment(u"""
1030 property NAME:
1031     def __get__(self):
1032         return ATTR
1033     def __set__(self, value):
1034         ATTR = value
1035     def __del__(self):
1036         ATTR = None
1037     """, level='c_class')
1038     basic_property_ro = TreeFragment(u"""
1039 property NAME:
1040     def __get__(self):
1041         return ATTR
1042     """, level='c_class')
1043
1044     struct_or_union_wrapper = TreeFragment(u"""
1045 cdef class NAME:
1046     cdef TYPE value
1047     def __init__(self, MEMBER=None):
1048         cdef int count
1049         count = 0
1050         INIT_ASSIGNMENTS
1051         if IS_UNION and count > 1:
1052             raise ValueError, "At most one union member should be specified."
1053     def __str__(self):
1054         return STR_FORMAT % MEMBER_TUPLE
1055     def __repr__(self):
1056         return REPR_FORMAT % MEMBER_TUPLE
1057     """)
1058
1059     init_assignment = TreeFragment(u"""
1060 if VALUE is not None:
1061     ATTR = VALUE
1062     count += 1
1063     """)
1064
1065     def __call__(self, root):
1066         self.env_stack = [root.scope]
1067         # needed to determine if a cdef var is declared after it's used.
1068         self.seen_vars_stack = []
1069         return super(AnalyseDeclarationsTransform, self).__call__(root)
1070
1071     def visit_NameNode(self, node):
1072         self.seen_vars_stack[-1].add(node.name)
1073         return node
1074
1075     def visit_ModuleNode(self, node):
1076         self.seen_vars_stack.append(cython.set())
1077         node.analyse_declarations(self.env_stack[-1])
1078         self.visitchildren(node)
1079         self.seen_vars_stack.pop()
1080         return node
1081
1082     def visit_LambdaNode(self, node):
1083         node.analyse_declarations(self.env_stack[-1])
1084         self.visitchildren(node)
1085         return node
1086
1087     def visit_ClassDefNode(self, node):
1088         self.env_stack.append(node.scope)
1089         self.visitchildren(node)
1090         self.env_stack.pop()
1091         return node
1092
1093     def visit_CClassDefNode(self, node):
1094         node = self.visit_ClassDefNode(node)
1095         if node.scope and node.scope.implemented:
1096             stats = []
1097             for entry in node.scope.var_entries:
1098                 if entry.needs_property:
1099                     property = self.create_Property(entry)
1100                     property.analyse_declarations(node.scope)
1101                     self.visit(property)
1102                     stats.append(property)
1103             if stats:
1104                 node.body.stats += stats
1105         return node
1106
1107     def visit_FuncDefNode(self, node):
1108         self.seen_vars_stack.append(cython.set())
1109         lenv = node.local_scope
1110         node.body.analyse_control_flow(lenv) # this will be totally refactored
1111         node.declare_arguments(lenv)
1112         for var, type_node in node.directive_locals.items():
1113             if not lenv.lookup_here(var):   # don't redeclare args
1114                 type = type_node.analyse_as_type(lenv)
1115                 if type:
1116                     lenv.declare_var(var, type, type_node.pos)
1117                 else:
1118                     error(type_node.pos, "Not a type")
1119         node.body.analyse_declarations(lenv)
1120         self.env_stack.append(lenv)
1121         self.visitchildren(node)
1122         self.env_stack.pop()
1123         self.seen_vars_stack.pop()
1124         return node
1125
1126     def visit_ScopedExprNode(self, node):
1127         env = self.env_stack[-1]
1128         node.analyse_declarations(env)
1129         # the node may or may not have a local scope
1130         if node.has_local_scope:
1131             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1132             self.env_stack.append(node.expr_scope)
1133             node.analyse_scoped_declarations(node.expr_scope)
1134             self.visitchildren(node)
1135             self.env_stack.pop()
1136             self.seen_vars_stack.pop()
1137         else:
1138             node.analyse_scoped_declarations(env)
1139             self.visitchildren(node)
1140         return node
1141
1142     def visit_TempResultFromStatNode(self, node):
1143         self.visitchildren(node)
1144         node.analyse_declarations(self.env_stack[-1])
1145         return node
1146
1147     def visit_CStructOrUnionDefNode(self, node):
1148         # Create a wrapper node if needed.
1149         # We want to use the struct type information (so it can't happen
1150         # before this phase) but also create new objects to be declared
1151         # (so it can't happen later).
1152         # Note that we don't return the original node, as it is
1153         # never used after this phase.
1154         if True: # private (default)
1155             return None
1156
1157         self_value = ExprNodes.AttributeNode(
1158             pos = node.pos,
1159             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1160             attribute = EncodedString(u"value"))
1161         var_entries = node.entry.type.scope.var_entries
1162         attributes = []
1163         for entry in var_entries:
1164             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1165                                                       obj = self_value,
1166                                                       attribute = entry.name))
1167         # __init__ assignments
1168         init_assignments = []
1169         for entry, attr in zip(var_entries, attributes):
1170             # TODO: branch on visibility
1171             init_assignments.append(self.init_assignment.substitute({
1172                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1173                     u"ATTR": attr,
1174                 }, pos = entry.pos))
1175
1176         # create the class
1177         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1178         wrapper_class = self.struct_or_union_wrapper.substitute({
1179             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1180             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1181             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1182             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1183             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1184         }, pos = node.pos).stats[0]
1185         wrapper_class.class_name = node.name
1186         wrapper_class.shadow = True
1187         class_body = wrapper_class.body.stats
1188
1189         # fix value type
1190         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1191         class_body[0].base_type.name = node.name
1192
1193         # fix __init__ arguments
1194         init_method = class_body[1]
1195         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1196         arg_template = init_method.args[1]
1197         if not node.entry.type.is_struct:
1198             arg_template.kw_only = True
1199         del init_method.args[1]
1200         for entry, attr in zip(var_entries, attributes):
1201             arg = copy.deepcopy(arg_template)
1202             arg.declarator.name = entry.name
1203             init_method.args.append(arg)
1204
1205         # setters/getters
1206         for entry, attr in zip(var_entries, attributes):
1207             # TODO: branch on visibility
1208             if entry.type.is_pyobject:
1209                 template = self.basic_pyobject_property
1210             else:
1211                 template = self.basic_property
1212             property = template.substitute({
1213                     u"ATTR": attr,
1214                 }, pos = entry.pos).stats[0]
1215             property.name = entry.name
1216             wrapper_class.body.stats.append(property)
1217
1218         wrapper_class.analyse_declarations(self.env_stack[-1])
1219         return self.visit_CClassDefNode(wrapper_class)
1220
1221     # Some nodes are no longer needed after declaration
1222     # analysis and can be dropped. The analysis was performed
1223     # on these nodes in a seperate recursive process from the
1224     # enclosing function or module, so we can simply drop them.
1225     def visit_CDeclaratorNode(self, node):
1226         # necessary to ensure that all CNameDeclaratorNodes are visited.
1227         self.visitchildren(node)
1228         return node
1229
1230     def visit_CTypeDefNode(self, node):
1231         return node
1232
1233     def visit_CBaseTypeNode(self, node):
1234         return None
1235
1236     def visit_CEnumDefNode(self, node):
1237         if node.visibility == 'public':
1238             return node
1239         else:
1240             return None
1241
1242     def visit_CNameDeclaratorNode(self, node):
1243         if node.name in self.seen_vars_stack[-1]:
1244             entry = self.env_stack[-1].lookup(node.name)
1245             if (entry is None or entry.visibility != 'extern'
1246                 and not entry.scope.is_c_class_scope):
1247                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1248         self.visitchildren(node)
1249         return node
1250
1251     def visit_CVarDefNode(self, node):
1252         # to ensure all CNameDeclaratorNodes are visited.
1253         self.visitchildren(node)
1254         return None
1255
1256     def create_Property(self, entry):
1257         if entry.visibility == 'public':
1258             if entry.type.is_pyobject:
1259                 template = self.basic_pyobject_property
1260             else:
1261                 template = self.basic_property
1262         elif entry.visibility == 'readonly':
1263             template = self.basic_property_ro
1264         property = template.substitute({
1265                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1266                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1267                                                  attribute=entry.name),
1268             }, pos=entry.pos).stats[0]
1269         property.name = entry.name
1270         # ---------------------------------------
1271         # XXX This should go to AutoDocTransforms
1272         # ---------------------------------------
1273         if (Options.docstrings and
1274             self.current_directives['embedsignature']):
1275             attr_name = entry.name
1276             type_name = entry.type.declaration_code("", for_display=1)
1277             default_value = ''
1278             if not entry.type.is_pyobject:
1279                 type_name = "'%s'" % type_name
1280             elif entry.type.is_extension_type:
1281                 type_name = entry.type.module_name + '.' + type_name
1282             if entry.init is not None:
1283                 default_value = ' = ' + entry.init
1284             elif entry.init_to_none:
1285                 default_value = ' = ' + repr(None)
1286             docstring = attr_name + ': ' + type_name + default_value
1287             property.doc = EncodedString(docstring)
1288         # ---------------------------------------
1289         return property
1290
1291 class AnalyseExpressionsTransform(CythonTransform):
1292
1293     def visit_ModuleNode(self, node):
1294         node.scope.infer_types()
1295         node.body.analyse_expressions(node.scope)
1296         self.visitchildren(node)
1297         return node
1298
1299     def visit_FuncDefNode(self, node):
1300         node.local_scope.infer_types()
1301         node.body.analyse_expressions(node.local_scope)
1302         self.visitchildren(node)
1303         return node
1304
1305     def visit_ScopedExprNode(self, node):
1306         if node.has_local_scope:
1307             node.expr_scope.infer_types()
1308             node.analyse_scoped_expressions(node.expr_scope)
1309         self.visitchildren(node)
1310         return node
1311
1312 class ExpandInplaceOperators(EnvTransform):
1313
1314     def visit_InPlaceAssignmentNode(self, node):
1315         lhs = node.lhs
1316         rhs = node.rhs
1317         if lhs.type.is_cpp_class:
1318             # No getting around this exact operator here.
1319             return node
1320         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1321             # There is code to handle this case.
1322             return node
1323
1324         env = self.current_env()
1325         def side_effect_free_reference(node, setting=False):
1326             if isinstance(node, ExprNodes.NameNode):
1327                 return node, []
1328             elif node.type.is_pyobject and not setting:
1329                 node = LetRefNode(node)
1330                 return node, [node]
1331             elif isinstance(node, ExprNodes.IndexNode):
1332                 if node.is_buffer_access:
1333                     raise ValueError, "Buffer access"
1334                 base, temps = side_effect_free_reference(node.base)
1335                 index = LetRefNode(node.index)
1336                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1337             elif isinstance(node, ExprNodes.AttributeNode):
1338                 obj, temps = side_effect_free_reference(node.obj)
1339                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1340             else:
1341                 node = LetRefNode(node)
1342                 return node, [node]
1343         try:
1344             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1345         except ValueError:
1346             return node
1347         dup = lhs.__class__(**lhs.__dict__)
1348         binop = ExprNodes.binop_node(node.pos,
1349                                      operator = node.operator,
1350                                      operand1 = dup,
1351                                      operand2 = rhs,
1352                                      inplace=True)
1353         # Manually analyse types for new node.
1354         lhs.analyse_target_types(env)
1355         dup.analyse_types(env)
1356         binop.analyse_operation(env)
1357         node = Nodes.SingleAssignmentNode(
1358             node.pos,
1359             lhs = lhs,
1360             rhs=binop.coerce_to(lhs.type, env))
1361         # Use LetRefNode to avoid side effects.
1362         let_ref_nodes.reverse()
1363         for t in let_ref_nodes:
1364             node = LetNode(t, node)
1365         return node
1366
1367     def visit_ExprNode(self, node):
1368         # In-place assignments can't happen within an expression.
1369         return node
1370
1371
1372 class AlignFunctionDefinitions(CythonTransform):
1373     """
1374     This class takes the signatures from a .pxd file and applies them to
1375     the def methods in a .py file.
1376     """
1377
1378     def visit_ModuleNode(self, node):
1379         self.scope = node.scope
1380         self.directives = node.directives
1381         self.visitchildren(node)
1382         return node
1383
1384     def visit_PyClassDefNode(self, node):
1385         pxd_def = self.scope.lookup(node.name)
1386         if pxd_def:
1387             if pxd_def.is_cclass:
1388                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1389             else:
1390                 error(node.pos, "'%s' redeclared" % node.name)
1391                 if pxd_def.pos:
1392                     error(pxd_def.pos, "previous declaration here")
1393                 return None
1394         else:
1395             return node
1396
1397     def visit_CClassDefNode(self, node, pxd_def=None):
1398         if pxd_def is None:
1399             pxd_def = self.scope.lookup(node.class_name)
1400         if pxd_def:
1401             outer_scope = self.scope
1402             self.scope = pxd_def.type.scope
1403         self.visitchildren(node)
1404         if pxd_def:
1405             self.scope = outer_scope
1406         return node
1407
1408     def visit_DefNode(self, node):
1409         pxd_def = self.scope.lookup(node.name)
1410         if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
1411             if not pxd_def.is_cfunction:
1412                 error(node.pos, "'%s' redeclared" % node.name)
1413                 if pxd_def.pos:
1414                     error(pxd_def.pos, "previous declaration here")
1415                 return None
1416             node = node.as_cfunction(pxd_def)
1417         elif (self.scope.is_module_scope and self.directives['auto_cpdef']
1418               and node.is_cdef_func_compatible()):
1419             node = node.as_cfunction(scope=self.scope)
1420         # Enable this when nested cdef functions are allowed.
1421         # self.visitchildren(node)
1422         return node
1423
1424
1425 class YieldNodeCollector(TreeVisitor):
1426
1427     def __init__(self):
1428         super(YieldNodeCollector, self).__init__()
1429         self.yields = []
1430         self.returns = []
1431         self.has_return_value = False
1432
1433     def visit_Node(self, node):
1434         return self.visitchildren(node)
1435
1436     def visit_YieldExprNode(self, node):
1437         if self.has_return_value:
1438             error(node.pos, "'yield' outside function")
1439         self.yields.append(node)
1440         self.visitchildren(node)
1441
1442     def visit_ReturnStatNode(self, node):
1443         if node.value:
1444             self.has_return_value = True
1445             if self.yields:
1446                 error(node.pos, "'return' with argument inside generator")
1447         self.returns.append(node)
1448
1449     def visit_ClassDefNode(self, node):
1450         pass
1451
1452     def visit_FuncDefNode(self, node):
1453         pass
1454
1455     def visit_LambdaNode(self, node):
1456         pass
1457
1458     def visit_GeneratorExpressionNode(self, node):
1459         pass
1460
1461 class MarkClosureVisitor(CythonTransform):
1462
1463     def visit_ModuleNode(self, node):
1464         self.needs_closure = False
1465         self.visitchildren(node)
1466         return node
1467
1468     def visit_FuncDefNode(self, node):
1469         self.needs_closure = False
1470         self.visitchildren(node)
1471         node.needs_closure = self.needs_closure
1472         self.needs_closure = True
1473
1474         collector = YieldNodeCollector()
1475         collector.visitchildren(node)
1476
1477         if collector.yields:
1478             for i, yield_expr in enumerate(collector.yields):
1479                 yield_expr.label_num = i + 1
1480
1481             gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1482                                                name=node.name,
1483                                                body=node.body)
1484             generator = Nodes.GeneratorDefNode(pos=node.pos,
1485                                                name=node.name,
1486                                                args=node.args,
1487                                                star_arg=node.star_arg,
1488                                                starstar_arg=node.starstar_arg,
1489                                                doc=node.doc,
1490                                                decorators=node.decorators,
1491                                                gbody=gbody,
1492                                                lambda_name=node.lambda_name)
1493             return generator
1494         return node
1495
1496     def visit_CFuncDefNode(self, node):
1497         self.visit_FuncDefNode(node)
1498         if node.needs_closure:
1499             error(node.pos, "closures inside cdef functions not yet supported")
1500         return node
1501
1502     def visit_LambdaNode(self, node):
1503         self.needs_closure = False
1504         self.visitchildren(node)
1505         node.needs_closure = self.needs_closure
1506         self.needs_closure = True
1507         return node
1508
1509     def visit_ClassDefNode(self, node):
1510         self.visitchildren(node)
1511         self.needs_closure = True
1512         return node
1513
1514 class CreateClosureClasses(CythonTransform):
1515     # Output closure classes in module scope for all functions
1516     # that really need it.
1517
1518     def __init__(self, context):
1519         super(CreateClosureClasses, self).__init__(context)
1520         self.path = []
1521         self.in_lambda = False
1522         self.generator_class = None
1523
1524     def visit_ModuleNode(self, node):
1525         self.module_scope = node.scope
1526         self.visitchildren(node)
1527         return node
1528
1529     def create_generator_class(self, target_module_scope, pos):
1530         if self.generator_class:
1531             return self.generator_class
1532         # XXX: make generator class creation cleaner
1533         entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1534                     objstruct_cname='__pyx_Generator_object',
1535                     typeobj_cname='__pyx_Generator_type',
1536                     pos=pos, defining=True, implementing=True)
1537         klass = entry.type.scope
1538         klass.is_internal = True
1539         klass.directives = {'final': True}
1540
1541         body_type = PyrexTypes.create_typedef_type('generator_body',
1542                                                    PyrexTypes.c_void_ptr_type,
1543                                                    '__pyx_generator_body_t')
1544         klass.declare_var(pos=pos, name='body', cname='body',
1545                           type=body_type, is_cdef=True)
1546         klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1547                           is_cdef=True)
1548         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1549                           is_cdef=True)
1550         klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
1551                           type=PyrexTypes.py_object_type, is_cdef=True)
1552         klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
1553                           type=PyrexTypes.py_object_type, is_cdef=True)
1554         klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
1555                           type=PyrexTypes.py_object_type, is_cdef=True)
1556
1557         import TypeSlots
1558         e = klass.declare_pyfunction('send', pos)
1559         e.func_cname = '__Pyx_Generator_Send'
1560         e.signature = TypeSlots.binaryfunc
1561
1562         e = klass.declare_pyfunction('close', pos)
1563         e.func_cname = '__Pyx_Generator_Close'
1564         e.signature = TypeSlots.unaryfunc
1565
1566         e = klass.declare_pyfunction('throw', pos)
1567         e.func_cname = '__Pyx_Generator_Throw'
1568         e.signature = TypeSlots.pyfunction_signature
1569
1570         e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1571         e.func_cname = 'PyObject_SelfIter'
1572
1573         e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1574         e.func_cname = '__Pyx_Generator_Next'
1575
1576         self.generator_class = entry.type
1577         return self.generator_class
1578
1579     def find_entries_used_in_closures(self, node):
1580         from_closure = []
1581         in_closure = []
1582         for name, entry in node.local_scope.entries.items():
1583             if entry.from_closure:
1584                 from_closure.append((name, entry))
1585             elif entry.in_closure:
1586                 in_closure.append((name, entry))
1587         return from_closure, in_closure
1588
1589     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1590         # skip generator body
1591         if node.is_generator_body:
1592             return
1593         # move local variables into closure
1594         if node.is_generator:
1595             for entry in node.local_scope.entries.values():
1596                 if not entry.from_closure:
1597                     entry.in_closure = True
1598
1599         from_closure, in_closure = self.find_entries_used_in_closures(node)
1600         in_closure.sort()
1601
1602         # Now from the begining
1603         node.needs_closure = False
1604         node.needs_outer_scope = False
1605
1606         func_scope = node.local_scope
1607         cscope = node.entry.scope
1608         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1609             cscope = cscope.outer_scope
1610
1611         if not from_closure and (self.path or inner_node):
1612             if not inner_node:
1613                 if not node.assmt:
1614                     raise InternalError, "DefNode does not have assignment node"
1615                 inner_node = node.assmt.rhs
1616             inner_node.needs_self_code = False
1617             node.needs_outer_scope = False
1618
1619         base_type = None
1620         if node.is_generator:
1621             base_type = self.create_generator_class(target_module_scope, node.pos)
1622         elif not in_closure and not from_closure:
1623             return
1624         elif not in_closure:
1625             func_scope.is_passthrough = True
1626             func_scope.scope_class = cscope.scope_class
1627             node.needs_outer_scope = True
1628             return
1629
1630         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1631
1632         entry = target_module_scope.declare_c_class(
1633             name=as_name, pos=node.pos, defining=True,
1634             implementing=True, base_type=base_type)
1635
1636         func_scope.scope_class = entry
1637         class_scope = entry.type.scope
1638         class_scope.is_internal = True
1639         class_scope.directives = {'final': True}
1640
1641         if from_closure:
1642             assert cscope.is_closure_scope
1643             class_scope.declare_var(pos=node.pos,
1644                                     name=Naming.outer_scope_cname,
1645                                     cname=Naming.outer_scope_cname,
1646                                     type=cscope.scope_class.type,
1647                                     is_cdef=True)
1648             node.needs_outer_scope = True
1649         for name, entry in in_closure:
1650             closure_entry = class_scope.declare_var(pos=entry.pos,
1651                                     name=entry.name,
1652                                     cname=entry.cname,
1653                                     type=entry.type,
1654                                     is_cdef=True)
1655             if entry.is_declared_generic:
1656                 closure_entry.is_declared_generic = 1
1657         node.needs_closure = True
1658         # Do it here because other classes are already checked
1659         target_module_scope.check_c_class(func_scope.scope_class)
1660
1661     def visit_LambdaNode(self, node):
1662         was_in_lambda = self.in_lambda
1663         self.in_lambda = True
1664         self.create_class_from_scope(node.def_node, self.module_scope, node)
1665         self.visitchildren(node)
1666         self.in_lambda = was_in_lambda
1667         return node
1668
1669     def visit_FuncDefNode(self, node):
1670         if self.in_lambda:
1671             self.visitchildren(node)
1672             return node
1673         if node.needs_closure or self.path:
1674             self.create_class_from_scope(node, self.module_scope)
1675             self.path.append(node)
1676             self.visitchildren(node)
1677             self.path.pop()
1678         return node
1679
1680
1681 class GilCheck(VisitorTransform):
1682     """
1683     Call `node.gil_check(env)` on each node to make sure we hold the
1684     GIL when we need it.  Raise an error when on Python operations
1685     inside a `nogil` environment.
1686     """
1687     def __call__(self, root):
1688         self.env_stack = [root.scope]
1689         self.nogil = False
1690         return super(GilCheck, self).__call__(root)
1691
1692     def visit_FuncDefNode(self, node):
1693         self.env_stack.append(node.local_scope)
1694         was_nogil = self.nogil
1695         self.nogil = node.local_scope.nogil
1696         if self.nogil and node.nogil_check:
1697             node.nogil_check(node.local_scope)
1698         self.visitchildren(node)
1699         self.env_stack.pop()
1700         self.nogil = was_nogil
1701         return node
1702
1703     def visit_GILStatNode(self, node):
1704         env = self.env_stack[-1]
1705         if self.nogil and node.nogil_check: node.nogil_check()
1706         was_nogil = self.nogil
1707         self.nogil = (node.state == 'nogil')
1708         self.visitchildren(node)
1709         self.nogil = was_nogil
1710         return node
1711
1712     def visit_Node(self, node):
1713         if self.env_stack and self.nogil and node.nogil_check:
1714             node.nogil_check(self.env_stack[-1])
1715         self.visitchildren(node)
1716         return node
1717
1718
1719 class TransformBuiltinMethods(EnvTransform):
1720
1721     def visit_SingleAssignmentNode(self, node):
1722         if node.declaration_only:
1723             return None
1724         else:
1725             self.visitchildren(node)
1726             return node
1727
1728     def visit_AttributeNode(self, node):
1729         self.visitchildren(node)
1730         return self.visit_cython_attribute(node)
1731
1732     def visit_NameNode(self, node):
1733         return self.visit_cython_attribute(node)
1734
1735     def visit_cython_attribute(self, node):
1736         attribute = node.as_cython_attribute()
1737         if attribute:
1738             if attribute == u'compiled':
1739                 node = ExprNodes.BoolNode(node.pos, value=True)
1740             elif attribute == u'NULL':
1741                 node = ExprNodes.NullNode(node.pos)
1742             elif attribute in (u'set', u'frozenset'):
1743                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1744                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1745             elif not PyrexTypes.parse_basic_type(attribute):
1746                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1747         return node
1748
1749     def _inject_locals(self, node, func_name):
1750         # locals()/dir() builtins
1751         lenv = self.current_env()
1752         entry = lenv.lookup_here(func_name)
1753         if entry:
1754             # not the builtin
1755             return node
1756         pos = node.pos
1757         if func_name == 'locals':
1758             if len(node.args) > 0:
1759                 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
1760                       % len(node.args))
1761                 return node
1762             items = [ ExprNodes.DictItemNode(pos,
1763                                              key=ExprNodes.StringNode(pos, value=var),
1764                                              value=ExprNodes.NameNode(pos, name=var))
1765                       for var in lenv.entries ]
1766             return ExprNodes.DictNode(pos, key_value_pairs=items)
1767         else:
1768             if len(node.args) > 1:
1769                 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
1770                       % len(node.args))
1771                 return node
1772             elif len(node.args) == 1:
1773                 # optimised in Builtin.py
1774                 return node
1775             items = [ ExprNodes.StringNode(pos, value=var) for var in lenv.entries ]
1776             return ExprNodes.ListNode(pos, args=items)
1777
1778     def visit_SimpleCallNode(self, node):
1779         if isinstance(node.function, ExprNodes.NameNode):
1780             func_name = node.function.name
1781             if func_name in ('dir', 'locals'):
1782                 return self._inject_locals(node, func_name)
1783
1784         # cython.foo
1785         function = node.function.as_cython_attribute()
1786         if function:
1787             if function in InterpretCompilerDirectives.unop_method_nodes:
1788                 if len(node.args) != 1:
1789                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1790                 else:
1791                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1792             elif function in InterpretCompilerDirectives.binop_method_nodes:
1793                 if len(node.args) != 2:
1794                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1795                 else:
1796                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1797             elif function == u'cast':
1798                 if len(node.args) != 2:
1799                     error(node.function.pos, u"cast() takes exactly two arguments")
1800                 else:
1801                     type = node.args[0].analyse_as_type(self.current_env())
1802                     if type:
1803                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1804                     else:
1805                         error(node.args[0].pos, "Not a type")
1806             elif function == u'sizeof':
1807                 if len(node.args) != 1:
1808                     error(node.function.pos, u"sizeof() takes exactly one argument")
1809                 else:
1810                     type = node.args[0].analyse_as_type(self.current_env())
1811                     if type:
1812                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1813                     else:
1814                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1815             elif function == 'cmod':
1816                 if len(node.args) != 2:
1817                     error(node.function.pos, u"cmod() takes exactly two arguments")
1818                 else:
1819                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1820                     node.cdivision = True
1821             elif function == 'cdiv':
1822                 if len(node.args) != 2:
1823                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1824                 else:
1825                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1826                     node.cdivision = True
1827             elif function == u'set':
1828                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1829             else:
1830                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1831
1832         self.visitchildren(node)
1833         return node
1834
1835
1836 class DebugTransform(CythonTransform):
1837     """
1838     Create debug information and all functions' visibility to extern in order
1839     to enable debugging.
1840     """
1841
1842     def __init__(self, context, options, result):
1843         super(DebugTransform, self).__init__(context)
1844         self.visited = cython.set()
1845         # our treebuilder and debug output writer
1846         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1847         self.tb = self.context.gdb_debug_outputwriter
1848         #self.c_output_file = options.output_file
1849         self.c_output_file = result.c_file
1850
1851         # Closure support, basically treat nested functions as if the AST were
1852         # never nested
1853         self.nested_funcdefs = []
1854
1855         # tells visit_NameNode whether it should register step-into functions
1856         self.register_stepinto = False
1857
1858     def visit_ModuleNode(self, node):
1859         self.tb.module_name = node.full_module_name
1860         attrs = dict(
1861             module_name=node.full_module_name,
1862             filename=node.pos[0].filename,
1863             c_filename=self.c_output_file)
1864
1865         self.tb.start('Module', attrs)
1866
1867         # serialize functions
1868         self.tb.start('Functions')
1869         # First, serialize functions normally...
1870         self.visitchildren(node)
1871
1872         # ... then, serialize nested functions
1873         for nested_funcdef in self.nested_funcdefs:
1874             self.visit_FuncDefNode(nested_funcdef)
1875
1876         self.register_stepinto = True
1877         self.serialize_modulenode_as_function(node)
1878         self.register_stepinto = False
1879         self.tb.end('Functions')
1880
1881         # 2.3 compatibility. Serialize global variables
1882         self.tb.start('Globals')
1883         entries = {}
1884
1885         for k, v in node.scope.entries.iteritems():
1886             if (v.qualified_name not in self.visited and not
1887                 v.name.startswith('__pyx_') and not
1888                 v.type.is_cfunction and not
1889                 v.type.is_extension_type):
1890                 entries[k]= v
1891
1892         self.serialize_local_variables(entries)
1893         self.tb.end('Globals')
1894         # self.tb.end('Module') # end Module after the line number mapping in
1895         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1896         return node
1897
1898     def visit_FuncDefNode(self, node):
1899         self.visited.add(node.local_scope.qualified_name)
1900
1901         if getattr(node, 'is_wrapper', False):
1902             return node
1903
1904         if self.register_stepinto:
1905             self.nested_funcdefs.append(node)
1906             return node
1907
1908         # node.entry.visibility = 'extern'
1909         if node.py_func is None:
1910             pf_cname = ''
1911         else:
1912             pf_cname = node.py_func.entry.func_cname
1913
1914         attrs = dict(
1915             name=node.entry.name,
1916             cname=node.entry.func_cname,
1917             pf_cname=pf_cname,
1918             qualified_name=node.local_scope.qualified_name,
1919             lineno=str(node.pos[1]))
1920
1921         self.tb.start('Function', attrs=attrs)
1922
1923         self.tb.start('Locals')
1924         self.serialize_local_variables(node.local_scope.entries)
1925         self.tb.end('Locals')
1926
1927         self.tb.start('Arguments')
1928         for arg in node.local_scope.arg_entries:
1929             self.tb.start(arg.name)
1930             self.tb.end(arg.name)
1931         self.tb.end('Arguments')
1932
1933         self.tb.start('StepIntoFunctions')
1934         self.register_stepinto = True
1935         self.visitchildren(node)
1936         self.register_stepinto = False
1937         self.tb.end('StepIntoFunctions')
1938         self.tb.end('Function')
1939
1940         return node
1941
1942     def visit_NameNode(self, node):
1943         if (self.register_stepinto and
1944             node.type.is_cfunction and
1945             getattr(node, 'is_called', False) and
1946             node.entry.func_cname is not None):
1947             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1948             # declared functions are not 'in_cinclude'.
1949             # This means we will list called 'cdef' functions as
1950             # "step into functions", but this is not an issue as they will be
1951             # recognized as Cython functions anyway.
1952             attrs = dict(name=node.entry.func_cname)
1953             self.tb.start('StepIntoFunction', attrs=attrs)
1954             self.tb.end('StepIntoFunction')
1955
1956         self.visitchildren(node)
1957         return node
1958
1959     def serialize_modulenode_as_function(self, node):
1960         """
1961         Serialize the module-level code as a function so the debugger will know
1962         it's a "relevant frame" and it will know where to set the breakpoint
1963         for 'break modulename'.
1964         """
1965         name = node.full_module_name.rpartition('.')[-1]
1966
1967         cname_py2 = 'init' + name
1968         cname_py3 = 'PyInit_' + name
1969
1970         py2_attrs = dict(
1971             name=name,
1972             cname=cname_py2,
1973             pf_cname='',
1974             # Ignore the qualified_name, breakpoints should be set using
1975             # `cy break modulename:lineno` for module-level breakpoints.
1976             qualified_name='',
1977             lineno='1',
1978             is_initmodule_function="True",
1979         )
1980
1981         py3_attrs = dict(py2_attrs, cname=cname_py3)
1982
1983         self._serialize_modulenode_as_function(node, py2_attrs)
1984         self._serialize_modulenode_as_function(node, py3_attrs)
1985
1986     def _serialize_modulenode_as_function(self, node, attrs):
1987         self.tb.start('Function', attrs=attrs)
1988
1989         self.tb.start('Locals')
1990         self.serialize_local_variables(node.scope.entries)
1991         self.tb.end('Locals')
1992
1993         self.tb.start('Arguments')
1994         self.tb.end('Arguments')
1995
1996         self.tb.start('StepIntoFunctions')
1997         self.register_stepinto = True
1998         self.visitchildren(node)
1999         self.register_stepinto = False
2000         self.tb.end('StepIntoFunctions')
2001
2002         self.tb.end('Function')
2003
2004     def serialize_local_variables(self, entries):
2005         for entry in entries.values():
2006             if entry.type.is_pyobject:
2007                 vartype = 'PythonObject'
2008             else:
2009                 vartype = 'CObject'
2010
2011             if entry.from_closure:
2012                 # We're dealing with a closure where a variable from an outer
2013                 # scope is accessed, get it from the scope object.
2014                 cname = '%s->%s' % (Naming.cur_scope_cname,
2015                                     entry.outer_entry.cname)
2016
2017                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2018                                       entry.scope.name,
2019                                       entry.name)
2020             elif entry.in_closure:
2021                 cname = '%s->%s' % (Naming.cur_scope_cname,
2022                                     entry.cname)
2023                 qname = entry.qualified_name
2024             else:
2025                 cname = entry.cname
2026                 qname = entry.qualified_name
2027
2028             if not entry.pos:
2029                 # this happens for variables that are not in the user's code,
2030                 # e.g. for the global __builtins__, __doc__, etc. We can just
2031                 # set the lineno to 0 for those.
2032                 lineno = '0'
2033             else:
2034                 lineno = str(entry.pos[1])
2035
2036             attrs = dict(
2037                 name=entry.name,
2038                 cname=cname,
2039                 qualified_name=qname,
2040                 type=vartype,
2041                 lineno=lineno)
2042
2043             self.tb.start('LocalVar', attrs)
2044             self.tb.end('LocalVar')
2045