Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67 class NormalizeTree(CythonTransform):
  68     """
  69     This transform fixes up a few things after parsing
  70     in order to make the parse tree more suitable for
  71     transforms.
  72
  73     a) After parsing, blocks with only one statement will
  74     be represented by that statement, not by a StatListNode.
  75     When doing transforms this is annoying and inconsistent,
  76     as one cannot in general remove a statement in a consistent
  77     way and so on. This transform wraps any single statements
  78     in a StatListNode containing a single statement.
  79
  80     b) The PassStatNode is a noop and serves no purpose beyond
  81     plugging such one-statement blocks; i.e., once parsed a
  82 `    "pass" can just as well be represented using an empty
  83     StatListNode. This means less special cases to worry about
  84     in subsequent transforms (one always checks to see if a
  85     StatListNode has no children to see if the block is empty).
  86     """
  87
  88     def __init__(self, context):
  89         super(NormalizeTree, self).__init__(context)
  90         self.is_in_statlist = False
  91         self.is_in_expr = False
  92
  93     def visit_ExprNode(self, node):
  94         stacktmp = self.is_in_expr
  95         self.is_in_expr = True
  96         self.visitchildren(node)
  97         self.is_in_expr = stacktmp
  98         return node
  99
 100     def visit_StatNode(self, node, is_listcontainer=False):
 101         stacktmp = self.is_in_statlist
 102         self.is_in_statlist = is_listcontainer
 103         self.visitchildren(node)
 104         self.is_in_statlist = stacktmp
 105         if not self.is_in_statlist and not self.is_in_expr:
 106             return Nodes.StatListNode(pos=node.pos, stats=[node])
 107         else:
 108             return node
 109
 110     def visit_StatListNode(self, node):
 111         self.is_in_statlist = True
 112         self.visitchildren(node)
 113         self.is_in_statlist = False
 114         return node
 115
 116     def visit_ParallelAssignmentNode(self, node):
 117         return self.visit_StatNode(node, True)
 118
 119     def visit_CEnumDefNode(self, node):
 120         return self.visit_StatNode(node, True)
 121
 122     def visit_CStructOrUnionDefNode(self, node):
 123         return self.visit_StatNode(node, True)
 124
 125     # Eliminate PassStatNode
 126     def visit_PassStatNode(self, node):
 127         if not self.is_in_statlist:
 128             return Nodes.StatListNode(pos=node.pos, stats=[])
 129         else:
 130             return []
 131
 132     def visit_CDeclaratorNode(self, node):
 133         return node
 134
 135
 136 class PostParseError(CompileError): pass
 137
 138 # error strings checked by unit tests, so define them
 139 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 140 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 141 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 142 class PostParse(ScopeTrackingTransform):
 143     """
 144     Basic interpretation of the parse tree, as well as validity
 145     checking that can be done on a very basic level on the parse
 146     tree (while still not being a problem with the basic syntax,
 147     as such).
 148
 149     Specifically:
 150     - Default values to cdef assignments are turned into single
 151     assignments following the declaration (everywhere but in class
 152     bodies, where they raise a compile error)
 153
 154     - Interpret some node structures into Python runtime values.
 155     Some nodes take compile-time arguments (currently:
 156     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 157     which should be interpreted. This happens in a general way
 158     and other steps should be taken to ensure validity.
 159
 160     Type arguments cannot be interpreted in this way.
 161
 162     - For __cythonbufferdefaults__ the arguments are checked for
 163     validity.
 164
 165     TemplatedTypeNode has its directives interpreted:
 166     Any first positional argument goes into the "dtype" attribute,
 167     any "ndim" keyword argument goes into the "ndim" attribute and
 168     so on. Also it is checked that the directive combination is valid.
 169     - __cythonbufferdefaults__ attributes are parsed and put into the
 170     type information.
 171
 172     Note: Currently Parsing.py does a lot of interpretation and
 173     reorganization that can be refactored into this transform
 174     if a more pure Abstract Syntax Tree is wanted.
 175     """
 176
 177     def __init__(self, context):
 178         super(PostParse, self).__init__(context)
 179         self.specialattribute_handlers = {
 180             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 181         }
 182
 183     def visit_ModuleNode(self, node):
 184         self.lambda_counter = 1
 185         self.genexpr_counter = 1
 186         return super(PostParse, self).visit_ModuleNode(node)
 187
 188     def visit_LambdaNode(self, node):
 189         # unpack a lambda expression into the corresponding DefNode
 190         lambda_id = self.lambda_counter
 191         self.lambda_counter += 1
 192         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 193         collector = YieldNodeCollector()
 194         collector.visitchildren(node.result_expr)
 195         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 196             body = ExprNodes.YieldExprNode(
 197                 node.result_expr.pos, arg=node.result_expr)
 198             body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
 199         else:
 200             body = Nodes.ReturnStatNode(
 201                 node.result_expr.pos, value=node.result_expr)
 202         node.def_node = Nodes.DefNode(
 203             node.pos, name=node.name, lambda_name=node.lambda_name,
 204             args=node.args, star_arg=node.star_arg,
 205             starstar_arg=node.starstar_arg,
 206             body=body, doc=None)
 207         self.visitchildren(node)
 208         return node
 209
 210     def visit_GeneratorExpressionNode(self, node):
 211         # unpack a generator expression into the corresponding DefNode
 212         genexpr_id = self.genexpr_counter
 213         self.genexpr_counter += 1
 214         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 215
 216         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 217                                       doc=None,
 218                                       args=[], star_arg=None,
 219                                       starstar_arg=None,
 220                                       body=node.loop)
 221         self.visitchildren(node)
 222         return node
 223
 224     # cdef variables
 225     def handle_bufferdefaults(self, decl):
 226         if not isinstance(decl.default, ExprNodes.DictNode):
 227             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 228         self.scope_node.buffer_defaults_node = decl.default
 229         self.scope_node.buffer_defaults_pos = decl.pos
 230
 231     def visit_CVarDefNode(self, node):
 232         # This assumes only plain names and pointers are assignable on
 233         # declaration. Also, it makes use of the fact that a cdef decl
 234         # must appear before the first use, so we don't have to deal with
 235         # "i = 3; cdef int i = i" and can simply move the nodes around.
 236         try:
 237             self.visitchildren(node)
 238             stats = [node]
 239             newdecls = []
 240             for decl in node.declarators:
 241                 declbase = decl
 242                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 243                     declbase = declbase.base
 244                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 245                     if declbase.default is not None:
 246                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 247                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 248                                 handler = self.specialattribute_handlers.get(decl.name)
 249                                 if handler:
 250                                     if decl is not declbase:
 251                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 252                                     handler(decl)
 253                                     continue # Remove declaration
 254                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 255                         first_assignment = self.scope_type != 'module'
 256                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 257                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 258                             rhs=declbase.default, first=first_assignment))
 259                         declbase.default = None
 260                 newdecls.append(decl)
 261             node.declarators = newdecls
 262             return stats
 263         except PostParseError, e:
 264             # An error in a cdef clause is ok, simply remove the declaration
 265             # and try to move on to report more errors
 266             self.context.nonfatal_error(e)
 267             return None
 268
 269     # Split parallel assignments (a,b = b,a) into separate partial
 270     # assignments that are executed rhs-first using temps.  This
 271     # restructuring must be applied before type analysis so that known
 272     # types on rhs and lhs can be matched directly.  It is required in
 273     # the case that the types cannot be coerced to a Python type in
 274     # order to assign from a tuple.
 275
 276     def visit_SingleAssignmentNode(self, node):
 277         self.visitchildren(node)
 278         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 279
 280     def visit_CascadedAssignmentNode(self, node):
 281         self.visitchildren(node)
 282         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 283
 284     def _visit_assignment_node(self, node, expr_list):
 285         """Flatten parallel assignments into separate single
 286         assignments or cascaded assignments.
 287         """
 288         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 289             # no parallel assignments => nothing to do
 290             return node
 291
 292         expr_list_list = []
 293         flatten_parallel_assignments(expr_list, expr_list_list)
 294         temp_refs = []
 295         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 296
 297         nodes = []
 298         for expr_list in expr_list_list:
 299             lhs_list = expr_list[:-1]
 300             rhs = expr_list[-1]
 301             if len(lhs_list) == 1:
 302                 node = Nodes.SingleAssignmentNode(rhs.pos,
 303                     lhs = lhs_list[0], rhs = rhs)
 304             else:
 305                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 306                     lhs_list = lhs_list, rhs = rhs)
 307             nodes.append(node)
 308
 309         if len(nodes) == 1:
 310             assign_node = nodes[0]
 311         else:
 312             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 313
 314         if temp_refs:
 315             duplicates_and_temps = [ (temp.expression, temp)
 316                                      for temp in temp_refs ]
 317             sort_common_subsequences(duplicates_and_temps)
 318             for _, temp_ref in duplicates_and_temps[::-1]:
 319                 assign_node = LetNode(temp_ref, assign_node)
 320
 321         return assign_node
 322
 323     def _flatten_sequence(self, seq, result):
 324         for arg in seq.args:
 325             if arg.is_sequence_constructor:
 326                 self._flatten_sequence(arg, result)
 327             else:
 328                 result.append(arg)
 329         return result
 330
 331     def visit_DelStatNode(self, node):
 332         self.visitchildren(node)
 333         node.args = self._flatten_sequence(node, [])
 334         return node
 335
 336
 337 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 338     """Replace rhs items by LetRefNodes if they appear more than once.
 339     Creates a sequence of LetRefNodes that set up the required temps
 340     and appends them to ref_node_sequence.  The input list is modified
 341     in-place.
 342     """
 343     seen_nodes = cython.set()
 344     ref_nodes = {}
 345     def find_duplicates(node):
 346         if node.is_literal or node.is_name:
 347             # no need to replace those; can't include attributes here
 348             # as their access is not necessarily side-effect free
 349             return
 350         if node in seen_nodes:
 351             if node not in ref_nodes:
 352                 ref_node = LetRefNode(node)
 353                 ref_nodes[node] = ref_node
 354                 ref_node_sequence.append(ref_node)
 355         else:
 356             seen_nodes.add(node)
 357             if node.is_sequence_constructor:
 358                 for item in node.args:
 359                     find_duplicates(item)
 360
 361     for expr_list in expr_list_list:
 362         rhs = expr_list[-1]
 363         find_duplicates(rhs)
 364     if not ref_nodes:
 365         return
 366
 367     def substitute_nodes(node):
 368         if node in ref_nodes:
 369             return ref_nodes[node]
 370         elif node.is_sequence_constructor:
 371             node.args = list(map(substitute_nodes, node.args))
 372         return node
 373
 374     # replace nodes inside of the common subexpressions
 375     for node in ref_nodes:
 376         if node.is_sequence_constructor:
 377             node.args = list(map(substitute_nodes, node.args))
 378
 379     # replace common subexpressions on all rhs items
 380     for expr_list in expr_list_list:
 381         expr_list[-1] = substitute_nodes(expr_list[-1])
 382
 383 def sort_common_subsequences(items):
 384     """Sort items/subsequences so that all items and subsequences that
 385     an item contains appear before the item itself.  This is needed
 386     because each rhs item must only be evaluated once, so its value
 387     must be evaluated first and then reused when packing sequences
 388     that contain it.
 389
 390     This implies a partial order, and the sort must be stable to
 391     preserve the original order as much as possible, so we use a
 392     simple insertion sort (which is very fast for short sequences, the
 393     normal case in practice).
 394     """
 395     def contains(seq, x):
 396         for item in seq:
 397             if item is x:
 398                 return True
 399             elif item.is_sequence_constructor and contains(item.args, x):
 400                 return True
 401         return False
 402     def lower_than(a,b):
 403         return b.is_sequence_constructor and contains(b.args, a)
 404
 405     for pos, item in enumerate(items):
 406         key = item[1] # the ResultRefNode which has already been injected into the sequences
 407         new_pos = pos
 408         for i in xrange(pos-1, -1, -1):
 409             if lower_than(key, items[i][0]):
 410                 new_pos = i
 411         if new_pos != pos:
 412             for i in xrange(pos, new_pos, -1):
 413                 items[i] = items[i-1]
 414             items[new_pos] = item
 415
 416 def flatten_parallel_assignments(input, output):
 417     #  The input is a list of expression nodes, representing the LHSs
 418     #  and RHS of one (possibly cascaded) assignment statement.  For
 419     #  sequence constructors, rearranges the matching parts of both
 420     #  sides into a list of equivalent assignments between the
 421     #  individual elements.  This transformation is applied
 422     #  recursively, so that nested structures get matched as well.
 423     rhs = input[-1]
 424     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 425         output.append(input)
 426         return
 427
 428     complete_assignments = []
 429
 430     rhs_size = len(rhs.args)
 431     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 432     starred_assignments = []
 433     for lhs in input[:-1]:
 434         if not lhs.is_sequence_constructor:
 435             if lhs.is_starred:
 436                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 437             complete_assignments.append(lhs)
 438             continue
 439         lhs_size = len(lhs.args)
 440         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 441         if starred_targets > 1:
 442             error(lhs.pos, "more than 1 starred expression in assignment")
 443             output.append([lhs,rhs])
 444             continue
 445         elif lhs_size - starred_targets > rhs_size:
 446             error(lhs.pos, "need more than %d value%s to unpack"
 447                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 448             output.append([lhs,rhs])
 449             continue
 450         elif starred_targets:
 451             map_starred_assignment(lhs_targets, starred_assignments,
 452                                    lhs.args, rhs.args)
 453         elif lhs_size < rhs_size:
 454             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 455                   % (lhs_size, rhs_size))
 456             output.append([lhs,rhs])
 457             continue
 458         else:
 459             for targets, expr in zip(lhs_targets, lhs.args):
 460                 targets.append(expr)
 461
 462     if complete_assignments:
 463         complete_assignments.append(rhs)
 464         output.append(complete_assignments)
 465
 466     # recursively flatten partial assignments
 467     for cascade, rhs in zip(lhs_targets, rhs.args):
 468         if cascade:
 469             cascade.append(rhs)
 470             flatten_parallel_assignments(cascade, output)
 471
 472     # recursively flatten starred assignments
 473     for cascade in starred_assignments:
 474         if cascade[0].is_sequence_constructor:
 475             flatten_parallel_assignments(cascade, output)
 476         else:
 477             output.append(cascade)
 478
 479 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 480     # Appends the fixed-position LHS targets to the target list that
 481     # appear left and right of the starred argument.
 482     #
 483     # The starred_assignments list receives a new tuple
 484     # (lhs_target, rhs_values_list) that maps the remaining arguments
 485     # (those that match the starred target) to a list.
 486
 487     # left side of the starred target
 488     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 489         if expr.is_starred:
 490             starred = i
 491             lhs_remaining = len(lhs_args) - i - 1
 492             break
 493         targets.append(expr)
 494     else:
 495         raise InternalError("no starred arg found when splitting starred assignment")
 496
 497     # right side of the starred target
 498     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 499                                             lhs_args[starred + 1:])):
 500         targets.append(expr)
 501
 502     # the starred target itself, must be assigned a (potentially empty) list
 503     target = lhs_args[starred].target # unpack starred node
 504     starred_rhs = rhs_args[starred:]
 505     if lhs_remaining:
 506         starred_rhs = starred_rhs[:-lhs_remaining]
 507     if starred_rhs:
 508         pos = starred_rhs[0].pos
 509     else:
 510         pos = target.pos
 511     starred_assignments.append([
 512         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 513
 514
 515 class PxdPostParse(CythonTransform, SkipDeclarations):
 516     """
 517     Basic interpretation/validity checking that should only be
 518     done on pxd trees.
 519
 520     A lot of this checking currently happens in the parser; but
 521     what is listed below happens here.
 522
 523     - "def" functions are let through only if they fill the
 524     getbuffer/releasebuffer slots
 525
 526     - cdef functions are let through only if they are on the
 527     top level and are declared "inline"
 528     """
 529     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 530     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 531
 532     def __call__(self, node):
 533         self.scope_type = 'pxd'
 534         return super(PxdPostParse, self).__call__(node)
 535
 536     def visit_CClassDefNode(self, node):
 537         old = self.scope_type
 538         self.scope_type = 'cclass'
 539         self.visitchildren(node)
 540         self.scope_type = old
 541         return node
 542
 543     def visit_FuncDefNode(self, node):
 544         # FuncDefNode always come with an implementation (without
 545         # an imp they are CVarDefNodes..)
 546         err = self.ERR_INLINE_ONLY
 547
 548         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 549             and node.name in ('__getbuffer__', '__releasebuffer__')):
 550             err = None # allow these slots
 551
 552         if isinstance(node, Nodes.CFuncDefNode):
 553             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 554                 node.inline_in_pxd = True
 555                 if node.visibility != 'private':
 556                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 557                 elif node.api:
 558                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 559                 else:
 560                     err = None # allow inline function
 561             else:
 562                 err = self.ERR_INLINE_ONLY
 563
 564         if err:
 565             self.context.nonfatal_error(PostParseError(node.pos, err))
 566             return None
 567         else:
 568             return node
 569
 570 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 571     """
 572     After parsing, directives can be stored in a number of places:
 573     - #cython-comments at the top of the file (stored in ModuleNode)
 574     - Command-line arguments overriding these
 575     - @cython.directivename decorators
 576     - with cython.directivename: statements
 577
 578     This transform is responsible for interpreting these various sources
 579     and store the directive in two ways:
 580     - Set the directives attribute of the ModuleNode for global directives.
 581     - Use a CompilerDirectivesNode to override directives for a subtree.
 582
 583     (The first one is primarily to not have to modify with the tree
 584     structure, so that ModuleNode stay on top.)
 585
 586     The directives are stored in dictionaries from name to value in effect.
 587     Each such dictionary is always filled in for all possible directives,
 588     using default values where no value is given by the user.
 589
 590     The available directives are controlled in Options.py.
 591
 592     Note that we have to run this prior to analysis, and so some minor
 593     duplication of functionality has to occur: We manually track cimports
 594     and which names the "cython" module may have been imported to.
 595     """
 596     unop_method_nodes = {
 597         'typeof': ExprNodes.TypeofNode,
 598
 599         'operator.address': ExprNodes.AmpersandNode,
 600         'operator.dereference': ExprNodes.DereferenceNode,
 601         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 602         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 603         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 604         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 605
 606         # For backwards compatability.
 607         'address': ExprNodes.AmpersandNode,
 608     }
 609
 610     binop_method_nodes = {
 611         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 612     }
 613
 614     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 615                                   'cast', 'pointer', 'compiled', 'NULL'])
 616     special_methods.update(unop_method_nodes.keys())
 617
 618     def __init__(self, context, compilation_directive_defaults):
 619         super(InterpretCompilerDirectives, self).__init__(context)
 620         self.compilation_directive_defaults = {}
 621         for key, value in compilation_directive_defaults.items():
 622             self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
 623         self.cython_module_names = cython.set()
 624         self.directive_names = {}
 625
 626     def check_directive_scope(self, pos, directive, scope):
 627         legal_scopes = Options.directive_scopes.get(directive, None)
 628         if legal_scopes and scope not in legal_scopes:
 629             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 630                                         'is not allowed in %s scope' % (directive, scope)))
 631             return False
 632         else:
 633             return True
 634
 635     # Set up processing and handle the cython: comments.
 636     def visit_ModuleNode(self, node):
 637         for key, value in node.directive_comments.items():
 638             if not self.check_directive_scope(node.pos, key, 'module'):
 639                 self.wrong_scope_error(node.pos, key, 'module')
 640                 del node.directive_comments[key]
 641
 642         directives = copy.deepcopy(Options.directive_defaults)
 643         directives.update(copy.deepcopy(self.compilation_directive_defaults))
 644         directives.update(node.directive_comments)
 645         self.directives = directives
 646         node.directives = directives
 647         self.visitchildren(node)
 648         node.cython_module_names = self.cython_module_names
 649         return node
 650
 651     # The following four functions track imports and cimports that
 652     # begin with "cython"
 653     def is_cython_directive(self, name):
 654         return (name in Options.directive_types or
 655                 name in self.special_methods or
 656                 PyrexTypes.parse_basic_type(name))
 657
 658     def visit_CImportStatNode(self, node):
 659         if node.module_name == u"cython":
 660             self.cython_module_names.add(node.as_name or u"cython")
 661         elif node.module_name.startswith(u"cython."):
 662             if node.as_name:
 663                 self.directive_names[node.as_name] = node.module_name[7:]
 664             else:
 665                 self.cython_module_names.add(u"cython")
 666             # if this cimport was a compiler directive, we don't
 667             # want to leave the cimport node sitting in the tree
 668             return None
 669         return node
 670
 671     def visit_FromCImportStatNode(self, node):
 672         if (node.module_name == u"cython") or \
 673                node.module_name.startswith(u"cython."):
 674             submodule = (node.module_name + u".")[7:]
 675             newimp = []
 676             for pos, name, as_name, kind in node.imported_names:
 677                 full_name = submodule + name
 678                 if self.is_cython_directive(full_name):
 679                     if as_name is None:
 680                         as_name = full_name
 681                     self.directive_names[as_name] = full_name
 682                     if kind is not None:
 683                         self.context.nonfatal_error(PostParseError(pos,
 684                             "Compiler directive imports must be plain imports"))
 685                 else:
 686                     newimp.append((pos, name, as_name, kind))
 687             if not newimp:
 688                 return None
 689             node.imported_names = newimp
 690         return node
 691
 692     def visit_FromImportStatNode(self, node):
 693         if (node.module.module_name.value == u"cython") or \
 694                node.module.module_name.value.startswith(u"cython."):
 695             submodule = (node.module.module_name.value + u".")[7:]
 696             newimp = []
 697             for name, name_node in node.items:
 698                 full_name = submodule + name
 699                 if self.is_cython_directive(full_name):
 700                     self.directive_names[name_node.name] = full_name
 701                 else:
 702                     newimp.append((name, name_node))
 703             if not newimp:
 704                 return None
 705             node.items = newimp
 706         return node
 707
 708     def visit_SingleAssignmentNode(self, node):
 709         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 710                 node.rhs.module_name.value == u'cython'):
 711             node = Nodes.CImportStatNode(node.pos,
 712                                          module_name = u'cython',
 713                                          as_name = node.lhs.name)
 714             self.visit_CImportStatNode(node)
 715         else:
 716             self.visitchildren(node)
 717         return node
 718
 719     def visit_NameNode(self, node):
 720         if node.name in self.cython_module_names:
 721             node.is_cython_module = True
 722         else:
 723             node.cython_attribute = self.directive_names.get(node.name)
 724         return node
 725
 726     def try_to_parse_directives(self, node):
 727         # If node is the contents of an directive (in a with statement or
 728         # decorator), returns a list of (directivename, value) pairs.
 729         # Otherwise, returns None
 730         if isinstance(node, ExprNodes.CallNode):
 731             self.visit(node.function)
 732             optname = node.function.as_cython_attribute()
 733             if optname:
 734                 directivetype = Options.directive_types.get(optname)
 735                 if directivetype:
 736                     args, kwds = node.explicit_args_kwds()
 737                     directives = []
 738                     key_value_pairs = []
 739                     if kwds is not None and directivetype is not dict:
 740                         for keyvalue in kwds.key_value_pairs:
 741                             key, value = keyvalue
 742                             sub_optname = "%s.%s" % (optname, key.value)
 743                             if Options.directive_types.get(sub_optname):
 744                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 745                             else:
 746                                 key_value_pairs.append(keyvalue)
 747                         if not key_value_pairs:
 748                             kwds = None
 749                         else:
 750                             kwds.key_value_pairs = key_value_pairs
 751                         if directives and not kwds and not args:
 752                             return directives
 753                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 754                     return directives
 755         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 756             self.visit(node)
 757             optname = node.as_cython_attribute()
 758             if optname:
 759                 directivetype = Options.directive_types.get(optname)
 760                 if directivetype is bool:
 761                     return [(optname, True)]
 762                 elif directivetype is None:
 763                     return [(optname, None)]
 764                 else:
 765                     raise PostParseError(
 766                         node.pos, "The '%s' directive should be used as a function call." % optname)
 767         return None
 768
 769     def try_to_parse_directive(self, optname, args, kwds, pos):
 770         directivetype = Options.directive_types.get(optname)
 771         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 772             return optname, Options.directive_defaults[optname]
 773         elif directivetype is bool:
 774             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 775                 raise PostParseError(pos,
 776                     'The %s directive takes one compile-time boolean argument' % optname)
 777             return (optname, args[0].value)
 778         elif directivetype is str:
 779             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 780                                                                               ExprNodes.UnicodeNode)):
 781                 raise PostParseError(pos,
 782                     'The %s directive takes one compile-time string argument' % optname)
 783             return (optname, str(args[0].value))
 784         elif directivetype is dict:
 785             if len(args) != 0:
 786                 raise PostParseError(pos,
 787                     'The %s directive takes no prepositional arguments' % optname)
 788             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 789         elif directivetype is list:
 790             if kwds and len(kwds) != 0:
 791                 raise PostParseError(pos,
 792                     'The %s directive takes no keyword arguments' % optname)
 793             return optname, [ str(arg.value) for arg in args ]
 794         else:
 795             assert False
 796
 797     def visit_with_directives(self, body, directives):
 798         olddirectives = self.directives
 799         newdirectives = copy.copy(olddirectives)
 800         newdirectives.update(directives)
 801         self.directives = newdirectives
 802         assert isinstance(body, Nodes.StatListNode), body
 803         retbody = self.visit_Node(body)
 804         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 805                                                  directives=newdirectives)
 806         self.directives = olddirectives
 807         return directive
 808
 809     # Handle decorators
 810     def visit_FuncDefNode(self, node):
 811         directives = self._extract_directives(node, 'function')
 812         if not directives:
 813             return self.visit_Node(node)
 814         body = Nodes.StatListNode(node.pos, stats=[node])
 815         return self.visit_with_directives(body, directives)
 816
 817     def visit_CVarDefNode(self, node):
 818         if not node.decorators:
 819             return node
 820         for dec in node.decorators:
 821             for directive in self.try_to_parse_directives(dec.decorator) or ():
 822                 if directive is not None and directive[0] == u'locals':
 823                     node.directive_locals = directive[1]
 824                 else:
 825                     self.context.nonfatal_error(PostParseError(dec.pos,
 826                         "Cdef functions can only take cython.locals() decorator."))
 827         return node
 828
 829     def visit_CClassDefNode(self, node):
 830         directives = self._extract_directives(node, 'cclass')
 831         if not directives:
 832             return self.visit_Node(node)
 833         body = Nodes.StatListNode(node.pos, stats=[node])
 834         return self.visit_with_directives(body, directives)
 835
 836     def visit_PyClassDefNode(self, node):
 837         directives = self._extract_directives(node, 'class')
 838         if not directives:
 839             return self.visit_Node(node)
 840         body = Nodes.StatListNode(node.pos, stats=[node])
 841         return self.visit_with_directives(body, directives)
 842
 843     def _extract_directives(self, node, scope_name):
 844         if not node.decorators:
 845             return {}
 846         # Split the decorators into two lists -- real decorators and directives
 847         directives = []
 848         realdecs = []
 849         for dec in node.decorators:
 850             new_directives = self.try_to_parse_directives(dec.decorator)
 851             if new_directives is not None:
 852                 for directive in new_directives:
 853                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 854                         directives.append(directive)
 855             else:
 856                 realdecs.append(dec)
 857         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 858             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 859         else:
 860             node.decorators = realdecs
 861         # merge or override repeated directives
 862         optdict = {}
 863         directives.reverse() # Decorators coming first take precedence
 864         for directive in directives:
 865             name, value = directive
 866             if name in optdict:
 867                 old_value = optdict[name]
 868                 # keywords and arg lists can be merged, everything
 869                 # else overrides completely
 870                 if isinstance(old_value, dict):
 871                     old_value.update(value)
 872                 elif isinstance(old_value, list):
 873                     old_value.extend(value)
 874                 else:
 875                     optdict[name] = value
 876             else:
 877                 optdict[name] = value
 878         return optdict
 879
 880     # Handle with statements
 881     def visit_WithStatNode(self, node):
 882         directive_dict = {}
 883         for directive in self.try_to_parse_directives(node.manager) or []:
 884             if directive is not None:
 885                 if node.target is not None:
 886                     self.context.nonfatal_error(
 887                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 888                 else:
 889                     name, value = directive
 890                     if name == 'nogil':
 891                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 892                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 893                         return self.visit_Node(node)
 894                     if self.check_directive_scope(node.pos, name, 'with statement'):
 895                         directive_dict[name] = value
 896         if directive_dict:
 897             return self.visit_with_directives(node.body, directive_dict)
 898         return self.visit_Node(node)
 899
 900 class WithTransform(CythonTransform, SkipDeclarations):
 901
 902     # EXCINFO is manually set to a variable that contains
 903     # the exc_info() tuple that can be generated by the enclosing except
 904     # statement.
 905     template_without_target = TreeFragment(u"""
 906         MGR = EXPR
 907         EXIT = MGR.__exit__
 908         MGR.__enter__()
 909         EXC = True
 910         try:
 911             try:
 912                 EXCINFO = None
 913                 BODY
 914             except:
 915                 EXC = False
 916                 if not EXIT(*EXCINFO):
 917                     raise
 918         finally:
 919             if EXC:
 920                 EXIT(None, None, None)
 921     """, temps=[u'MGR', u'EXC', u"EXIT"],
 922     pipeline=[NormalizeTree(None)])
 923
 924     template_with_target = TreeFragment(u"""
 925         MGR = EXPR
 926         EXIT = MGR.__exit__
 927         VALUE = MGR.__enter__()
 928         EXC = True
 929         try:
 930             try:
 931                 EXCINFO = None
 932                 TARGET = VALUE
 933                 BODY
 934             except:
 935                 EXC = False
 936                 if not EXIT(*EXCINFO):
 937                     raise
 938         finally:
 939             if EXC:
 940                 EXIT(None, None, None)
 941             MGR = EXIT = VALUE = EXC = None
 942
 943     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 944     pipeline=[NormalizeTree(None)])
 945
 946     def visit_WithStatNode(self, node):
 947         # TODO: Cleanup badly needed
 948         TemplateTransform.temp_name_counter += 1
 949         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 950
 951         self.visitchildren(node, ['body'])
 952         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 953         if node.target is not None:
 954             result = self.template_with_target.substitute({
 955                 u'EXPR' : node.manager,
 956                 u'BODY' : node.body,
 957                 u'TARGET' : node.target,
 958                 u'EXCINFO' : excinfo_temp
 959                 }, pos=node.pos)
 960         else:
 961             result = self.template_without_target.substitute({
 962                 u'EXPR' : node.manager,
 963                 u'BODY' : node.body,
 964                 u'EXCINFO' : excinfo_temp
 965                 }, pos=node.pos)
 966
 967         # Set except excinfo target to EXCINFO
 968         try_except = result.stats[-1].body.stats[-1]
 969         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 970 #            excinfo_temp.ref(node.pos))
 971
 972 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 973 #            node.pos, temps=[excinfo_temp], body=try_except)
 974
 975         return result
 976
 977     def visit_ExprNode(self, node):
 978         # With statements are never inside expressions.
 979         return node
 980
 981
 982 class DecoratorTransform(CythonTransform, SkipDeclarations):
 983
 984     def visit_DefNode(self, func_node):
 985         self.visitchildren(func_node)
 986         if not func_node.decorators:
 987             return func_node
 988         return self._handle_decorators(
 989             func_node, func_node.name)
 990
 991     def visit_CClassDefNode(self, class_node):
 992         # This doesn't currently work, so it's disabled.
 993         #
 994         # Problem: assignments to cdef class names do not work.  They
 995         # would require an additional check anyway, as the extension
 996         # type must not change its C type, so decorators cannot
 997         # replace an extension type, just alter it and return it.
 998
 999         self.visitchildren(class_node)
1000         if not class_node.decorators:
1001             return class_node
1002         error(class_node.pos,
1003               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
1004         return class_node
1005         #return self._handle_decorators(
1006         #    class_node, class_node.class_name)
1007
1008     def visit_ClassDefNode(self, class_node):
1009         self.visitchildren(class_node)
1010         if not class_node.decorators:
1011             return class_node
1012         return self._handle_decorators(
1013             class_node, class_node.name)
1014
1015     def _handle_decorators(self, node, name):
1016         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1017         for decorator in node.decorators[::-1]:
1018             decorator_result = ExprNodes.SimpleCallNode(
1019                 decorator.pos,
1020                 function = decorator.decorator,
1021                 args = [decorator_result])
1022
1023         name_node = ExprNodes.NameNode(node.pos, name = name)
1024         reassignment = Nodes.SingleAssignmentNode(
1025             node.pos,
1026             lhs = name_node,
1027             rhs = decorator_result)
1028         return [node, reassignment]
1029
1030
1031 class AnalyseDeclarationsTransform(CythonTransform):
1032
1033     basic_property = TreeFragment(u"""
1034 property NAME:
1035     def __get__(self):
1036         return ATTR
1037     def __set__(self, value):
1038         ATTR = value
1039     """, level='c_class')
1040     basic_pyobject_property = TreeFragment(u"""
1041 property NAME:
1042     def __get__(self):
1043         return ATTR
1044     def __set__(self, value):
1045         ATTR = value
1046     def __del__(self):
1047         ATTR = None
1048     """, level='c_class')
1049     basic_property_ro = TreeFragment(u"""
1050 property NAME:
1051     def __get__(self):
1052         return ATTR
1053     """, level='c_class')
1054
1055     struct_or_union_wrapper = TreeFragment(u"""
1056 cdef class NAME:
1057     cdef TYPE value
1058     def __init__(self, MEMBER=None):
1059         cdef int count
1060         count = 0
1061         INIT_ASSIGNMENTS
1062         if IS_UNION and count > 1:
1063             raise ValueError, "At most one union member should be specified."
1064     def __str__(self):
1065         return STR_FORMAT % MEMBER_TUPLE
1066     def __repr__(self):
1067         return REPR_FORMAT % MEMBER_TUPLE
1068     """)
1069
1070     init_assignment = TreeFragment(u"""
1071 if VALUE is not None:
1072     ATTR = VALUE
1073     count += 1
1074     """)
1075
1076     def __call__(self, root):
1077         self.env_stack = [root.scope]
1078         # needed to determine if a cdef var is declared after it's used.
1079         self.seen_vars_stack = []
1080         return super(AnalyseDeclarationsTransform, self).__call__(root)
1081
1082     def visit_NameNode(self, node):
1083         self.seen_vars_stack[-1].add(node.name)
1084         return node
1085
1086     def visit_ModuleNode(self, node):
1087         self.seen_vars_stack.append(cython.set())
1088         node.analyse_declarations(self.env_stack[-1])
1089         self.visitchildren(node)
1090         self.seen_vars_stack.pop()
1091         return node
1092
1093     def visit_LambdaNode(self, node):
1094         node.analyse_declarations(self.env_stack[-1])
1095         self.visitchildren(node)
1096         return node
1097
1098     def visit_ClassDefNode(self, node):
1099         self.env_stack.append(node.scope)
1100         self.visitchildren(node)
1101         self.env_stack.pop()
1102         return node
1103
1104     def visit_CClassDefNode(self, node):
1105         node = self.visit_ClassDefNode(node)
1106         if node.scope and node.scope.implemented:
1107             stats = []
1108             for entry in node.scope.var_entries:
1109                 if entry.needs_property:
1110                     property = self.create_Property(entry)
1111                     property.analyse_declarations(node.scope)
1112                     self.visit(property)
1113                     stats.append(property)
1114             if stats:
1115                 node.body.stats += stats
1116         return node
1117
1118     def visit_FuncDefNode(self, node):
1119         self.seen_vars_stack.append(cython.set())
1120         lenv = node.local_scope
1121         node.body.analyse_control_flow(lenv) # this will be totally refactored
1122         node.declare_arguments(lenv)
1123         for var, type_node in node.directive_locals.items():
1124             if not lenv.lookup_here(var):   # don't redeclare args
1125                 type = type_node.analyse_as_type(lenv)
1126                 if type:
1127                     lenv.declare_var(var, type, type_node.pos)
1128                 else:
1129                     error(type_node.pos, "Not a type")
1130         node.body.analyse_declarations(lenv)
1131         self.env_stack.append(lenv)
1132         self.visitchildren(node)
1133         self.env_stack.pop()
1134         self.seen_vars_stack.pop()
1135         return node
1136
1137     def visit_ScopedExprNode(self, node):
1138         env = self.env_stack[-1]
1139         node.analyse_declarations(env)
1140         # the node may or may not have a local scope
1141         if node.has_local_scope:
1142             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1143             self.env_stack.append(node.expr_scope)
1144             node.analyse_scoped_declarations(node.expr_scope)
1145             self.visitchildren(node)
1146             self.env_stack.pop()
1147             self.seen_vars_stack.pop()
1148         else:
1149             node.analyse_scoped_declarations(env)
1150             self.visitchildren(node)
1151         return node
1152
1153     def visit_TempResultFromStatNode(self, node):
1154         self.visitchildren(node)
1155         node.analyse_declarations(self.env_stack[-1])
1156         return node
1157
1158     def visit_CStructOrUnionDefNode(self, node):
1159         # Create a wrapper node if needed.
1160         # We want to use the struct type information (so it can't happen
1161         # before this phase) but also create new objects to be declared
1162         # (so it can't happen later).
1163         # Note that we don't return the original node, as it is
1164         # never used after this phase.
1165         if True: # private (default)
1166             return None
1167
1168         self_value = ExprNodes.AttributeNode(
1169             pos = node.pos,
1170             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1171             attribute = EncodedString(u"value"))
1172         var_entries = node.entry.type.scope.var_entries
1173         attributes = []
1174         for entry in var_entries:
1175             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1176                                                       obj = self_value,
1177                                                       attribute = entry.name))
1178         # __init__ assignments
1179         init_assignments = []
1180         for entry, attr in zip(var_entries, attributes):
1181             # TODO: branch on visibility
1182             init_assignments.append(self.init_assignment.substitute({
1183                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1184                     u"ATTR": attr,
1185                 }, pos = entry.pos))
1186
1187         # create the class
1188         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1189         wrapper_class = self.struct_or_union_wrapper.substitute({
1190             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1191             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1192             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1193             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1194             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1195         }, pos = node.pos).stats[0]
1196         wrapper_class.class_name = node.name
1197         wrapper_class.shadow = True
1198         class_body = wrapper_class.body.stats
1199
1200         # fix value type
1201         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1202         class_body[0].base_type.name = node.name
1203
1204         # fix __init__ arguments
1205         init_method = class_body[1]
1206         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1207         arg_template = init_method.args[1]
1208         if not node.entry.type.is_struct:
1209             arg_template.kw_only = True
1210         del init_method.args[1]
1211         for entry, attr in zip(var_entries, attributes):
1212             arg = copy.deepcopy(arg_template)
1213             arg.declarator.name = entry.name
1214             init_method.args.append(arg)
1215
1216         # setters/getters
1217         for entry, attr in zip(var_entries, attributes):
1218             # TODO: branch on visibility
1219             if entry.type.is_pyobject:
1220                 template = self.basic_pyobject_property
1221             else:
1222                 template = self.basic_property
1223             property = template.substitute({
1224                     u"ATTR": attr,
1225                 }, pos = entry.pos).stats[0]
1226             property.name = entry.name
1227             wrapper_class.body.stats.append(property)
1228
1229         wrapper_class.analyse_declarations(self.env_stack[-1])
1230         return self.visit_CClassDefNode(wrapper_class)
1231
1232     # Some nodes are no longer needed after declaration
1233     # analysis and can be dropped. The analysis was performed
1234     # on these nodes in a seperate recursive process from the
1235     # enclosing function or module, so we can simply drop them.
1236     def visit_CDeclaratorNode(self, node):
1237         # necessary to ensure that all CNameDeclaratorNodes are visited.
1238         self.visitchildren(node)
1239         return node
1240
1241     def visit_CTypeDefNode(self, node):
1242         return node
1243
1244     def visit_CBaseTypeNode(self, node):
1245         return None
1246
1247     def visit_CEnumDefNode(self, node):
1248         if node.visibility == 'public':
1249             return node
1250         else:
1251             return None
1252
1253     def visit_CNameDeclaratorNode(self, node):
1254         if node.name in self.seen_vars_stack[-1]:
1255             entry = self.env_stack[-1].lookup(node.name)
1256             if (entry is None or entry.visibility != 'extern'
1257                 and not entry.scope.is_c_class_scope):
1258                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1259         self.visitchildren(node)
1260         return node
1261
1262     def visit_CVarDefNode(self, node):
1263         # to ensure all CNameDeclaratorNodes are visited.
1264         self.visitchildren(node)
1265         return None
1266
1267     def create_Property(self, entry):
1268         if entry.visibility == 'public':
1269             if entry.type.is_pyobject:
1270                 template = self.basic_pyobject_property
1271             else:
1272                 template = self.basic_property
1273         elif entry.visibility == 'readonly':
1274             template = self.basic_property_ro
1275         property = template.substitute({
1276                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1277                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1278                                                  attribute=entry.name),
1279             }, pos=entry.pos).stats[0]
1280         property.name = entry.name
1281         # ---------------------------------------
1282         # XXX This should go to AutoDocTransforms
1283         # ---------------------------------------
1284         if (Options.docstrings and
1285             self.current_directives['embedsignature']):
1286             attr_name = entry.name
1287             type_name = entry.type.declaration_code("", for_display=1)
1288             default_value = ''
1289             if not entry.type.is_pyobject:
1290                 type_name = "'%s'" % type_name
1291             elif entry.type.is_extension_type:
1292                 type_name = entry.type.module_name + '.' + type_name
1293             if entry.init is not None:
1294                 default_value = ' = ' + entry.init
1295             elif entry.init_to_none:
1296                 default_value = ' = ' + repr(None)
1297             docstring = attr_name + ': ' + type_name + default_value
1298             property.doc = EncodedString(docstring)
1299         # ---------------------------------------
1300         return property
1301
1302 class AnalyseExpressionsTransform(CythonTransform):
1303
1304     def visit_ModuleNode(self, node):
1305         node.scope.infer_types()
1306         node.body.analyse_expressions(node.scope)
1307         self.visitchildren(node)
1308         return node
1309
1310     def visit_FuncDefNode(self, node):
1311         node.local_scope.infer_types()
1312         node.body.analyse_expressions(node.local_scope)
1313         self.visitchildren(node)
1314         return node
1315
1316     def visit_ScopedExprNode(self, node):
1317         if node.has_local_scope:
1318             node.expr_scope.infer_types()
1319             node.analyse_scoped_expressions(node.expr_scope)
1320         self.visitchildren(node)
1321         return node
1322
1323 class ExpandInplaceOperators(EnvTransform):
1324
1325     def visit_InPlaceAssignmentNode(self, node):
1326         lhs = node.lhs
1327         rhs = node.rhs
1328         if lhs.type.is_cpp_class:
1329             # No getting around this exact operator here.
1330             return node
1331         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1332             # There is code to handle this case.
1333             return node
1334
1335         env = self.current_env()
1336         def side_effect_free_reference(node, setting=False):
1337             if isinstance(node, ExprNodes.NameNode):
1338                 return node, []
1339             elif node.type.is_pyobject and not setting:
1340                 node = LetRefNode(node)
1341                 return node, [node]
1342             elif isinstance(node, ExprNodes.IndexNode):
1343                 if node.is_buffer_access:
1344                     raise ValueError, "Buffer access"
1345                 base, temps = side_effect_free_reference(node.base)
1346                 index = LetRefNode(node.index)
1347                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1348             elif isinstance(node, ExprNodes.AttributeNode):
1349                 obj, temps = side_effect_free_reference(node.obj)
1350                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1351             else:
1352                 node = LetRefNode(node)
1353                 return node, [node]
1354         try:
1355             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1356         except ValueError:
1357             return node
1358         dup = lhs.__class__(**lhs.__dict__)
1359         binop = ExprNodes.binop_node(node.pos,
1360                                      operator = node.operator,
1361                                      operand1 = dup,
1362                                      operand2 = rhs,
1363                                      inplace=True)
1364         # Manually analyse types for new node.
1365         lhs.analyse_target_types(env)
1366         dup.analyse_types(env)
1367         binop.analyse_operation(env)
1368         node = Nodes.SingleAssignmentNode(
1369             node.pos,
1370             lhs = lhs,
1371             rhs=binop.coerce_to(lhs.type, env))
1372         # Use LetRefNode to avoid side effects.
1373         let_ref_nodes.reverse()
1374         for t in let_ref_nodes:
1375             node = LetNode(t, node)
1376         return node
1377
1378     def visit_ExprNode(self, node):
1379         # In-place assignments can't happen within an expression.
1380         return node
1381
1382
1383 class AlignFunctionDefinitions(CythonTransform):
1384     """
1385     This class takes the signatures from a .pxd file and applies them to
1386     the def methods in a .py file.
1387     """
1388
1389     def visit_ModuleNode(self, node):
1390         self.scope = node.scope
1391         self.directives = node.directives
1392         self.visitchildren(node)
1393         return node
1394
1395     def visit_PyClassDefNode(self, node):
1396         pxd_def = self.scope.lookup(node.name)
1397         if pxd_def:
1398             if pxd_def.is_cclass:
1399                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1400             else:
1401                 error(node.pos, "'%s' redeclared" % node.name)
1402                 if pxd_def.pos:
1403                     error(pxd_def.pos, "previous declaration here")
1404                 return None
1405         else:
1406             return node
1407
1408     def visit_CClassDefNode(self, node, pxd_def=None):
1409         if pxd_def is None:
1410             pxd_def = self.scope.lookup(node.class_name)
1411         if pxd_def:
1412             outer_scope = self.scope
1413             self.scope = pxd_def.type.scope
1414         self.visitchildren(node)
1415         if pxd_def:
1416             self.scope = outer_scope
1417         return node
1418
1419     def visit_DefNode(self, node):
1420         pxd_def = self.scope.lookup(node.name)
1421         if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
1422             if not pxd_def.is_cfunction:
1423                 error(node.pos, "'%s' redeclared" % node.name)
1424                 if pxd_def.pos:
1425                     error(pxd_def.pos, "previous declaration here")
1426                 return None
1427             node = node.as_cfunction(pxd_def)
1428         elif (self.scope.is_module_scope and self.directives['auto_cpdef']
1429               and node.is_cdef_func_compatible()):
1430             node = node.as_cfunction(scope=self.scope)
1431         # Enable this when nested cdef functions are allowed.
1432         # self.visitchildren(node)
1433         return node
1434
1435
1436 class YieldNodeCollector(TreeVisitor):
1437
1438     def __init__(self):
1439         super(YieldNodeCollector, self).__init__()
1440         self.yields = []
1441         self.returns = []
1442         self.has_return_value = False
1443
1444     def visit_Node(self, node):
1445         return self.visitchildren(node)
1446
1447     def visit_YieldExprNode(self, node):
1448         if self.has_return_value:
1449             error(node.pos, "'yield' outside function")
1450         self.yields.append(node)
1451         self.visitchildren(node)
1452
1453     def visit_ReturnStatNode(self, node):
1454         if node.value:
1455             self.has_return_value = True
1456             if self.yields:
1457                 error(node.pos, "'return' with argument inside generator")
1458         self.returns.append(node)
1459
1460     def visit_ClassDefNode(self, node):
1461         pass
1462
1463     def visit_FuncDefNode(self, node):
1464         pass
1465
1466     def visit_LambdaNode(self, node):
1467         pass
1468
1469     def visit_GeneratorExpressionNode(self, node):
1470         pass
1471
1472 class MarkClosureVisitor(CythonTransform):
1473
1474     def visit_ModuleNode(self, node):
1475         self.needs_closure = False
1476         self.visitchildren(node)
1477         return node
1478
1479     def visit_FuncDefNode(self, node):
1480         self.needs_closure = False
1481         self.visitchildren(node)
1482         node.needs_closure = self.needs_closure
1483         self.needs_closure = True
1484
1485         collector = YieldNodeCollector()
1486         collector.visitchildren(node)
1487
1488         if collector.yields:
1489             for i, yield_expr in enumerate(collector.yields):
1490                 yield_expr.label_num = i + 1
1491
1492             gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1493                                                name=node.name,
1494                                                body=node.body)
1495             generator = Nodes.GeneratorDefNode(pos=node.pos,
1496                                                name=node.name,
1497                                                args=node.args,
1498                                                star_arg=node.star_arg,
1499                                                starstar_arg=node.starstar_arg,
1500                                                doc=node.doc,
1501                                                decorators=node.decorators,
1502                                                gbody=gbody,
1503                                                lambda_name=node.lambda_name)
1504             return generator
1505         return node
1506
1507     def visit_CFuncDefNode(self, node):
1508         self.visit_FuncDefNode(node)
1509         if node.needs_closure:
1510             error(node.pos, "closures inside cdef functions not yet supported")
1511         return node
1512
1513     def visit_LambdaNode(self, node):
1514         self.needs_closure = False
1515         self.visitchildren(node)
1516         node.needs_closure = self.needs_closure
1517         self.needs_closure = True
1518         return node
1519
1520     def visit_ClassDefNode(self, node):
1521         self.visitchildren(node)
1522         self.needs_closure = True
1523         return node
1524
1525 class CreateClosureClasses(CythonTransform):
1526     # Output closure classes in module scope for all functions
1527     # that really need it.
1528
1529     def __init__(self, context):
1530         super(CreateClosureClasses, self).__init__(context)
1531         self.path = []
1532         self.in_lambda = False
1533         self.generator_class = None
1534
1535     def visit_ModuleNode(self, node):
1536         self.module_scope = node.scope
1537         self.visitchildren(node)
1538         return node
1539
1540     def create_generator_class(self, target_module_scope, pos):
1541         if self.generator_class:
1542             return self.generator_class
1543         # XXX: make generator class creation cleaner
1544         entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1545                     objstruct_cname='__pyx_Generator_object',
1546                     typeobj_cname='__pyx_Generator_type',
1547                     pos=pos, defining=True, implementing=True)
1548         klass = entry.type.scope
1549         klass.is_internal = True
1550         klass.directives = {'final': True}
1551
1552         body_type = PyrexTypes.create_typedef_type('generator_body',
1553                                                    PyrexTypes.c_void_ptr_type,
1554                                                    '__pyx_generator_body_t')
1555         klass.declare_var(pos=pos, name='body', cname='body',
1556                           type=body_type, is_cdef=True)
1557         klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1558                           is_cdef=True)
1559         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1560                           is_cdef=True)
1561
1562         import TypeSlots
1563         e = klass.declare_pyfunction('send', pos)
1564         e.func_cname = '__Pyx_Generator_Send'
1565         e.signature = TypeSlots.binaryfunc
1566
1567         e = klass.declare_pyfunction('close', pos)
1568         e.func_cname = '__Pyx_Generator_Close'
1569         e.signature = TypeSlots.unaryfunc
1570
1571         e = klass.declare_pyfunction('throw', pos)
1572         e.func_cname = '__Pyx_Generator_Throw'
1573         e.signature = TypeSlots.pyfunction_signature
1574
1575         e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1576         e.func_cname = 'PyObject_SelfIter'
1577
1578         e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1579         e.func_cname = '__Pyx_Generator_Next'
1580
1581         self.generator_class = entry.type
1582         return self.generator_class
1583
1584     def find_entries_used_in_closures(self, node):
1585         from_closure = []
1586         in_closure = []
1587         for name, entry in node.local_scope.entries.items():
1588             if entry.from_closure:
1589                 from_closure.append((name, entry))
1590             elif entry.in_closure:
1591                 in_closure.append((name, entry))
1592         return from_closure, in_closure
1593
1594     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1595         # skip generator body
1596         if node.is_generator_body:
1597             return
1598         # move local variables into closure
1599         if node.is_generator:
1600             for entry in node.local_scope.entries.values():
1601                 if not entry.from_closure:
1602                     entry.in_closure = True
1603
1604         from_closure, in_closure = self.find_entries_used_in_closures(node)
1605         in_closure.sort()
1606
1607         # Now from the begining
1608         node.needs_closure = False
1609         node.needs_outer_scope = False
1610
1611         func_scope = node.local_scope
1612         cscope = node.entry.scope
1613         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1614             cscope = cscope.outer_scope
1615
1616         if not from_closure and (self.path or inner_node):
1617             if not inner_node:
1618                 if not node.assmt:
1619                     raise InternalError, "DefNode does not have assignment node"
1620                 inner_node = node.assmt.rhs
1621             inner_node.needs_self_code = False
1622             node.needs_outer_scope = False
1623
1624         base_type = None
1625         if node.is_generator:
1626             base_type = self.create_generator_class(target_module_scope, node.pos)
1627         elif not in_closure and not from_closure:
1628             return
1629         elif not in_closure:
1630             func_scope.is_passthrough = True
1631             func_scope.scope_class = cscope.scope_class
1632             node.needs_outer_scope = True
1633             return
1634
1635         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1636
1637         entry = target_module_scope.declare_c_class(
1638             name=as_name, pos=node.pos, defining=True,
1639             implementing=True, base_type=base_type)
1640
1641         func_scope.scope_class = entry
1642         class_scope = entry.type.scope
1643         class_scope.is_internal = True
1644         class_scope.directives = {'final': True}
1645
1646         if from_closure:
1647             assert cscope.is_closure_scope
1648             class_scope.declare_var(pos=node.pos,
1649                                     name=Naming.outer_scope_cname,
1650                                     cname=Naming.outer_scope_cname,
1651                                     type=cscope.scope_class.type,
1652                                     is_cdef=True)
1653             node.needs_outer_scope = True
1654         for name, entry in in_closure:
1655             closure_entry = class_scope.declare_var(pos=entry.pos,
1656                                     name=entry.name,
1657                                     cname=entry.cname,
1658                                     type=entry.type,
1659                                     is_cdef=True)
1660             if entry.is_declared_generic:
1661                 closure_entry.is_declared_generic = 1
1662         node.needs_closure = True
1663         # Do it here because other classes are already checked
1664         target_module_scope.check_c_class(func_scope.scope_class)
1665
1666     def visit_LambdaNode(self, node):
1667         was_in_lambda = self.in_lambda
1668         self.in_lambda = True
1669         self.create_class_from_scope(node.def_node, self.module_scope, node)
1670         self.visitchildren(node)
1671         self.in_lambda = was_in_lambda
1672         return node
1673
1674     def visit_FuncDefNode(self, node):
1675         if self.in_lambda:
1676             self.visitchildren(node)
1677             return node
1678         if node.needs_closure or self.path:
1679             self.create_class_from_scope(node, self.module_scope)
1680             self.path.append(node)
1681             self.visitchildren(node)
1682             self.path.pop()
1683         return node
1684
1685
1686 class GilCheck(VisitorTransform):
1687     """
1688     Call `node.gil_check(env)` on each node to make sure we hold the
1689     GIL when we need it.  Raise an error when on Python operations
1690     inside a `nogil` environment.
1691     """
1692     def __call__(self, root):
1693         self.env_stack = [root.scope]
1694         self.nogil = False
1695         return super(GilCheck, self).__call__(root)
1696
1697     def visit_FuncDefNode(self, node):
1698         self.env_stack.append(node.local_scope)
1699         was_nogil = self.nogil
1700         self.nogil = node.local_scope.nogil
1701         if self.nogil and node.nogil_check:
1702             node.nogil_check(node.local_scope)
1703         self.visitchildren(node)
1704         self.env_stack.pop()
1705         self.nogil = was_nogil
1706         return node
1707
1708     def visit_GILStatNode(self, node):
1709         env = self.env_stack[-1]
1710         if self.nogil and node.nogil_check: node.nogil_check()
1711         was_nogil = self.nogil
1712         self.nogil = (node.state == 'nogil')
1713         self.visitchildren(node)
1714         self.nogil = was_nogil
1715         return node
1716
1717     def visit_Node(self, node):
1718         if self.env_stack and self.nogil and node.nogil_check:
1719             node.nogil_check(self.env_stack[-1])
1720         self.visitchildren(node)
1721         return node
1722
1723
1724 class TransformBuiltinMethods(EnvTransform):
1725
1726     def visit_SingleAssignmentNode(self, node):
1727         if node.declaration_only:
1728             return None
1729         else:
1730             self.visitchildren(node)
1731             return node
1732
1733     def visit_AttributeNode(self, node):
1734         self.visitchildren(node)
1735         return self.visit_cython_attribute(node)
1736
1737     def visit_NameNode(self, node):
1738         return self.visit_cython_attribute(node)
1739
1740     def visit_cython_attribute(self, node):
1741         attribute = node.as_cython_attribute()
1742         if attribute:
1743             if attribute == u'compiled':
1744                 node = ExprNodes.BoolNode(node.pos, value=True)
1745             elif attribute == u'NULL':
1746                 node = ExprNodes.NullNode(node.pos)
1747             elif attribute in (u'set', u'frozenset'):
1748                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1749                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1750             elif not PyrexTypes.parse_basic_type(attribute):
1751                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1752         return node
1753
1754     def _inject_locals(self, node, func_name):
1755         # locals()/dir() builtins
1756         lenv = self.current_env()
1757         entry = lenv.lookup_here(func_name)
1758         if entry:
1759             # not the builtin
1760             return node
1761         pos = node.pos
1762         if func_name == 'locals':
1763             if len(node.args) > 0:
1764                 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
1765                       % len(node.args))
1766                 return node
1767             items = [ ExprNodes.DictItemNode(pos,
1768                                              key=ExprNodes.StringNode(pos, value=var),
1769                                              value=ExprNodes.NameNode(pos, name=var))
1770                       for var in lenv.entries ]
1771             return ExprNodes.DictNode(pos, key_value_pairs=items)
1772         else:
1773             if len(node.args) > 1:
1774                 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
1775                       % len(node.args))
1776                 return node
1777             elif len(node.args) == 1:
1778                 # optimised in Builtin.py
1779                 return node
1780             items = [ ExprNodes.StringNode(pos, value=var) for var in lenv.entries ]
1781             return ExprNodes.ListNode(pos, args=items)
1782
1783     def visit_SimpleCallNode(self, node):
1784         if isinstance(node.function, ExprNodes.NameNode):
1785             func_name = node.function.name
1786             if func_name in ('dir', 'locals'):
1787                 return self._inject_locals(node, func_name)
1788
1789         # cython.foo
1790         function = node.function.as_cython_attribute()
1791         if function:
1792             if function in InterpretCompilerDirectives.unop_method_nodes:
1793                 if len(node.args) != 1:
1794                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1795                 else:
1796                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1797             elif function in InterpretCompilerDirectives.binop_method_nodes:
1798                 if len(node.args) != 2:
1799                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1800                 else:
1801                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1802             elif function == u'cast':
1803                 if len(node.args) != 2:
1804                     error(node.function.pos, u"cast() takes exactly two arguments")
1805                 else:
1806                     type = node.args[0].analyse_as_type(self.current_env())
1807                     if type:
1808                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1809                     else:
1810                         error(node.args[0].pos, "Not a type")
1811             elif function == u'sizeof':
1812                 if len(node.args) != 1:
1813                     error(node.function.pos, u"sizeof() takes exactly one argument")
1814                 else:
1815                     type = node.args[0].analyse_as_type(self.current_env())
1816                     if type:
1817                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1818                     else:
1819                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1820             elif function == 'cmod':
1821                 if len(node.args) != 2:
1822                     error(node.function.pos, u"cmod() takes exactly two arguments")
1823                 else:
1824                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1825                     node.cdivision = True
1826             elif function == 'cdiv':
1827                 if len(node.args) != 2:
1828                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1829                 else:
1830                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1831                     node.cdivision = True
1832             elif function == u'set':
1833                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1834             else:
1835                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1836
1837         self.visitchildren(node)
1838         return node
1839
1840
1841 class DebugTransform(CythonTransform):
1842     """
1843     Create debug information and all functions' visibility to extern in order
1844     to enable debugging.
1845     """
1846
1847     def __init__(self, context, options, result):
1848         super(DebugTransform, self).__init__(context)
1849         self.visited = cython.set()
1850         # our treebuilder and debug output writer
1851         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1852         self.tb = self.context.gdb_debug_outputwriter
1853         #self.c_output_file = options.output_file
1854         self.c_output_file = result.c_file
1855
1856         # Closure support, basically treat nested functions as if the AST were
1857         # never nested
1858         self.nested_funcdefs = []
1859
1860         # tells visit_NameNode whether it should register step-into functions
1861         self.register_stepinto = False
1862
1863     def visit_ModuleNode(self, node):
1864         self.tb.module_name = node.full_module_name
1865         attrs = dict(
1866             module_name=node.full_module_name,
1867             filename=node.pos[0].filename,
1868             c_filename=self.c_output_file)
1869
1870         self.tb.start('Module', attrs)
1871
1872         # serialize functions
1873         self.tb.start('Functions')
1874         # First, serialize functions normally...
1875         self.visitchildren(node)
1876
1877         # ... then, serialize nested functions
1878         for nested_funcdef in self.nested_funcdefs:
1879             self.visit_FuncDefNode(nested_funcdef)
1880
1881         self.register_stepinto = True
1882         self.serialize_modulenode_as_function(node)
1883         self.register_stepinto = False
1884         self.tb.end('Functions')
1885
1886         # 2.3 compatibility. Serialize global variables
1887         self.tb.start('Globals')
1888         entries = {}
1889
1890         for k, v in node.scope.entries.iteritems():
1891             if (v.qualified_name not in self.visited and not
1892                 v.name.startswith('__pyx_') and not
1893                 v.type.is_cfunction and not
1894                 v.type.is_extension_type):
1895                 entries[k]= v
1896
1897         self.serialize_local_variables(entries)
1898         self.tb.end('Globals')
1899         # self.tb.end('Module') # end Module after the line number mapping in
1900         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1901         return node
1902
1903     def visit_FuncDefNode(self, node):
1904         self.visited.add(node.local_scope.qualified_name)
1905
1906         if getattr(node, 'is_wrapper', False):
1907             return node
1908
1909         if self.register_stepinto:
1910             self.nested_funcdefs.append(node)
1911             return node
1912
1913         # node.entry.visibility = 'extern'
1914         if node.py_func is None:
1915             pf_cname = ''
1916         else:
1917             pf_cname = node.py_func.entry.func_cname
1918
1919         attrs = dict(
1920             name=node.entry.name,
1921             cname=node.entry.func_cname,
1922             pf_cname=pf_cname,
1923             qualified_name=node.local_scope.qualified_name,
1924             lineno=str(node.pos[1]))
1925
1926         self.tb.start('Function', attrs=attrs)
1927
1928         self.tb.start('Locals')
1929         self.serialize_local_variables(node.local_scope.entries)
1930         self.tb.end('Locals')
1931
1932         self.tb.start('Arguments')
1933         for arg in node.local_scope.arg_entries:
1934             self.tb.start(arg.name)
1935             self.tb.end(arg.name)
1936         self.tb.end('Arguments')
1937
1938         self.tb.start('StepIntoFunctions')
1939         self.register_stepinto = True
1940         self.visitchildren(node)
1941         self.register_stepinto = False
1942         self.tb.end('StepIntoFunctions')
1943         self.tb.end('Function')
1944
1945         return node
1946
1947     def visit_NameNode(self, node):
1948         if (self.register_stepinto and
1949             node.type.is_cfunction and
1950             getattr(node, 'is_called', False) and
1951             node.entry.func_cname is not None):
1952             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1953             # declared functions are not 'in_cinclude'.
1954             # This means we will list called 'cdef' functions as
1955             # "step into functions", but this is not an issue as they will be
1956             # recognized as Cython functions anyway.
1957             attrs = dict(name=node.entry.func_cname)
1958             self.tb.start('StepIntoFunction', attrs=attrs)
1959             self.tb.end('StepIntoFunction')
1960
1961         self.visitchildren(node)
1962         return node
1963
1964     def serialize_modulenode_as_function(self, node):
1965         """
1966         Serialize the module-level code as a function so the debugger will know
1967         it's a "relevant frame" and it will know where to set the breakpoint
1968         for 'break modulename'.
1969         """
1970         name = node.full_module_name.rpartition('.')[-1]
1971
1972         cname_py2 = 'init' + name
1973         cname_py3 = 'PyInit_' + name
1974
1975         py2_attrs = dict(
1976             name=name,
1977             cname=cname_py2,
1978             pf_cname='',
1979             # Ignore the qualified_name, breakpoints should be set using
1980             # `cy break modulename:lineno` for module-level breakpoints.
1981             qualified_name='',
1982             lineno='1',
1983             is_initmodule_function="True",
1984         )
1985
1986         py3_attrs = dict(py2_attrs, cname=cname_py3)
1987
1988         self._serialize_modulenode_as_function(node, py2_attrs)
1989         self._serialize_modulenode_as_function(node, py3_attrs)
1990
1991     def _serialize_modulenode_as_function(self, node, attrs):
1992         self.tb.start('Function', attrs=attrs)
1993
1994         self.tb.start('Locals')
1995         self.serialize_local_variables(node.scope.entries)
1996         self.tb.end('Locals')
1997
1998         self.tb.start('Arguments')
1999         self.tb.end('Arguments')
2000
2001         self.tb.start('StepIntoFunctions')
2002         self.register_stepinto = True
2003         self.visitchildren(node)
2004         self.register_stepinto = False
2005         self.tb.end('StepIntoFunctions')
2006
2007         self.tb.end('Function')
2008
2009     def serialize_local_variables(self, entries):
2010         for entry in entries.values():
2011             if entry.type.is_pyobject:
2012                 vartype = 'PythonObject'
2013             else:
2014                 vartype = 'CObject'
2015
2016             if entry.from_closure:
2017                 # We're dealing with a closure where a variable from an outer
2018                 # scope is accessed, get it from the scope object.
2019                 cname = '%s->%s' % (Naming.cur_scope_cname,
2020                                     entry.outer_entry.cname)
2021
2022                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2023                                       entry.scope.name,
2024                                       entry.name)
2025             elif entry.in_closure:
2026                 cname = '%s->%s' % (Naming.cur_scope_cname,
2027                                     entry.cname)
2028                 qname = entry.qualified_name
2029             else:
2030                 cname = entry.cname
2031                 qname = entry.qualified_name
2032
2033             if not entry.pos:
2034                 # this happens for variables that are not in the user's code,
2035                 # e.g. for the global __builtins__, __doc__, etc. We can just
2036                 # set the lineno to 0 for those.
2037                 lineno = '0'
2038             else:
2039                 lineno = str(entry.pos[1])
2040
2041             attrs = dict(
2042                 name=entry.name,
2043                 cname=cname,
2044                 qualified_name=qname,
2045                 type=vartype,
2046                 lineno=lineno)
2047
2048             self.tb.start('LocalVar', attrs)
2049             self.tb.end('LocalVar')
2050