Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67 class NormalizeTree(CythonTransform):
  68     """
  69     This transform fixes up a few things after parsing
  70     in order to make the parse tree more suitable for
  71     transforms.
  72
  73     a) After parsing, blocks with only one statement will
  74     be represented by that statement, not by a StatListNode.
  75     When doing transforms this is annoying and inconsistent,
  76     as one cannot in general remove a statement in a consistent
  77     way and so on. This transform wraps any single statements
  78     in a StatListNode containing a single statement.
  79
  80     b) The PassStatNode is a noop and serves no purpose beyond
  81     plugging such one-statement blocks; i.e., once parsed a
  82 `    "pass" can just as well be represented using an empty
  83     StatListNode. This means less special cases to worry about
  84     in subsequent transforms (one always checks to see if a
  85     StatListNode has no children to see if the block is empty).
  86     """
  87
  88     def __init__(self, context):
  89         super(NormalizeTree, self).__init__(context)
  90         self.is_in_statlist = False
  91         self.is_in_expr = False
  92
  93     def visit_ExprNode(self, node):
  94         stacktmp = self.is_in_expr
  95         self.is_in_expr = True
  96         self.visitchildren(node)
  97         self.is_in_expr = stacktmp
  98         return node
  99
 100     def visit_StatNode(self, node, is_listcontainer=False):
 101         stacktmp = self.is_in_statlist
 102         self.is_in_statlist = is_listcontainer
 103         self.visitchildren(node)
 104         self.is_in_statlist = stacktmp
 105         if not self.is_in_statlist and not self.is_in_expr:
 106             return Nodes.StatListNode(pos=node.pos, stats=[node])
 107         else:
 108             return node
 109
 110     def visit_StatListNode(self, node):
 111         self.is_in_statlist = True
 112         self.visitchildren(node)
 113         self.is_in_statlist = False
 114         return node
 115
 116     def visit_ParallelAssignmentNode(self, node):
 117         return self.visit_StatNode(node, True)
 118
 119     def visit_CEnumDefNode(self, node):
 120         return self.visit_StatNode(node, True)
 121
 122     def visit_CStructOrUnionDefNode(self, node):
 123         return self.visit_StatNode(node, True)
 124
 125     # Eliminate PassStatNode
 126     def visit_PassStatNode(self, node):
 127         if not self.is_in_statlist:
 128             return Nodes.StatListNode(pos=node.pos, stats=[])
 129         else:
 130             return []
 131
 132     def visit_CDeclaratorNode(self, node):
 133         return node
 134
 135
 136 class PostParseError(CompileError): pass
 137
 138 # error strings checked by unit tests, so define them
 139 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 140 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 141 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 142 class PostParse(ScopeTrackingTransform):
 143     """
 144     Basic interpretation of the parse tree, as well as validity
 145     checking that can be done on a very basic level on the parse
 146     tree (while still not being a problem with the basic syntax,
 147     as such).
 148
 149     Specifically:
 150     - Default values to cdef assignments are turned into single
 151     assignments following the declaration (everywhere but in class
 152     bodies, where they raise a compile error)
 153
 154     - Interpret some node structures into Python runtime values.
 155     Some nodes take compile-time arguments (currently:
 156     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 157     which should be interpreted. This happens in a general way
 158     and other steps should be taken to ensure validity.
 159
 160     Type arguments cannot be interpreted in this way.
 161
 162     - For __cythonbufferdefaults__ the arguments are checked for
 163     validity.
 164
 165     TemplatedTypeNode has its directives interpreted:
 166     Any first positional argument goes into the "dtype" attribute,
 167     any "ndim" keyword argument goes into the "ndim" attribute and
 168     so on. Also it is checked that the directive combination is valid.
 169     - __cythonbufferdefaults__ attributes are parsed and put into the
 170     type information.
 171
 172     Note: Currently Parsing.py does a lot of interpretation and
 173     reorganization that can be refactored into this transform
 174     if a more pure Abstract Syntax Tree is wanted.
 175     """
 176
 177     def __init__(self, context):
 178         super(PostParse, self).__init__(context)
 179         self.specialattribute_handlers = {
 180             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 181         }
 182
 183     def visit_ModuleNode(self, node):
 184         self.lambda_counter = 1
 185         self.genexpr_counter = 1
 186         return super(PostParse, self).visit_ModuleNode(node)
 187
 188     def visit_LambdaNode(self, node):
 189         # unpack a lambda expression into the corresponding DefNode
 190         lambda_id = self.lambda_counter
 191         self.lambda_counter += 1
 192         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 193         collector = YieldNodeCollector()
 194         collector.visitchildren(node.result_expr)
 195         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 196             body = ExprNodes.YieldExprNode(
 197                 node.result_expr.pos, arg=node.result_expr)
 198             body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
 199         else:
 200             body = Nodes.ReturnStatNode(
 201                 node.result_expr.pos, value=node.result_expr)
 202         node.def_node = Nodes.DefNode(
 203             node.pos, name=node.name, lambda_name=node.lambda_name,
 204             args=node.args, star_arg=node.star_arg,
 205             starstar_arg=node.starstar_arg,
 206             body=body, doc=None)
 207         self.visitchildren(node)
 208         return node
 209
 210     def visit_GeneratorExpressionNode(self, node):
 211         # unpack a generator expression into the corresponding DefNode
 212         genexpr_id = self.genexpr_counter
 213         self.genexpr_counter += 1
 214         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 215
 216         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 217                                       doc=None,
 218                                       args=[], star_arg=None,
 219                                       starstar_arg=None,
 220                                       body=node.loop)
 221         self.visitchildren(node)
 222         return node
 223
 224     # cdef variables
 225     def handle_bufferdefaults(self, decl):
 226         if not isinstance(decl.default, ExprNodes.DictNode):
 227             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 228         self.scope_node.buffer_defaults_node = decl.default
 229         self.scope_node.buffer_defaults_pos = decl.pos
 230
 231     def visit_CVarDefNode(self, node):
 232         # This assumes only plain names and pointers are assignable on
 233         # declaration. Also, it makes use of the fact that a cdef decl
 234         # must appear before the first use, so we don't have to deal with
 235         # "i = 3; cdef int i = i" and can simply move the nodes around.
 236         try:
 237             self.visitchildren(node)
 238             stats = [node]
 239             newdecls = []
 240             for decl in node.declarators:
 241                 declbase = decl
 242                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 243                     declbase = declbase.base
 244                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 245                     if declbase.default is not None:
 246                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 247                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 248                                 handler = self.specialattribute_handlers.get(decl.name)
 249                                 if handler:
 250                                     if decl is not declbase:
 251                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 252                                     handler(decl)
 253                                     continue # Remove declaration
 254                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 255                         first_assignment = self.scope_type != 'module'
 256                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 257                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 258                             rhs=declbase.default, first=first_assignment))
 259                         declbase.default = None
 260                 newdecls.append(decl)
 261             node.declarators = newdecls
 262             return stats
 263         except PostParseError, e:
 264             # An error in a cdef clause is ok, simply remove the declaration
 265             # and try to move on to report more errors
 266             self.context.nonfatal_error(e)
 267             return None
 268
 269     # Split parallel assignments (a,b = b,a) into separate partial
 270     # assignments that are executed rhs-first using temps.  This
 271     # restructuring must be applied before type analysis so that known
 272     # types on rhs and lhs can be matched directly.  It is required in
 273     # the case that the types cannot be coerced to a Python type in
 274     # order to assign from a tuple.
 275
 276     def visit_SingleAssignmentNode(self, node):
 277         self.visitchildren(node)
 278         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 279
 280     def visit_CascadedAssignmentNode(self, node):
 281         self.visitchildren(node)
 282         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 283
 284     def _visit_assignment_node(self, node, expr_list):
 285         """Flatten parallel assignments into separate single
 286         assignments or cascaded assignments.
 287         """
 288         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 289             # no parallel assignments => nothing to do
 290             return node
 291
 292         expr_list_list = []
 293         flatten_parallel_assignments(expr_list, expr_list_list)
 294         temp_refs = []
 295         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 296
 297         nodes = []
 298         for expr_list in expr_list_list:
 299             lhs_list = expr_list[:-1]
 300             rhs = expr_list[-1]
 301             if len(lhs_list) == 1:
 302                 node = Nodes.SingleAssignmentNode(rhs.pos,
 303                     lhs = lhs_list[0], rhs = rhs)
 304             else:
 305                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 306                     lhs_list = lhs_list, rhs = rhs)
 307             nodes.append(node)
 308
 309         if len(nodes) == 1:
 310             assign_node = nodes[0]
 311         else:
 312             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 313
 314         if temp_refs:
 315             duplicates_and_temps = [ (temp.expression, temp)
 316                                      for temp in temp_refs ]
 317             sort_common_subsequences(duplicates_and_temps)
 318             for _, temp_ref in duplicates_and_temps[::-1]:
 319                 assign_node = LetNode(temp_ref, assign_node)
 320
 321         return assign_node
 322
 323 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 324     """Replace rhs items by LetRefNodes if they appear more than once.
 325     Creates a sequence of LetRefNodes that set up the required temps
 326     and appends them to ref_node_sequence.  The input list is modified
 327     in-place.
 328     """
 329     seen_nodes = cython.set()
 330     ref_nodes = {}
 331     def find_duplicates(node):
 332         if node.is_literal or node.is_name:
 333             # no need to replace those; can't include attributes here
 334             # as their access is not necessarily side-effect free
 335             return
 336         if node in seen_nodes:
 337             if node not in ref_nodes:
 338                 ref_node = LetRefNode(node)
 339                 ref_nodes[node] = ref_node
 340                 ref_node_sequence.append(ref_node)
 341         else:
 342             seen_nodes.add(node)
 343             if node.is_sequence_constructor:
 344                 for item in node.args:
 345                     find_duplicates(item)
 346
 347     for expr_list in expr_list_list:
 348         rhs = expr_list[-1]
 349         find_duplicates(rhs)
 350     if not ref_nodes:
 351         return
 352
 353     def substitute_nodes(node):
 354         if node in ref_nodes:
 355             return ref_nodes[node]
 356         elif node.is_sequence_constructor:
 357             node.args = list(map(substitute_nodes, node.args))
 358         return node
 359
 360     # replace nodes inside of the common subexpressions
 361     for node in ref_nodes:
 362         if node.is_sequence_constructor:
 363             node.args = list(map(substitute_nodes, node.args))
 364
 365     # replace common subexpressions on all rhs items
 366     for expr_list in expr_list_list:
 367         expr_list[-1] = substitute_nodes(expr_list[-1])
 368
 369 def sort_common_subsequences(items):
 370     """Sort items/subsequences so that all items and subsequences that
 371     an item contains appear before the item itself.  This is needed
 372     because each rhs item must only be evaluated once, so its value
 373     must be evaluated first and then reused when packing sequences
 374     that contain it.
 375
 376     This implies a partial order, and the sort must be stable to
 377     preserve the original order as much as possible, so we use a
 378     simple insertion sort (which is very fast for short sequences, the
 379     normal case in practice).
 380     """
 381     def contains(seq, x):
 382         for item in seq:
 383             if item is x:
 384                 return True
 385             elif item.is_sequence_constructor and contains(item.args, x):
 386                 return True
 387         return False
 388     def lower_than(a,b):
 389         return b.is_sequence_constructor and contains(b.args, a)
 390
 391     for pos, item in enumerate(items):
 392         key = item[1] # the ResultRefNode which has already been injected into the sequences
 393         new_pos = pos
 394         for i in xrange(pos-1, -1, -1):
 395             if lower_than(key, items[i][0]):
 396                 new_pos = i
 397         if new_pos != pos:
 398             for i in xrange(pos, new_pos, -1):
 399                 items[i] = items[i-1]
 400             items[new_pos] = item
 401
 402 def flatten_parallel_assignments(input, output):
 403     #  The input is a list of expression nodes, representing the LHSs
 404     #  and RHS of one (possibly cascaded) assignment statement.  For
 405     #  sequence constructors, rearranges the matching parts of both
 406     #  sides into a list of equivalent assignments between the
 407     #  individual elements.  This transformation is applied
 408     #  recursively, so that nested structures get matched as well.
 409     rhs = input[-1]
 410     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 411         output.append(input)
 412         return
 413
 414     complete_assignments = []
 415
 416     rhs_size = len(rhs.args)
 417     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 418     starred_assignments = []
 419     for lhs in input[:-1]:
 420         if not lhs.is_sequence_constructor:
 421             if lhs.is_starred:
 422                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 423             complete_assignments.append(lhs)
 424             continue
 425         lhs_size = len(lhs.args)
 426         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 427         if starred_targets > 1:
 428             error(lhs.pos, "more than 1 starred expression in assignment")
 429             output.append([lhs,rhs])
 430             continue
 431         elif lhs_size - starred_targets > rhs_size:
 432             error(lhs.pos, "need more than %d value%s to unpack"
 433                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 434             output.append([lhs,rhs])
 435             continue
 436         elif starred_targets:
 437             map_starred_assignment(lhs_targets, starred_assignments,
 438                                    lhs.args, rhs.args)
 439         elif lhs_size < rhs_size:
 440             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 441                   % (lhs_size, rhs_size))
 442             output.append([lhs,rhs])
 443             continue
 444         else:
 445             for targets, expr in zip(lhs_targets, lhs.args):
 446                 targets.append(expr)
 447
 448     if complete_assignments:
 449         complete_assignments.append(rhs)
 450         output.append(complete_assignments)
 451
 452     # recursively flatten partial assignments
 453     for cascade, rhs in zip(lhs_targets, rhs.args):
 454         if cascade:
 455             cascade.append(rhs)
 456             flatten_parallel_assignments(cascade, output)
 457
 458     # recursively flatten starred assignments
 459     for cascade in starred_assignments:
 460         if cascade[0].is_sequence_constructor:
 461             flatten_parallel_assignments(cascade, output)
 462         else:
 463             output.append(cascade)
 464
 465 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 466     # Appends the fixed-position LHS targets to the target list that
 467     # appear left and right of the starred argument.
 468     #
 469     # The starred_assignments list receives a new tuple
 470     # (lhs_target, rhs_values_list) that maps the remaining arguments
 471     # (those that match the starred target) to a list.
 472
 473     # left side of the starred target
 474     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 475         if expr.is_starred:
 476             starred = i
 477             lhs_remaining = len(lhs_args) - i - 1
 478             break
 479         targets.append(expr)
 480     else:
 481         raise InternalError("no starred arg found when splitting starred assignment")
 482
 483     # right side of the starred target
 484     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 485                                             lhs_args[starred + 1:])):
 486         targets.append(expr)
 487
 488     # the starred target itself, must be assigned a (potentially empty) list
 489     target = lhs_args[starred].target # unpack starred node
 490     starred_rhs = rhs_args[starred:]
 491     if lhs_remaining:
 492         starred_rhs = starred_rhs[:-lhs_remaining]
 493     if starred_rhs:
 494         pos = starred_rhs[0].pos
 495     else:
 496         pos = target.pos
 497     starred_assignments.append([
 498         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 499
 500
 501 class PxdPostParse(CythonTransform, SkipDeclarations):
 502     """
 503     Basic interpretation/validity checking that should only be
 504     done on pxd trees.
 505
 506     A lot of this checking currently happens in the parser; but
 507     what is listed below happens here.
 508
 509     - "def" functions are let through only if they fill the
 510     getbuffer/releasebuffer slots
 511
 512     - cdef functions are let through only if they are on the
 513     top level and are declared "inline"
 514     """
 515     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 516     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 517
 518     def __call__(self, node):
 519         self.scope_type = 'pxd'
 520         return super(PxdPostParse, self).__call__(node)
 521
 522     def visit_CClassDefNode(self, node):
 523         old = self.scope_type
 524         self.scope_type = 'cclass'
 525         self.visitchildren(node)
 526         self.scope_type = old
 527         return node
 528
 529     def visit_FuncDefNode(self, node):
 530         # FuncDefNode always come with an implementation (without
 531         # an imp they are CVarDefNodes..)
 532         err = self.ERR_INLINE_ONLY
 533
 534         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 535             and node.name in ('__getbuffer__', '__releasebuffer__')):
 536             err = None # allow these slots
 537
 538         if isinstance(node, Nodes.CFuncDefNode):
 539             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 540                 node.inline_in_pxd = True
 541                 if node.visibility != 'private':
 542                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 543                 elif node.api:
 544                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 545                 else:
 546                     err = None # allow inline function
 547             else:
 548                 err = self.ERR_INLINE_ONLY
 549
 550         if err:
 551             self.context.nonfatal_error(PostParseError(node.pos, err))
 552             return None
 553         else:
 554             return node
 555
 556 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 557     """
 558     After parsing, directives can be stored in a number of places:
 559     - #cython-comments at the top of the file (stored in ModuleNode)
 560     - Command-line arguments overriding these
 561     - @cython.directivename decorators
 562     - with cython.directivename: statements
 563
 564     This transform is responsible for interpreting these various sources
 565     and store the directive in two ways:
 566     - Set the directives attribute of the ModuleNode for global directives.
 567     - Use a CompilerDirectivesNode to override directives for a subtree.
 568
 569     (The first one is primarily to not have to modify with the tree
 570     structure, so that ModuleNode stay on top.)
 571
 572     The directives are stored in dictionaries from name to value in effect.
 573     Each such dictionary is always filled in for all possible directives,
 574     using default values where no value is given by the user.
 575
 576     The available directives are controlled in Options.py.
 577
 578     Note that we have to run this prior to analysis, and so some minor
 579     duplication of functionality has to occur: We manually track cimports
 580     and which names the "cython" module may have been imported to.
 581     """
 582     unop_method_nodes = {
 583         'typeof': ExprNodes.TypeofNode,
 584
 585         'operator.address': ExprNodes.AmpersandNode,
 586         'operator.dereference': ExprNodes.DereferenceNode,
 587         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 588         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 589         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 590         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 591
 592         # For backwards compatability.
 593         'address': ExprNodes.AmpersandNode,
 594     }
 595
 596     binop_method_nodes = {
 597         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 598     }
 599
 600     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 601                                   'cast', 'pointer', 'compiled', 'NULL'])
 602     special_methods.update(unop_method_nodes.keys())
 603
 604     def __init__(self, context, compilation_directive_defaults):
 605         super(InterpretCompilerDirectives, self).__init__(context)
 606         self.compilation_directive_defaults = {}
 607         for key, value in compilation_directive_defaults.items():
 608             self.compilation_directive_defaults[unicode(key)] = value
 609         self.cython_module_names = cython.set()
 610         self.directive_names = {}
 611
 612     def check_directive_scope(self, pos, directive, scope):
 613         legal_scopes = Options.directive_scopes.get(directive, None)
 614         if legal_scopes and scope not in legal_scopes:
 615             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 616                                         'is not allowed in %s scope' % (directive, scope)))
 617             return False
 618         else:
 619             return True
 620
 621     # Set up processing and handle the cython: comments.
 622     def visit_ModuleNode(self, node):
 623         for key, value in node.directive_comments.items():
 624             if not self.check_directive_scope(node.pos, key, 'module'):
 625                 self.wrong_scope_error(node.pos, key, 'module')
 626                 del node.directive_comments[key]
 627
 628         directives = copy.copy(Options.directive_defaults)
 629         directives.update(self.compilation_directive_defaults)
 630         directives.update(node.directive_comments)
 631         self.directives = directives
 632         node.directives = directives
 633         self.visitchildren(node)
 634         node.cython_module_names = self.cython_module_names
 635         return node
 636
 637     # The following four functions track imports and cimports that
 638     # begin with "cython"
 639     def is_cython_directive(self, name):
 640         return (name in Options.directive_types or
 641                 name in self.special_methods or
 642                 PyrexTypes.parse_basic_type(name))
 643
 644     def visit_CImportStatNode(self, node):
 645         if node.module_name == u"cython":
 646             self.cython_module_names.add(node.as_name or u"cython")
 647         elif node.module_name.startswith(u"cython."):
 648             if node.as_name:
 649                 self.directive_names[node.as_name] = node.module_name[7:]
 650             else:
 651                 self.cython_module_names.add(u"cython")
 652             # if this cimport was a compiler directive, we don't
 653             # want to leave the cimport node sitting in the tree
 654             return None
 655         return node
 656
 657     def visit_FromCImportStatNode(self, node):
 658         if (node.module_name == u"cython") or \
 659                node.module_name.startswith(u"cython."):
 660             submodule = (node.module_name + u".")[7:]
 661             newimp = []
 662             for pos, name, as_name, kind in node.imported_names:
 663                 full_name = submodule + name
 664                 if self.is_cython_directive(full_name):
 665                     if as_name is None:
 666                         as_name = full_name
 667                     self.directive_names[as_name] = full_name
 668                     if kind is not None:
 669                         self.context.nonfatal_error(PostParseError(pos,
 670                             "Compiler directive imports must be plain imports"))
 671                 else:
 672                     newimp.append((pos, name, as_name, kind))
 673             if not newimp:
 674                 return None
 675             node.imported_names = newimp
 676         return node
 677
 678     def visit_FromImportStatNode(self, node):
 679         if (node.module.module_name.value == u"cython") or \
 680                node.module.module_name.value.startswith(u"cython."):
 681             submodule = (node.module.module_name.value + u".")[7:]
 682             newimp = []
 683             for name, name_node in node.items:
 684                 full_name = submodule + name
 685                 if self.is_cython_directive(full_name):
 686                     self.directive_names[name_node.name] = full_name
 687                 else:
 688                     newimp.append((name, name_node))
 689             if not newimp:
 690                 return None
 691             node.items = newimp
 692         return node
 693
 694     def visit_SingleAssignmentNode(self, node):
 695         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 696                 node.rhs.module_name.value == u'cython'):
 697             node = Nodes.CImportStatNode(node.pos,
 698                                          module_name = u'cython',
 699                                          as_name = node.lhs.name)
 700             self.visit_CImportStatNode(node)
 701         else:
 702             self.visitchildren(node)
 703         return node
 704
 705     def visit_NameNode(self, node):
 706         if node.name in self.cython_module_names:
 707             node.is_cython_module = True
 708         else:
 709             node.cython_attribute = self.directive_names.get(node.name)
 710         return node
 711
 712     def try_to_parse_directives(self, node):
 713         # If node is the contents of an directive (in a with statement or
 714         # decorator), returns a list of (directivename, value) pairs.
 715         # Otherwise, returns None
 716         if isinstance(node, ExprNodes.CallNode):
 717             self.visit(node.function)
 718             optname = node.function.as_cython_attribute()
 719             if optname:
 720                 directivetype = Options.directive_types.get(optname)
 721                 if directivetype:
 722                     args, kwds = node.explicit_args_kwds()
 723                     directives = []
 724                     key_value_pairs = []
 725                     if kwds is not None and directivetype is not dict:
 726                         for keyvalue in kwds.key_value_pairs:
 727                             key, value = keyvalue
 728                             sub_optname = "%s.%s" % (optname, key.value)
 729                             if Options.directive_types.get(sub_optname):
 730                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 731                             else:
 732                                 key_value_pairs.append(keyvalue)
 733                         if not key_value_pairs:
 734                             kwds = None
 735                         else:
 736                             kwds.key_value_pairs = key_value_pairs
 737                         if directives and not kwds and not args:
 738                             return directives
 739                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 740                     return directives
 741         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 742             self.visit(node)
 743             optname = node.as_cython_attribute()
 744             if optname:
 745                 directivetype = Options.directive_types.get(optname)
 746                 if directivetype is bool:
 747                     return [(optname, True)]
 748                 elif directivetype is None:
 749                     return [(optname, None)]
 750                 else:
 751                     raise PostParseError(
 752                         node.pos, "The '%s' directive should be used as a function call." % optname)
 753         return None
 754
 755     def try_to_parse_directive(self, optname, args, kwds, pos):
 756         directivetype = Options.directive_types.get(optname)
 757         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 758             return optname, Options.directive_defaults[optname]
 759         elif directivetype is bool:
 760             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 761                 raise PostParseError(pos,
 762                     'The %s directive takes one compile-time boolean argument' % optname)
 763             return (optname, args[0].value)
 764         elif directivetype is str:
 765             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 766                                                                               ExprNodes.UnicodeNode)):
 767                 raise PostParseError(pos,
 768                     'The %s directive takes one compile-time string argument' % optname)
 769             return (optname, str(args[0].value))
 770         elif directivetype is dict:
 771             if len(args) != 0:
 772                 raise PostParseError(pos,
 773                     'The %s directive takes no prepositional arguments' % optname)
 774             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 775         elif directivetype is list:
 776             if kwds and len(kwds) != 0:
 777                 raise PostParseError(pos,
 778                     'The %s directive takes no keyword arguments' % optname)
 779             return optname, [ str(arg.value) for arg in args ]
 780         else:
 781             assert False
 782
 783     def visit_with_directives(self, body, directives):
 784         olddirectives = self.directives
 785         newdirectives = copy.copy(olddirectives)
 786         newdirectives.update(directives)
 787         self.directives = newdirectives
 788         assert isinstance(body, Nodes.StatListNode), body
 789         retbody = self.visit_Node(body)
 790         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 791                                                  directives=newdirectives)
 792         self.directives = olddirectives
 793         return directive
 794
 795     # Handle decorators
 796     def visit_FuncDefNode(self, node):
 797         directives = self._extract_directives(node, 'function')
 798         if not directives:
 799             return self.visit_Node(node)
 800         body = Nodes.StatListNode(node.pos, stats=[node])
 801         return self.visit_with_directives(body, directives)
 802
 803     def visit_CVarDefNode(self, node):
 804         if not node.decorators:
 805             return node
 806         for dec in node.decorators:
 807             for directive in self.try_to_parse_directives(dec.decorator) or ():
 808                 if directive is not None and directive[0] == u'locals':
 809                     node.directive_locals = directive[1]
 810                 else:
 811                     self.context.nonfatal_error(PostParseError(dec.pos,
 812                         "Cdef functions can only take cython.locals() decorator."))
 813         return node
 814
 815     def visit_CClassDefNode(self, node):
 816         directives = self._extract_directives(node, 'cclass')
 817         if not directives:
 818             return self.visit_Node(node)
 819         body = Nodes.StatListNode(node.pos, stats=[node])
 820         return self.visit_with_directives(body, directives)
 821
 822     def visit_PyClassDefNode(self, node):
 823         directives = self._extract_directives(node, 'class')
 824         if not directives:
 825             return self.visit_Node(node)
 826         body = Nodes.StatListNode(node.pos, stats=[node])
 827         return self.visit_with_directives(body, directives)
 828
 829     def _extract_directives(self, node, scope_name):
 830         if not node.decorators:
 831             return {}
 832         # Split the decorators into two lists -- real decorators and directives
 833         directives = []
 834         realdecs = []
 835         for dec in node.decorators:
 836             new_directives = self.try_to_parse_directives(dec.decorator)
 837             if new_directives is not None:
 838                 for directive in new_directives:
 839                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 840                         directives.append(directive)
 841             else:
 842                 realdecs.append(dec)
 843         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 844             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 845         else:
 846             node.decorators = realdecs
 847         # merge or override repeated directives
 848         optdict = {}
 849         directives.reverse() # Decorators coming first take precedence
 850         for directive in directives:
 851             name, value = directive
 852             if name in optdict:
 853                 old_value = optdict[name]
 854                 # keywords and arg lists can be merged, everything
 855                 # else overrides completely
 856                 if isinstance(old_value, dict):
 857                     old_value.update(value)
 858                 elif isinstance(old_value, list):
 859                     old_value.extend(value)
 860                 else:
 861                     optdict[name] = value
 862             else:
 863                 optdict[name] = value
 864         return optdict
 865
 866     # Handle with statements
 867     def visit_WithStatNode(self, node):
 868         directive_dict = {}
 869         for directive in self.try_to_parse_directives(node.manager) or []:
 870             if directive is not None:
 871                 if node.target is not None:
 872                     self.context.nonfatal_error(
 873                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 874                 else:
 875                     name, value = directive
 876                     if name == 'nogil':
 877                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 878                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 879                         return self.visit_Node(node)
 880                     if self.check_directive_scope(node.pos, name, 'with statement'):
 881                         directive_dict[name] = value
 882         if directive_dict:
 883             return self.visit_with_directives(node.body, directive_dict)
 884         return self.visit_Node(node)
 885
 886 class WithTransform(CythonTransform, SkipDeclarations):
 887
 888     # EXCINFO is manually set to a variable that contains
 889     # the exc_info() tuple that can be generated by the enclosing except
 890     # statement.
 891     template_without_target = TreeFragment(u"""
 892         MGR = EXPR
 893         EXIT = MGR.__exit__
 894         MGR.__enter__()
 895         EXC = True
 896         try:
 897             try:
 898                 EXCINFO = None
 899                 BODY
 900             except:
 901                 EXC = False
 902                 if not EXIT(*EXCINFO):
 903                     raise
 904         finally:
 905             if EXC:
 906                 EXIT(None, None, None)
 907     """, temps=[u'MGR', u'EXC', u"EXIT"],
 908     pipeline=[NormalizeTree(None)])
 909
 910     template_with_target = TreeFragment(u"""
 911         MGR = EXPR
 912         EXIT = MGR.__exit__
 913         VALUE = MGR.__enter__()
 914         EXC = True
 915         try:
 916             try:
 917                 EXCINFO = None
 918                 TARGET = VALUE
 919                 BODY
 920             except:
 921                 EXC = False
 922                 if not EXIT(*EXCINFO):
 923                     raise
 924         finally:
 925             if EXC:
 926                 EXIT(None, None, None)
 927             MGR = EXIT = VALUE = EXC = None
 928
 929     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 930     pipeline=[NormalizeTree(None)])
 931
 932     def visit_WithStatNode(self, node):
 933         # TODO: Cleanup badly needed
 934         TemplateTransform.temp_name_counter += 1
 935         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 936
 937         self.visitchildren(node, ['body'])
 938         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 939         if node.target is not None:
 940             result = self.template_with_target.substitute({
 941                 u'EXPR' : node.manager,
 942                 u'BODY' : node.body,
 943                 u'TARGET' : node.target,
 944                 u'EXCINFO' : excinfo_temp
 945                 }, pos=node.pos)
 946         else:
 947             result = self.template_without_target.substitute({
 948                 u'EXPR' : node.manager,
 949                 u'BODY' : node.body,
 950                 u'EXCINFO' : excinfo_temp
 951                 }, pos=node.pos)
 952
 953         # Set except excinfo target to EXCINFO
 954         try_except = result.stats[-1].body.stats[-1]
 955         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 956 #            excinfo_temp.ref(node.pos))
 957
 958 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 959 #            node.pos, temps=[excinfo_temp], body=try_except)
 960
 961         return result
 962
 963     def visit_ExprNode(self, node):
 964         # With statements are never inside expressions.
 965         return node
 966
 967
 968 class DecoratorTransform(CythonTransform, SkipDeclarations):
 969
 970     def visit_DefNode(self, func_node):
 971         self.visitchildren(func_node)
 972         if not func_node.decorators:
 973             return func_node
 974         return self._handle_decorators(
 975             func_node, func_node.name)
 976
 977     def visit_CClassDefNode(self, class_node):
 978         # This doesn't currently work, so it's disabled.
 979         #
 980         # Problem: assignments to cdef class names do not work.  They
 981         # would require an additional check anyway, as the extension
 982         # type must not change its C type, so decorators cannot
 983         # replace an extension type, just alter it and return it.
 984
 985         self.visitchildren(class_node)
 986         if not class_node.decorators:
 987             return class_node
 988         error(class_node.pos,
 989               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 990         return class_node
 991         #return self._handle_decorators(
 992         #    class_node, class_node.class_name)
 993
 994     def visit_ClassDefNode(self, class_node):
 995         self.visitchildren(class_node)
 996         if not class_node.decorators:
 997             return class_node
 998         return self._handle_decorators(
 999             class_node, class_node.name)
1000
1001     def _handle_decorators(self, node, name):
1002         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1003         for decorator in node.decorators[::-1]:
1004             decorator_result = ExprNodes.SimpleCallNode(
1005                 decorator.pos,
1006                 function = decorator.decorator,
1007                 args = [decorator_result])
1008
1009         name_node = ExprNodes.NameNode(node.pos, name = name)
1010         reassignment = Nodes.SingleAssignmentNode(
1011             node.pos,
1012             lhs = name_node,
1013             rhs = decorator_result)
1014         return [node, reassignment]
1015
1016
1017 class AnalyseDeclarationsTransform(CythonTransform):
1018
1019     basic_property = TreeFragment(u"""
1020 property NAME:
1021     def __get__(self):
1022         return ATTR
1023     def __set__(self, value):
1024         ATTR = value
1025     """, level='c_class')
1026     basic_pyobject_property = TreeFragment(u"""
1027 property NAME:
1028     def __get__(self):
1029         return ATTR
1030     def __set__(self, value):
1031         ATTR = value
1032     def __del__(self):
1033         ATTR = None
1034     """, level='c_class')
1035     basic_property_ro = TreeFragment(u"""
1036 property NAME:
1037     def __get__(self):
1038         return ATTR
1039     """, level='c_class')
1040
1041     struct_or_union_wrapper = TreeFragment(u"""
1042 cdef class NAME:
1043     cdef TYPE value
1044     def __init__(self, MEMBER=None):
1045         cdef int count
1046         count = 0
1047         INIT_ASSIGNMENTS
1048         if IS_UNION and count > 1:
1049             raise ValueError, "At most one union member should be specified."
1050     def __str__(self):
1051         return STR_FORMAT % MEMBER_TUPLE
1052     def __repr__(self):
1053         return REPR_FORMAT % MEMBER_TUPLE
1054     """)
1055
1056     init_assignment = TreeFragment(u"""
1057 if VALUE is not None:
1058     ATTR = VALUE
1059     count += 1
1060     """)
1061
1062     def __call__(self, root):
1063         self.env_stack = [root.scope]
1064         # needed to determine if a cdef var is declared after it's used.
1065         self.seen_vars_stack = []
1066         return super(AnalyseDeclarationsTransform, self).__call__(root)
1067
1068     def visit_NameNode(self, node):
1069         self.seen_vars_stack[-1].add(node.name)
1070         return node
1071
1072     def visit_ModuleNode(self, node):
1073         self.seen_vars_stack.append(cython.set())
1074         node.analyse_declarations(self.env_stack[-1])
1075         self.visitchildren(node)
1076         self.seen_vars_stack.pop()
1077         return node
1078
1079     def visit_LambdaNode(self, node):
1080         node.analyse_declarations(self.env_stack[-1])
1081         self.visitchildren(node)
1082         return node
1083
1084     def visit_ClassDefNode(self, node):
1085         self.env_stack.append(node.scope)
1086         self.visitchildren(node)
1087         self.env_stack.pop()
1088         return node
1089
1090     def visit_CClassDefNode(self, node):
1091         node = self.visit_ClassDefNode(node)
1092         if node.scope and node.scope.implemented:
1093             stats = []
1094             for entry in node.scope.var_entries:
1095                 if entry.needs_property:
1096                     property = self.create_Property(entry)
1097                     property.analyse_declarations(node.scope)
1098                     self.visit(property)
1099                     stats.append(property)
1100             if stats:
1101                 node.body.stats += stats
1102         return node
1103
1104     def visit_FuncDefNode(self, node):
1105         self.seen_vars_stack.append(cython.set())
1106         lenv = node.local_scope
1107         node.body.analyse_control_flow(lenv) # this will be totally refactored
1108         node.declare_arguments(lenv)
1109         for var, type_node in node.directive_locals.items():
1110             if not lenv.lookup_here(var):   # don't redeclare args
1111                 type = type_node.analyse_as_type(lenv)
1112                 if type:
1113                     lenv.declare_var(var, type, type_node.pos)
1114                 else:
1115                     error(type_node.pos, "Not a type")
1116         node.body.analyse_declarations(lenv)
1117         self.env_stack.append(lenv)
1118         self.visitchildren(node)
1119         self.env_stack.pop()
1120         self.seen_vars_stack.pop()
1121         return node
1122
1123     def visit_ScopedExprNode(self, node):
1124         env = self.env_stack[-1]
1125         node.analyse_declarations(env)
1126         # the node may or may not have a local scope
1127         if node.has_local_scope:
1128             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1129             self.env_stack.append(node.expr_scope)
1130             node.analyse_scoped_declarations(node.expr_scope)
1131             self.visitchildren(node)
1132             self.env_stack.pop()
1133             self.seen_vars_stack.pop()
1134         else:
1135             node.analyse_scoped_declarations(env)
1136             self.visitchildren(node)
1137         return node
1138
1139     def visit_TempResultFromStatNode(self, node):
1140         self.visitchildren(node)
1141         node.analyse_declarations(self.env_stack[-1])
1142         return node
1143
1144     def visit_CStructOrUnionDefNode(self, node):
1145         # Create a wrapper node if needed.
1146         # We want to use the struct type information (so it can't happen
1147         # before this phase) but also create new objects to be declared
1148         # (so it can't happen later).
1149         # Note that we don't return the original node, as it is
1150         # never used after this phase.
1151         if True: # private (default)
1152             return None
1153
1154         self_value = ExprNodes.AttributeNode(
1155             pos = node.pos,
1156             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1157             attribute = EncodedString(u"value"))
1158         var_entries = node.entry.type.scope.var_entries
1159         attributes = []
1160         for entry in var_entries:
1161             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1162                                                       obj = self_value,
1163                                                       attribute = entry.name))
1164         # __init__ assignments
1165         init_assignments = []
1166         for entry, attr in zip(var_entries, attributes):
1167             # TODO: branch on visibility
1168             init_assignments.append(self.init_assignment.substitute({
1169                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1170                     u"ATTR": attr,
1171                 }, pos = entry.pos))
1172
1173         # create the class
1174         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1175         wrapper_class = self.struct_or_union_wrapper.substitute({
1176             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1177             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1178             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1179             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1180             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1181         }, pos = node.pos).stats[0]
1182         wrapper_class.class_name = node.name
1183         wrapper_class.shadow = True
1184         class_body = wrapper_class.body.stats
1185
1186         # fix value type
1187         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1188         class_body[0].base_type.name = node.name
1189
1190         # fix __init__ arguments
1191         init_method = class_body[1]
1192         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1193         arg_template = init_method.args[1]
1194         if not node.entry.type.is_struct:
1195             arg_template.kw_only = True
1196         del init_method.args[1]
1197         for entry, attr in zip(var_entries, attributes):
1198             arg = copy.deepcopy(arg_template)
1199             arg.declarator.name = entry.name
1200             init_method.args.append(arg)
1201
1202         # setters/getters
1203         for entry, attr in zip(var_entries, attributes):
1204             # TODO: branch on visibility
1205             if entry.type.is_pyobject:
1206                 template = self.basic_pyobject_property
1207             else:
1208                 template = self.basic_property
1209             property = template.substitute({
1210                     u"ATTR": attr,
1211                 }, pos = entry.pos).stats[0]
1212             property.name = entry.name
1213             wrapper_class.body.stats.append(property)
1214
1215         wrapper_class.analyse_declarations(self.env_stack[-1])
1216         return self.visit_CClassDefNode(wrapper_class)
1217
1218     # Some nodes are no longer needed after declaration
1219     # analysis and can be dropped. The analysis was performed
1220     # on these nodes in a seperate recursive process from the
1221     # enclosing function or module, so we can simply drop them.
1222     def visit_CDeclaratorNode(self, node):
1223         # necessary to ensure that all CNameDeclaratorNodes are visited.
1224         self.visitchildren(node)
1225         return node
1226
1227     def visit_CTypeDefNode(self, node):
1228         return node
1229
1230     def visit_CBaseTypeNode(self, node):
1231         return None
1232
1233     def visit_CEnumDefNode(self, node):
1234         if node.visibility == 'public':
1235             return node
1236         else:
1237             return None
1238
1239     def visit_CNameDeclaratorNode(self, node):
1240         if node.name in self.seen_vars_stack[-1]:
1241             entry = self.env_stack[-1].lookup(node.name)
1242             if entry is None or entry.visibility != 'extern':
1243                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1244         self.visitchildren(node)
1245         return node
1246
1247     def visit_CVarDefNode(self, node):
1248         # to ensure all CNameDeclaratorNodes are visited.
1249         self.visitchildren(node)
1250         return None
1251
1252     def create_Property(self, entry):
1253         if entry.visibility == 'public':
1254             if entry.type.is_pyobject:
1255                 template = self.basic_pyobject_property
1256             else:
1257                 template = self.basic_property
1258         elif entry.visibility == 'readonly':
1259             template = self.basic_property_ro
1260         property = template.substitute({
1261                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1262                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1263                                                  attribute=entry.name),
1264             }, pos=entry.pos).stats[0]
1265         property.name = entry.name
1266         # ---------------------------------------
1267         # XXX This should go to AutoDocTransforms
1268         # ---------------------------------------
1269         if (Options.docstrings and
1270             self.current_directives['embedsignature']):
1271             attr_name = entry.name
1272             type_name = entry.type.declaration_code("", for_display=1)
1273             default_value = ''
1274             if not entry.type.is_pyobject:
1275                 type_name = "'%s'" % type_name
1276             elif entry.type.is_extension_type:
1277                 type_name = entry.type.module_name + '.' + type_name
1278             if entry.init is not None:
1279                 default_value = ' = ' + entry.init
1280             elif entry.init_to_none:
1281                 default_value = ' = ' + repr(None)
1282             docstring = attr_name + ': ' + type_name + default_value
1283             property.doc = EncodedString(docstring)
1284         # ---------------------------------------
1285         return property
1286
1287 class AnalyseExpressionsTransform(CythonTransform):
1288
1289     def visit_ModuleNode(self, node):
1290         node.scope.infer_types()
1291         node.body.analyse_expressions(node.scope)
1292         self.visitchildren(node)
1293         return node
1294
1295     def visit_FuncDefNode(self, node):
1296         node.local_scope.infer_types()
1297         node.body.analyse_expressions(node.local_scope)
1298         self.visitchildren(node)
1299         return node
1300
1301     def visit_ScopedExprNode(self, node):
1302         if node.has_local_scope:
1303             node.expr_scope.infer_types()
1304             node.analyse_scoped_expressions(node.expr_scope)
1305         self.visitchildren(node)
1306         return node
1307
1308 class ExpandInplaceOperators(EnvTransform):
1309
1310     def visit_InPlaceAssignmentNode(self, node):
1311         lhs = node.lhs
1312         rhs = node.rhs
1313         if lhs.type.is_cpp_class:
1314             # No getting around this exact operator here.
1315             return node
1316         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1317             # There is code to handle this case.
1318             return node
1319
1320         env = self.current_env()
1321         def side_effect_free_reference(node, setting=False):
1322             if isinstance(node, ExprNodes.NameNode):
1323                 return node, []
1324             elif node.type.is_pyobject and not setting:
1325                 node = LetRefNode(node)
1326                 return node, [node]
1327             elif isinstance(node, ExprNodes.IndexNode):
1328                 if node.is_buffer_access:
1329                     raise ValueError, "Buffer access"
1330                 base, temps = side_effect_free_reference(node.base)
1331                 index = LetRefNode(node.index)
1332                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1333             elif isinstance(node, ExprNodes.AttributeNode):
1334                 obj, temps = side_effect_free_reference(node.obj)
1335                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1336             else:
1337                 node = LetRefNode(node)
1338                 return node, [node]
1339         try:
1340             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1341         except ValueError:
1342             return node
1343         dup = lhs.__class__(**lhs.__dict__)
1344         binop = ExprNodes.binop_node(node.pos,
1345                                      operator = node.operator,
1346                                      operand1 = dup,
1347                                      operand2 = rhs,
1348                                      inplace=True)
1349         # Manually analyse types for new node.
1350         lhs.analyse_target_types(env)
1351         dup.analyse_types(env)
1352         binop.analyse_operation(env)
1353         node = Nodes.SingleAssignmentNode(
1354             node.pos,
1355             lhs = lhs,
1356             rhs=binop.coerce_to(lhs.type, env))
1357         # Use LetRefNode to avoid side effects.
1358         let_ref_nodes.reverse()
1359         for t in let_ref_nodes:
1360             node = LetNode(t, node)
1361         return node
1362
1363     def visit_ExprNode(self, node):
1364         # In-place assignments can't happen within an expression.
1365         return node
1366
1367
1368 class AlignFunctionDefinitions(CythonTransform):
1369     """
1370     This class takes the signatures from a .pxd file and applies them to
1371     the def methods in a .py file.
1372     """
1373
1374     def visit_ModuleNode(self, node):
1375         self.scope = node.scope
1376         self.directives = node.directives
1377         self.visitchildren(node)
1378         return node
1379
1380     def visit_PyClassDefNode(self, node):
1381         pxd_def = self.scope.lookup(node.name)
1382         if pxd_def:
1383             if pxd_def.is_cclass:
1384                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1385             else:
1386                 error(node.pos, "'%s' redeclared" % node.name)
1387                 error(pxd_def.pos, "previous declaration here")
1388                 return None
1389         else:
1390             return node
1391
1392     def visit_CClassDefNode(self, node, pxd_def=None):
1393         if pxd_def is None:
1394             pxd_def = self.scope.lookup(node.class_name)
1395         if pxd_def:
1396             outer_scope = self.scope
1397             self.scope = pxd_def.type.scope
1398         self.visitchildren(node)
1399         if pxd_def:
1400             self.scope = outer_scope
1401         return node
1402
1403     def visit_DefNode(self, node):
1404         pxd_def = self.scope.lookup(node.name)
1405         if pxd_def:
1406             if not pxd_def.is_cfunction:
1407                 error(node.pos, "'%s' redeclared" % node.name)
1408                 error(pxd_def.pos, "previous declaration here")
1409                 return None
1410             node = node.as_cfunction(pxd_def)
1411         elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1412             node = node.as_cfunction(scope=self.scope)
1413         # Enable this when internal def functions are allowed.
1414         # self.visitchildren(node)
1415         return node
1416
1417
1418 class YieldNodeCollector(TreeVisitor):
1419
1420     def __init__(self):
1421         super(YieldNodeCollector, self).__init__()
1422         self.yields = []
1423         self.returns = []
1424         self.has_return_value = False
1425
1426     def visit_Node(self, node):
1427         return self.visitchildren(node)
1428
1429     def visit_YieldExprNode(self, node):
1430         if self.has_return_value:
1431             error(node.pos, "'yield' outside function")
1432         self.yields.append(node)
1433         self.visitchildren(node)
1434
1435     def visit_ReturnStatNode(self, node):
1436         if node.value:
1437             self.has_return_value = True
1438             if self.yields:
1439                 error(node.pos, "'return' with argument inside generator")
1440         self.returns.append(node)
1441
1442     def visit_ClassDefNode(self, node):
1443         pass
1444
1445     def visit_DefNode(self, node):
1446         pass
1447
1448     def visit_LambdaNode(self, node):
1449         pass
1450
1451     def visit_GeneratorExpressionNode(self, node):
1452         pass
1453
1454 class MarkClosureVisitor(CythonTransform):
1455
1456     def visit_ModuleNode(self, node):
1457         self.needs_closure = False
1458         self.visitchildren(node)
1459         return node
1460
1461     def visit_FuncDefNode(self, node):
1462         self.needs_closure = False
1463         self.visitchildren(node)
1464         node.needs_closure = self.needs_closure
1465         self.needs_closure = True
1466
1467         collector = YieldNodeCollector()
1468         collector.visitchildren(node)
1469
1470         if collector.yields:
1471             for i, yield_expr in enumerate(collector.yields):
1472                 yield_expr.label_num = i + 1
1473
1474             gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1475                                                name=node.name,
1476                                                body=node.body)
1477             generator = Nodes.GeneratorDefNode(pos=node.pos,
1478                                                name=node.name,
1479                                                args=node.args,
1480                                                star_arg=node.star_arg,
1481                                                starstar_arg=node.starstar_arg,
1482                                                doc=node.doc,
1483                                                decorators=node.decorators,
1484                                                gbody=gbody,
1485                                                lambda_name=node.lambda_name)
1486             return generator
1487         return node
1488
1489     def visit_CFuncDefNode(self, node):
1490         self.visit_FuncDefNode(node)
1491         if node.needs_closure:
1492             error(node.pos, "closures inside cdef functions not yet supported")
1493         return node
1494
1495     def visit_LambdaNode(self, node):
1496         self.needs_closure = False
1497         self.visitchildren(node)
1498         node.needs_closure = self.needs_closure
1499         self.needs_closure = True
1500         return node
1501
1502     def visit_ClassDefNode(self, node):
1503         self.visitchildren(node)
1504         self.needs_closure = True
1505         return node
1506
1507 class CreateClosureClasses(CythonTransform):
1508     # Output closure classes in module scope for all functions
1509     # that really need it.
1510
1511     def __init__(self, context):
1512         super(CreateClosureClasses, self).__init__(context)
1513         self.path = []
1514         self.in_lambda = False
1515         self.generator_class = None
1516
1517     def visit_ModuleNode(self, node):
1518         self.module_scope = node.scope
1519         self.visitchildren(node)
1520         return node
1521
1522     def create_generator_class(self, target_module_scope, pos):
1523         if self.generator_class:
1524             return self.generator_class
1525         # XXX: make generator class creation cleaner
1526         entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1527                     objstruct_cname='__pyx_Generator_object',
1528                     typeobj_cname='__pyx_Generator_type',
1529                     pos=pos, defining=True, implementing=True)
1530         klass = entry.type.scope
1531         klass.is_internal = True
1532         klass.directives = {'final': True}
1533
1534         body_type = PyrexTypes.create_typedef_type('generator_body',
1535                                                    PyrexTypes.c_void_ptr_type,
1536                                                    '__pyx_generator_body_t')
1537         klass.declare_var(pos=pos, name='body', cname='body',
1538                           type=body_type, is_cdef=True)
1539         klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1540                           is_cdef=True)
1541         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1542                           is_cdef=True)
1543
1544         import TypeSlots
1545         e = klass.declare_pyfunction('send', pos)
1546         e.func_cname = '__Pyx_Generator_Send'
1547         e.signature = TypeSlots.binaryfunc
1548
1549         e = klass.declare_pyfunction('close', pos)
1550         e.func_cname = '__Pyx_Generator_Close'
1551         e.signature = TypeSlots.unaryfunc
1552
1553         e = klass.declare_pyfunction('throw', pos)
1554         e.func_cname = '__Pyx_Generator_Throw'
1555         e.signature = TypeSlots.pyfunction_signature
1556
1557         e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1558         e.func_cname = 'PyObject_SelfIter'
1559
1560         e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1561         e.func_cname = '__Pyx_Generator_Next'
1562
1563         self.generator_class = entry.type
1564         return self.generator_class
1565
1566     def find_entries_used_in_closures(self, node):
1567         from_closure = []
1568         in_closure = []
1569         for name, entry in node.local_scope.entries.items():
1570             if entry.from_closure:
1571                 from_closure.append((name, entry))
1572             elif entry.in_closure:
1573                 in_closure.append((name, entry))
1574         return from_closure, in_closure
1575
1576     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1577         # skip generator body
1578         if node.is_generator_body:
1579             return
1580         # move local variables into closure
1581         if node.is_generator:
1582             for entry in node.local_scope.entries.values():
1583                 if not entry.from_closure:
1584                     entry.in_closure = True
1585
1586         from_closure, in_closure = self.find_entries_used_in_closures(node)
1587         in_closure.sort()
1588
1589         # Now from the begining
1590         node.needs_closure = False
1591         node.needs_outer_scope = False
1592
1593         func_scope = node.local_scope
1594         cscope = node.entry.scope
1595         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1596             cscope = cscope.outer_scope
1597
1598         if not from_closure and (self.path or inner_node):
1599             if not inner_node:
1600                 if not node.assmt:
1601                     raise InternalError, "DefNode does not have assignment node"
1602                 inner_node = node.assmt.rhs
1603             inner_node.needs_self_code = False
1604             node.needs_outer_scope = False
1605
1606         if node.is_generator:
1607             generator_class = self.create_generator_class(target_module_scope, node.pos)
1608         elif not in_closure and not from_closure:
1609             return
1610         elif not in_closure:
1611             func_scope.is_passthrough = True
1612             func_scope.scope_class = cscope.scope_class
1613             node.needs_outer_scope = True
1614             return
1615
1616         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1617
1618         if node.is_generator:
1619             entry = target_module_scope.declare_c_class(name = as_name,
1620                         pos = node.pos, defining = True, implementing = True, base_type=generator_class)
1621         else:
1622             entry = target_module_scope.declare_c_class(name = as_name,
1623                         pos = node.pos, defining = True, implementing = True)
1624         func_scope.scope_class = entry
1625         class_scope = entry.type.scope
1626         class_scope.is_internal = True
1627         class_scope.directives = {'final': True}
1628
1629         if from_closure:
1630             assert cscope.is_closure_scope
1631             class_scope.declare_var(pos=node.pos,
1632                                     name=Naming.outer_scope_cname,
1633                                     cname=Naming.outer_scope_cname,
1634                                     type=cscope.scope_class.type,
1635                                     is_cdef=True)
1636             node.needs_outer_scope = True
1637         for name, entry in in_closure:
1638             closure_entry = class_scope.declare_var(pos=entry.pos,
1639                                     name=entry.name,
1640                                     cname=entry.cname,
1641                                     type=entry.type,
1642                                     is_cdef=True)
1643             if entry.is_declared_generic:
1644                 closure_entry.is_declared_generic = 1
1645         node.needs_closure = True
1646         # Do it here because other classes are already checked
1647         target_module_scope.check_c_class(func_scope.scope_class)
1648
1649     def visit_LambdaNode(self, node):
1650         was_in_lambda = self.in_lambda
1651         self.in_lambda = True
1652         self.create_class_from_scope(node.def_node, self.module_scope, node)
1653         self.visitchildren(node)
1654         self.in_lambda = was_in_lambda
1655         return node
1656
1657     def visit_FuncDefNode(self, node):
1658         if self.in_lambda:
1659             self.visitchildren(node)
1660             return node
1661         if node.needs_closure or self.path:
1662             self.create_class_from_scope(node, self.module_scope)
1663             self.path.append(node)
1664             self.visitchildren(node)
1665             self.path.pop()
1666         return node
1667
1668
1669 class GilCheck(VisitorTransform):
1670     """
1671     Call `node.gil_check(env)` on each node to make sure we hold the
1672     GIL when we need it.  Raise an error when on Python operations
1673     inside a `nogil` environment.
1674     """
1675     def __call__(self, root):
1676         self.env_stack = [root.scope]
1677         self.nogil = False
1678         return super(GilCheck, self).__call__(root)
1679
1680     def visit_FuncDefNode(self, node):
1681         self.env_stack.append(node.local_scope)
1682         was_nogil = self.nogil
1683         self.nogil = node.local_scope.nogil
1684         if self.nogil and node.nogil_check:
1685             node.nogil_check(node.local_scope)
1686         self.visitchildren(node)
1687         self.env_stack.pop()
1688         self.nogil = was_nogil
1689         return node
1690
1691     def visit_GILStatNode(self, node):
1692         env = self.env_stack[-1]
1693         if self.nogil and node.nogil_check: node.nogil_check()
1694         was_nogil = self.nogil
1695         self.nogil = (node.state == 'nogil')
1696         self.visitchildren(node)
1697         self.nogil = was_nogil
1698         return node
1699
1700     def visit_Node(self, node):
1701         if self.env_stack and self.nogil and node.nogil_check:
1702             node.nogil_check(self.env_stack[-1])
1703         self.visitchildren(node)
1704         return node
1705
1706
1707 class TransformBuiltinMethods(EnvTransform):
1708
1709     def visit_SingleAssignmentNode(self, node):
1710         if node.declaration_only:
1711             return None
1712         else:
1713             self.visitchildren(node)
1714             return node
1715
1716     def visit_AttributeNode(self, node):
1717         self.visitchildren(node)
1718         return self.visit_cython_attribute(node)
1719
1720     def visit_NameNode(self, node):
1721         return self.visit_cython_attribute(node)
1722
1723     def visit_cython_attribute(self, node):
1724         attribute = node.as_cython_attribute()
1725         if attribute:
1726             if attribute == u'compiled':
1727                 node = ExprNodes.BoolNode(node.pos, value=True)
1728             elif attribute == u'NULL':
1729                 node = ExprNodes.NullNode(node.pos)
1730             elif attribute in (u'set', u'frozenset'):
1731                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1732                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1733             elif not PyrexTypes.parse_basic_type(attribute):
1734                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1735         return node
1736
1737     def visit_SimpleCallNode(self, node):
1738
1739         # locals builtin
1740         if isinstance(node.function, ExprNodes.NameNode):
1741             if node.function.name == 'locals':
1742                 lenv = self.current_env()
1743                 entry = lenv.lookup_here('locals')
1744                 if entry:
1745                     # not the builtin 'locals'
1746                     return node
1747                 if len(node.args) > 0:
1748                     error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1749                     return node
1750                 pos = node.pos
1751                 items = [ ExprNodes.DictItemNode(pos,
1752                                                  key=ExprNodes.StringNode(pos, value=var),
1753                                                  value=ExprNodes.NameNode(pos, name=var))
1754                           for var in lenv.entries ]
1755                 return ExprNodes.DictNode(pos, key_value_pairs=items)
1756
1757         # cython.foo
1758         function = node.function.as_cython_attribute()
1759         if function:
1760             if function in InterpretCompilerDirectives.unop_method_nodes:
1761                 if len(node.args) != 1:
1762                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1763                 else:
1764                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1765             elif function in InterpretCompilerDirectives.binop_method_nodes:
1766                 if len(node.args) != 2:
1767                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1768                 else:
1769                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1770             elif function == u'cast':
1771                 if len(node.args) != 2:
1772                     error(node.function.pos, u"cast() takes exactly two arguments")
1773                 else:
1774                     type = node.args[0].analyse_as_type(self.current_env())
1775                     if type:
1776                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1777                     else:
1778                         error(node.args[0].pos, "Not a type")
1779             elif function == u'sizeof':
1780                 if len(node.args) != 1:
1781                     error(node.function.pos, u"sizeof() takes exactly one argument")
1782                 else:
1783                     type = node.args[0].analyse_as_type(self.current_env())
1784                     if type:
1785                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1786                     else:
1787                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1788             elif function == 'cmod':
1789                 if len(node.args) != 2:
1790                     error(node.function.pos, u"cmod() takes exactly two arguments")
1791                 else:
1792                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1793                     node.cdivision = True
1794             elif function == 'cdiv':
1795                 if len(node.args) != 2:
1796                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1797                 else:
1798                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1799                     node.cdivision = True
1800             elif function == u'set':
1801                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1802             else:
1803                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1804
1805         self.visitchildren(node)
1806         return node
1807
1808
1809 class DebugTransform(CythonTransform):
1810     """
1811     Create debug information and all functions' visibility to extern in order
1812     to enable debugging.
1813     """
1814
1815     def __init__(self, context, options, result):
1816         super(DebugTransform, self).__init__(context)
1817         self.visited = cython.set()
1818         # our treebuilder and debug output writer
1819         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1820         self.tb = self.context.gdb_debug_outputwriter
1821         #self.c_output_file = options.output_file
1822         self.c_output_file = result.c_file
1823
1824         # Closure support, basically treat nested functions as if the AST were
1825         # never nested
1826         self.nested_funcdefs = []
1827
1828         # tells visit_NameNode whether it should register step-into functions
1829         self.register_stepinto = False
1830
1831     def visit_ModuleNode(self, node):
1832         self.tb.module_name = node.full_module_name
1833         attrs = dict(
1834             module_name=node.full_module_name,
1835             filename=node.pos[0].filename,
1836             c_filename=self.c_output_file)
1837
1838         self.tb.start('Module', attrs)
1839
1840         # serialize functions
1841         self.tb.start('Functions')
1842         # First, serialize functions normally...
1843         self.visitchildren(node)
1844
1845         # ... then, serialize nested functions
1846         for nested_funcdef in self.nested_funcdefs:
1847             self.visit_FuncDefNode(nested_funcdef)
1848
1849         self.register_stepinto = True
1850         self.serialize_modulenode_as_function(node)
1851         self.register_stepinto = False
1852         self.tb.end('Functions')
1853
1854         # 2.3 compatibility. Serialize global variables
1855         self.tb.start('Globals')
1856         entries = {}
1857
1858         for k, v in node.scope.entries.iteritems():
1859             if (v.qualified_name not in self.visited and not
1860                 v.name.startswith('__pyx_') and not
1861                 v.type.is_cfunction and not
1862                 v.type.is_extension_type):
1863                 entries[k]= v
1864
1865         self.serialize_local_variables(entries)
1866         self.tb.end('Globals')
1867         # self.tb.end('Module') # end Module after the line number mapping in
1868         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1869         return node
1870
1871     def visit_FuncDefNode(self, node):
1872         self.visited.add(node.local_scope.qualified_name)
1873
1874         if getattr(node, 'is_wrapper', False):
1875             return node
1876
1877         if self.register_stepinto:
1878             self.nested_funcdefs.append(node)
1879             return node
1880
1881         # node.entry.visibility = 'extern'
1882         if node.py_func is None:
1883             pf_cname = ''
1884         else:
1885             pf_cname = node.py_func.entry.func_cname
1886
1887         attrs = dict(
1888             name=node.entry.name,
1889             cname=node.entry.func_cname,
1890             pf_cname=pf_cname,
1891             qualified_name=node.local_scope.qualified_name,
1892             lineno=str(node.pos[1]))
1893
1894         self.tb.start('Function', attrs=attrs)
1895
1896         self.tb.start('Locals')
1897         self.serialize_local_variables(node.local_scope.entries)
1898         self.tb.end('Locals')
1899
1900         self.tb.start('Arguments')
1901         for arg in node.local_scope.arg_entries:
1902             self.tb.start(arg.name)
1903             self.tb.end(arg.name)
1904         self.tb.end('Arguments')
1905
1906         self.tb.start('StepIntoFunctions')
1907         self.register_stepinto = True
1908         self.visitchildren(node)
1909         self.register_stepinto = False
1910         self.tb.end('StepIntoFunctions')
1911         self.tb.end('Function')
1912
1913         return node
1914
1915     def visit_NameNode(self, node):
1916         if (self.register_stepinto and
1917             node.type.is_cfunction and
1918             getattr(node, 'is_called', False) and
1919             node.entry.func_cname is not None):
1920             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1921             # declared functions are not 'in_cinclude'.
1922             # This means we will list called 'cdef' functions as
1923             # "step into functions", but this is not an issue as they will be
1924             # recognized as Cython functions anyway.
1925             attrs = dict(name=node.entry.func_cname)
1926             self.tb.start('StepIntoFunction', attrs=attrs)
1927             self.tb.end('StepIntoFunction')
1928
1929         self.visitchildren(node)
1930         return node
1931
1932     def serialize_modulenode_as_function(self, node):
1933         """
1934         Serialize the module-level code as a function so the debugger will know
1935         it's a "relevant frame" and it will know where to set the breakpoint
1936         for 'break modulename'.
1937         """
1938         name = node.full_module_name.rpartition('.')[-1]
1939
1940         cname_py2 = 'init' + name
1941         cname_py3 = 'PyInit_' + name
1942
1943         py2_attrs = dict(
1944             name=name,
1945             cname=cname_py2,
1946             pf_cname='',
1947             # Ignore the qualified_name, breakpoints should be set using
1948             # `cy break modulename:lineno` for module-level breakpoints.
1949             qualified_name='',
1950             lineno='1',
1951             is_initmodule_function="True",
1952         )
1953
1954         py3_attrs = dict(py2_attrs, cname=cname_py3)
1955
1956         self._serialize_modulenode_as_function(node, py2_attrs)
1957         self._serialize_modulenode_as_function(node, py3_attrs)
1958
1959     def _serialize_modulenode_as_function(self, node, attrs):
1960         self.tb.start('Function', attrs=attrs)
1961
1962         self.tb.start('Locals')
1963         self.serialize_local_variables(node.scope.entries)
1964         self.tb.end('Locals')
1965
1966         self.tb.start('Arguments')
1967         self.tb.end('Arguments')
1968
1969         self.tb.start('StepIntoFunctions')
1970         self.register_stepinto = True
1971         self.visitchildren(node)
1972         self.register_stepinto = False
1973         self.tb.end('StepIntoFunctions')
1974
1975         self.tb.end('Function')
1976
1977     def serialize_local_variables(self, entries):
1978         for entry in entries.values():
1979             if entry.type.is_pyobject:
1980                 vartype = 'PythonObject'
1981             else:
1982                 vartype = 'CObject'
1983
1984             if entry.from_closure:
1985                 # We're dealing with a closure where a variable from an outer
1986                 # scope is accessed, get it from the scope object.
1987                 cname = '%s->%s' % (Naming.cur_scope_cname,
1988                                     entry.outer_entry.cname)
1989
1990                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1991                                       entry.scope.name,
1992                                       entry.name)
1993             elif entry.in_closure:
1994                 cname = '%s->%s' % (Naming.cur_scope_cname,
1995                                     entry.cname)
1996                 qname = entry.qualified_name
1997             else:
1998                 cname = entry.cname
1999                 qname = entry.qualified_name
2000
2001             if not entry.pos:
2002                 # this happens for variables that are not in the user's code,
2003                 # e.g. for the global __builtins__, __doc__, etc. We can just
2004                 # set the lineno to 0 for those.
2005                 lineno = '0'
2006             else:
2007                 lineno = str(entry.pos[1])
2008
2009             attrs = dict(
2010                 name=entry.name,
2011                 cname=cname,
2012                 qualified_name=qname,
2013                 type=vartype,
2014                 lineno=lineno)
2015
2016             self.tb.start('LocalVar', attrs)
2017             self.tb.end('LocalVar')
2018