Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67
  68 class NormalizeTree(CythonTransform):
  69     """
  70     This transform fixes up a few things after parsing
  71     in order to make the parse tree more suitable for
  72     transforms.
  73
  74     a) After parsing, blocks with only one statement will
  75     be represented by that statement, not by a StatListNode.
  76     When doing transforms this is annoying and inconsistent,
  77     as one cannot in general remove a statement in a consistent
  78     way and so on. This transform wraps any single statements
  79     in a StatListNode containing a single statement.
  80
  81     b) The PassStatNode is a noop and serves no purpose beyond
  82     plugging such one-statement blocks; i.e., once parsed a
  83 `    "pass" can just as well be represented using an empty
  84     StatListNode. This means less special cases to worry about
  85     in subsequent transforms (one always checks to see if a
  86     StatListNode has no children to see if the block is empty).
  87     """
  88
  89     def __init__(self, context):
  90         super(NormalizeTree, self).__init__(context)
  91         self.is_in_statlist = False
  92         self.is_in_expr = False
  93
  94     def visit_ExprNode(self, node):
  95         stacktmp = self.is_in_expr
  96         self.is_in_expr = True
  97         self.visitchildren(node)
  98         self.is_in_expr = stacktmp
  99         return node
 100
 101     def visit_StatNode(self, node, is_listcontainer=False):
 102         stacktmp = self.is_in_statlist
 103         self.is_in_statlist = is_listcontainer
 104         self.visitchildren(node)
 105         self.is_in_statlist = stacktmp
 106         if not self.is_in_statlist and not self.is_in_expr:
 107             return Nodes.StatListNode(pos=node.pos, stats=[node])
 108         else:
 109             return node
 110
 111     def visit_StatListNode(self, node):
 112         self.is_in_statlist = True
 113         self.visitchildren(node)
 114         self.is_in_statlist = False
 115         return node
 116
 117     def visit_ParallelAssignmentNode(self, node):
 118         return self.visit_StatNode(node, True)
 119
 120     def visit_CEnumDefNode(self, node):
 121         return self.visit_StatNode(node, True)
 122
 123     def visit_CStructOrUnionDefNode(self, node):
 124         return self.visit_StatNode(node, True)
 125
 126     # Eliminate PassStatNode
 127     def visit_PassStatNode(self, node):
 128         if not self.is_in_statlist:
 129             return Nodes.StatListNode(pos=node.pos, stats=[])
 130         else:
 131             return []
 132
 133     def visit_CDeclaratorNode(self, node):
 134         return node
 135
 136
 137 class PostParseError(CompileError): pass
 138
 139 # error strings checked by unit tests, so define them
 140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 143 class PostParse(ScopeTrackingTransform):
 144     """
 145     Basic interpretation of the parse tree, as well as validity
 146     checking that can be done on a very basic level on the parse
 147     tree (while still not being a problem with the basic syntax,
 148     as such).
 149
 150     Specifically:
 151     - Default values to cdef assignments are turned into single
 152     assignments following the declaration (everywhere but in class
 153     bodies, where they raise a compile error)
 154
 155     - Interpret some node structures into Python runtime values.
 156     Some nodes take compile-time arguments (currently:
 157     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 158     which should be interpreted. This happens in a general way
 159     and other steps should be taken to ensure validity.
 160
 161     Type arguments cannot be interpreted in this way.
 162
 163     - For __cythonbufferdefaults__ the arguments are checked for
 164     validity.
 165
 166     TemplatedTypeNode has its directives interpreted:
 167     Any first positional argument goes into the "dtype" attribute,
 168     any "ndim" keyword argument goes into the "ndim" attribute and
 169     so on. Also it is checked that the directive combination is valid.
 170     - __cythonbufferdefaults__ attributes are parsed and put into the
 171     type information.
 172
 173     Note: Currently Parsing.py does a lot of interpretation and
 174     reorganization that can be refactored into this transform
 175     if a more pure Abstract Syntax Tree is wanted.
 176     """
 177
 178     def __init__(self, context):
 179         super(PostParse, self).__init__(context)
 180         self.specialattribute_handlers = {
 181             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 182         }
 183
 184     def visit_ModuleNode(self, node):
 185         self.lambda_counter = 1
 186         self.genexpr_counter = 1
 187         return super(PostParse, self).visit_ModuleNode(node)
 188
 189     def visit_LambdaNode(self, node):
 190         # unpack a lambda expression into the corresponding DefNode
 191         lambda_id = self.lambda_counter
 192         self.lambda_counter += 1
 193         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 194         collector = YieldNodeCollector()
 195         collector.visitchildren(node.result_expr)
 196         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 197             body = ExprNodes.YieldExprNode(
 198                 node.result_expr.pos, arg=node.result_expr)
 199             body = Nodes.ExprStatNode(node.result_expr.pos, expr=body)
 200         else:
 201             body = Nodes.ReturnStatNode(
 202                 node.result_expr.pos, value=node.result_expr)
 203         node.def_node = Nodes.DefNode(
 204             node.pos, name=node.name, lambda_name=node.lambda_name,
 205             args=node.args, star_arg=node.star_arg,
 206             starstar_arg=node.starstar_arg,
 207             body=body, doc=None)
 208         self.visitchildren(node)
 209         return node
 210
 211     def visit_GeneratorExpressionNode(self, node):
 212         # unpack a generator expression into the corresponding DefNode
 213         genexpr_id = self.genexpr_counter
 214         self.genexpr_counter += 1
 215         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 216
 217         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 218                                       doc=None,
 219                                       args=[], star_arg=None,
 220                                       starstar_arg=None,
 221                                       body=node.loop)
 222         self.visitchildren(node)
 223         return node
 224
 225     # cdef variables
 226     def handle_bufferdefaults(self, decl):
 227         if not isinstance(decl.default, ExprNodes.DictNode):
 228             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 229         self.scope_node.buffer_defaults_node = decl.default
 230         self.scope_node.buffer_defaults_pos = decl.pos
 231
 232     def visit_CVarDefNode(self, node):
 233         # This assumes only plain names and pointers are assignable on
 234         # declaration. Also, it makes use of the fact that a cdef decl
 235         # must appear before the first use, so we don't have to deal with
 236         # "i = 3; cdef int i = i" and can simply move the nodes around.
 237         try:
 238             self.visitchildren(node)
 239             stats = [node]
 240             newdecls = []
 241             for decl in node.declarators:
 242                 declbase = decl
 243                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 244                     declbase = declbase.base
 245                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 246                     if declbase.default is not None:
 247                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 248                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 249                                 handler = self.specialattribute_handlers.get(decl.name)
 250                                 if handler:
 251                                     if decl is not declbase:
 252                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 253                                     handler(decl)
 254                                     continue # Remove declaration
 255                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 256                         first_assignment = self.scope_type != 'module'
 257                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 258                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 259                             rhs=declbase.default, first=first_assignment))
 260                         declbase.default = None
 261                 newdecls.append(decl)
 262             node.declarators = newdecls
 263             return stats
 264         except PostParseError, e:
 265             # An error in a cdef clause is ok, simply remove the declaration
 266             # and try to move on to report more errors
 267             self.context.nonfatal_error(e)
 268             return None
 269
 270     # Split parallel assignments (a,b = b,a) into separate partial
 271     # assignments that are executed rhs-first using temps.  This
 272     # restructuring must be applied before type analysis so that known
 273     # types on rhs and lhs can be matched directly.  It is required in
 274     # the case that the types cannot be coerced to a Python type in
 275     # order to assign from a tuple.
 276
 277     def visit_SingleAssignmentNode(self, node):
 278         self.visitchildren(node)
 279         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 280
 281     def visit_CascadedAssignmentNode(self, node):
 282         self.visitchildren(node)
 283         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 284
 285     def _visit_assignment_node(self, node, expr_list):
 286         """Flatten parallel assignments into separate single
 287         assignments or cascaded assignments.
 288         """
 289         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 290             # no parallel assignments => nothing to do
 291             return node
 292
 293         expr_list_list = []
 294         flatten_parallel_assignments(expr_list, expr_list_list)
 295         temp_refs = []
 296         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 297
 298         nodes = []
 299         for expr_list in expr_list_list:
 300             lhs_list = expr_list[:-1]
 301             rhs = expr_list[-1]
 302             if len(lhs_list) == 1:
 303                 node = Nodes.SingleAssignmentNode(rhs.pos,
 304                     lhs = lhs_list[0], rhs = rhs)
 305             else:
 306                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 307                     lhs_list = lhs_list, rhs = rhs)
 308             nodes.append(node)
 309
 310         if len(nodes) == 1:
 311             assign_node = nodes[0]
 312         else:
 313             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 314
 315         if temp_refs:
 316             duplicates_and_temps = [ (temp.expression, temp)
 317                                      for temp in temp_refs ]
 318             sort_common_subsequences(duplicates_and_temps)
 319             for _, temp_ref in duplicates_and_temps[::-1]:
 320                 assign_node = LetNode(temp_ref, assign_node)
 321
 322         return assign_node
 323
 324 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 325     """Replace rhs items by LetRefNodes if they appear more than once.
 326     Creates a sequence of LetRefNodes that set up the required temps
 327     and appends them to ref_node_sequence.  The input list is modified
 328     in-place.
 329     """
 330     seen_nodes = cython.set()
 331     ref_nodes = {}
 332     def find_duplicates(node):
 333         if node.is_literal or node.is_name:
 334             # no need to replace those; can't include attributes here
 335             # as their access is not necessarily side-effect free
 336             return
 337         if node in seen_nodes:
 338             if node not in ref_nodes:
 339                 ref_node = LetRefNode(node)
 340                 ref_nodes[node] = ref_node
 341                 ref_node_sequence.append(ref_node)
 342         else:
 343             seen_nodes.add(node)
 344             if node.is_sequence_constructor:
 345                 for item in node.args:
 346                     find_duplicates(item)
 347
 348     for expr_list in expr_list_list:
 349         rhs = expr_list[-1]
 350         find_duplicates(rhs)
 351     if not ref_nodes:
 352         return
 353
 354     def substitute_nodes(node):
 355         if node in ref_nodes:
 356             return ref_nodes[node]
 357         elif node.is_sequence_constructor:
 358             node.args = list(map(substitute_nodes, node.args))
 359         return node
 360
 361     # replace nodes inside of the common subexpressions
 362     for node in ref_nodes:
 363         if node.is_sequence_constructor:
 364             node.args = list(map(substitute_nodes, node.args))
 365
 366     # replace common subexpressions on all rhs items
 367     for expr_list in expr_list_list:
 368         expr_list[-1] = substitute_nodes(expr_list[-1])
 369
 370 def sort_common_subsequences(items):
 371     """Sort items/subsequences so that all items and subsequences that
 372     an item contains appear before the item itself.  This is needed
 373     because each rhs item must only be evaluated once, so its value
 374     must be evaluated first and then reused when packing sequences
 375     that contain it.
 376
 377     This implies a partial order, and the sort must be stable to
 378     preserve the original order as much as possible, so we use a
 379     simple insertion sort (which is very fast for short sequences, the
 380     normal case in practice).
 381     """
 382     def contains(seq, x):
 383         for item in seq:
 384             if item is x:
 385                 return True
 386             elif item.is_sequence_constructor and contains(item.args, x):
 387                 return True
 388         return False
 389     def lower_than(a,b):
 390         return b.is_sequence_constructor and contains(b.args, a)
 391
 392     for pos, item in enumerate(items):
 393         key = item[1] # the ResultRefNode which has already been injected into the sequences
 394         new_pos = pos
 395         for i in xrange(pos-1, -1, -1):
 396             if lower_than(key, items[i][0]):
 397                 new_pos = i
 398         if new_pos != pos:
 399             for i in xrange(pos, new_pos, -1):
 400                 items[i] = items[i-1]
 401             items[new_pos] = item
 402
 403 def flatten_parallel_assignments(input, output):
 404     #  The input is a list of expression nodes, representing the LHSs
 405     #  and RHS of one (possibly cascaded) assignment statement.  For
 406     #  sequence constructors, rearranges the matching parts of both
 407     #  sides into a list of equivalent assignments between the
 408     #  individual elements.  This transformation is applied
 409     #  recursively, so that nested structures get matched as well.
 410     rhs = input[-1]
 411     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 412         output.append(input)
 413         return
 414
 415     complete_assignments = []
 416
 417     rhs_size = len(rhs.args)
 418     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 419     starred_assignments = []
 420     for lhs in input[:-1]:
 421         if not lhs.is_sequence_constructor:
 422             if lhs.is_starred:
 423                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 424             complete_assignments.append(lhs)
 425             continue
 426         lhs_size = len(lhs.args)
 427         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 428         if starred_targets > 1:
 429             error(lhs.pos, "more than 1 starred expression in assignment")
 430             output.append([lhs,rhs])
 431             continue
 432         elif lhs_size - starred_targets > rhs_size:
 433             error(lhs.pos, "need more than %d value%s to unpack"
 434                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 435             output.append([lhs,rhs])
 436             continue
 437         elif starred_targets:
 438             map_starred_assignment(lhs_targets, starred_assignments,
 439                                    lhs.args, rhs.args)
 440         elif lhs_size < rhs_size:
 441             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 442                   % (lhs_size, rhs_size))
 443             output.append([lhs,rhs])
 444             continue
 445         else:
 446             for targets, expr in zip(lhs_targets, lhs.args):
 447                 targets.append(expr)
 448
 449     if complete_assignments:
 450         complete_assignments.append(rhs)
 451         output.append(complete_assignments)
 452
 453     # recursively flatten partial assignments
 454     for cascade, rhs in zip(lhs_targets, rhs.args):
 455         if cascade:
 456             cascade.append(rhs)
 457             flatten_parallel_assignments(cascade, output)
 458
 459     # recursively flatten starred assignments
 460     for cascade in starred_assignments:
 461         if cascade[0].is_sequence_constructor:
 462             flatten_parallel_assignments(cascade, output)
 463         else:
 464             output.append(cascade)
 465
 466 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 467     # Appends the fixed-position LHS targets to the target list that
 468     # appear left and right of the starred argument.
 469     #
 470     # The starred_assignments list receives a new tuple
 471     # (lhs_target, rhs_values_list) that maps the remaining arguments
 472     # (those that match the starred target) to a list.
 473
 474     # left side of the starred target
 475     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 476         if expr.is_starred:
 477             starred = i
 478             lhs_remaining = len(lhs_args) - i - 1
 479             break
 480         targets.append(expr)
 481     else:
 482         raise InternalError("no starred arg found when splitting starred assignment")
 483
 484     # right side of the starred target
 485     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 486                                             lhs_args[-lhs_remaining:])):
 487         targets.append(expr)
 488
 489     # the starred target itself, must be assigned a (potentially empty) list
 490     target = lhs_args[starred].target # unpack starred node
 491     starred_rhs = rhs_args[starred:]
 492     if lhs_remaining:
 493         starred_rhs = starred_rhs[:-lhs_remaining]
 494     if starred_rhs:
 495         pos = starred_rhs[0].pos
 496     else:
 497         pos = target.pos
 498     starred_assignments.append([
 499         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 500
 501
 502 class PxdPostParse(CythonTransform, SkipDeclarations):
 503     """
 504     Basic interpretation/validity checking that should only be
 505     done on pxd trees.
 506
 507     A lot of this checking currently happens in the parser; but
 508     what is listed below happens here.
 509
 510     - "def" functions are let through only if they fill the
 511     getbuffer/releasebuffer slots
 512
 513     - cdef functions are let through only if they are on the
 514     top level and are declared "inline"
 515     """
 516     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 517     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 518
 519     def __call__(self, node):
 520         self.scope_type = 'pxd'
 521         return super(PxdPostParse, self).__call__(node)
 522
 523     def visit_CClassDefNode(self, node):
 524         old = self.scope_type
 525         self.scope_type = 'cclass'
 526         self.visitchildren(node)
 527         self.scope_type = old
 528         return node
 529
 530     def visit_FuncDefNode(self, node):
 531         # FuncDefNode always come with an implementation (without
 532         # an imp they are CVarDefNodes..)
 533         err = self.ERR_INLINE_ONLY
 534
 535         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 536             and node.name in ('__getbuffer__', '__releasebuffer__')):
 537             err = None # allow these slots
 538
 539         if isinstance(node, Nodes.CFuncDefNode):
 540             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 541                 node.inline_in_pxd = True
 542                 if node.visibility != 'private':
 543                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 544                 elif node.api:
 545                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 546                 else:
 547                     err = None # allow inline function
 548             else:
 549                 err = self.ERR_INLINE_ONLY
 550
 551         if err:
 552             self.context.nonfatal_error(PostParseError(node.pos, err))
 553             return None
 554         else:
 555             return node
 556
 557 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 558     """
 559     After parsing, directives can be stored in a number of places:
 560     - #cython-comments at the top of the file (stored in ModuleNode)
 561     - Command-line arguments overriding these
 562     - @cython.directivename decorators
 563     - with cython.directivename: statements
 564
 565     This transform is responsible for interpreting these various sources
 566     and store the directive in two ways:
 567     - Set the directives attribute of the ModuleNode for global directives.
 568     - Use a CompilerDirectivesNode to override directives for a subtree.
 569
 570     (The first one is primarily to not have to modify with the tree
 571     structure, so that ModuleNode stay on top.)
 572
 573     The directives are stored in dictionaries from name to value in effect.
 574     Each such dictionary is always filled in for all possible directives,
 575     using default values where no value is given by the user.
 576
 577     The available directives are controlled in Options.py.
 578
 579     Note that we have to run this prior to analysis, and so some minor
 580     duplication of functionality has to occur: We manually track cimports
 581     and which names the "cython" module may have been imported to.
 582     """
 583     unop_method_nodes = {
 584         'typeof': ExprNodes.TypeofNode,
 585
 586         'operator.address': ExprNodes.AmpersandNode,
 587         'operator.dereference': ExprNodes.DereferenceNode,
 588         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 589         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 590         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 591         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 592
 593         # For backwards compatability.
 594         'address': ExprNodes.AmpersandNode,
 595     }
 596
 597     binop_method_nodes = {
 598         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 599     }
 600
 601     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 602                                   'cast', 'pointer', 'compiled', 'NULL'])
 603     special_methods.update(unop_method_nodes.keys())
 604
 605     def __init__(self, context, compilation_directive_defaults):
 606         super(InterpretCompilerDirectives, self).__init__(context)
 607         self.compilation_directive_defaults = {}
 608         for key, value in compilation_directive_defaults.items():
 609             self.compilation_directive_defaults[unicode(key)] = value
 610         self.cython_module_names = cython.set()
 611         self.directive_names = {}
 612
 613     def check_directive_scope(self, pos, directive, scope):
 614         legal_scopes = Options.directive_scopes.get(directive, None)
 615         if legal_scopes and scope not in legal_scopes:
 616             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 617                                         'is not allowed in %s scope' % (directive, scope)))
 618             return False
 619         else:
 620             return True
 621
 622     # Set up processing and handle the cython: comments.
 623     def visit_ModuleNode(self, node):
 624         for key, value in node.directive_comments.items():
 625             if not self.check_directive_scope(node.pos, key, 'module'):
 626                 self.wrong_scope_error(node.pos, key, 'module')
 627                 del node.directive_comments[key]
 628
 629         directives = copy.copy(Options.directive_defaults)
 630         directives.update(self.compilation_directive_defaults)
 631         directives.update(node.directive_comments)
 632         self.directives = directives
 633         node.directives = directives
 634         self.visitchildren(node)
 635         node.cython_module_names = self.cython_module_names
 636         return node
 637
 638     # The following four functions track imports and cimports that
 639     # begin with "cython"
 640     def is_cython_directive(self, name):
 641         return (name in Options.directive_types or
 642                 name in self.special_methods or
 643                 PyrexTypes.parse_basic_type(name))
 644
 645     def visit_CImportStatNode(self, node):
 646         if node.module_name == u"cython":
 647             self.cython_module_names.add(node.as_name or u"cython")
 648         elif node.module_name.startswith(u"cython."):
 649             if node.as_name:
 650                 self.directive_names[node.as_name] = node.module_name[7:]
 651             else:
 652                 self.cython_module_names.add(u"cython")
 653             # if this cimport was a compiler directive, we don't
 654             # want to leave the cimport node sitting in the tree
 655             return None
 656         return node
 657
 658     def visit_FromCImportStatNode(self, node):
 659         if (node.module_name == u"cython") or \
 660                node.module_name.startswith(u"cython."):
 661             submodule = (node.module_name + u".")[7:]
 662             newimp = []
 663             for pos, name, as_name, kind in node.imported_names:
 664                 full_name = submodule + name
 665                 if self.is_cython_directive(full_name):
 666                     if as_name is None:
 667                         as_name = full_name
 668                     self.directive_names[as_name] = full_name
 669                     if kind is not None:
 670                         self.context.nonfatal_error(PostParseError(pos,
 671                             "Compiler directive imports must be plain imports"))
 672                 else:
 673                     newimp.append((pos, name, as_name, kind))
 674             if not newimp:
 675                 return None
 676             node.imported_names = newimp
 677         return node
 678
 679     def visit_FromImportStatNode(self, node):
 680         if (node.module.module_name.value == u"cython") or \
 681                node.module.module_name.value.startswith(u"cython."):
 682             submodule = (node.module.module_name.value + u".")[7:]
 683             newimp = []
 684             for name, name_node in node.items:
 685                 full_name = submodule + name
 686                 if self.is_cython_directive(full_name):
 687                     self.directive_names[name_node.name] = full_name
 688                 else:
 689                     newimp.append((name, name_node))
 690             if not newimp:
 691                 return None
 692             node.items = newimp
 693         return node
 694
 695     def visit_SingleAssignmentNode(self, node):
 696         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 697                 node.rhs.module_name.value == u'cython'):
 698             node = Nodes.CImportStatNode(node.pos,
 699                                          module_name = u'cython',
 700                                          as_name = node.lhs.name)
 701             self.visit_CImportStatNode(node)
 702         else:
 703             self.visitchildren(node)
 704         return node
 705
 706     def visit_NameNode(self, node):
 707         if node.name in self.cython_module_names:
 708             node.is_cython_module = True
 709         else:
 710             node.cython_attribute = self.directive_names.get(node.name)
 711         return node
 712
 713     def try_to_parse_directives(self, node):
 714         # If node is the contents of an directive (in a with statement or
 715         # decorator), returns a list of (directivename, value) pairs.
 716         # Otherwise, returns None
 717         if isinstance(node, ExprNodes.CallNode):
 718             self.visit(node.function)
 719             optname = node.function.as_cython_attribute()
 720             if optname:
 721                 directivetype = Options.directive_types.get(optname)
 722                 if directivetype:
 723                     args, kwds = node.explicit_args_kwds()
 724                     directives = []
 725                     key_value_pairs = []
 726                     if kwds is not None and directivetype is not dict:
 727                         for keyvalue in kwds.key_value_pairs:
 728                             key, value = keyvalue
 729                             sub_optname = "%s.%s" % (optname, key.value)
 730                             if Options.directive_types.get(sub_optname):
 731                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 732                             else:
 733                                 key_value_pairs.append(keyvalue)
 734                         if not key_value_pairs:
 735                             kwds = None
 736                         else:
 737                             kwds.key_value_pairs = key_value_pairs
 738                         if directives and not kwds and not args:
 739                             return directives
 740                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 741                     return directives
 742         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 743             self.visit(node)
 744             optname = node.as_cython_attribute()
 745             if optname:
 746                 directivetype = Options.directive_types.get(optname)
 747                 if directivetype is bool:
 748                     return [(optname, True)]
 749                 elif directivetype is None:
 750                     return [(optname, None)]
 751                 else:
 752                     raise PostParseError(
 753                         node.pos, "The '%s' directive should be used as a function call." % optname)
 754         return None
 755
 756     def try_to_parse_directive(self, optname, args, kwds, pos):
 757         directivetype = Options.directive_types.get(optname)
 758         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 759             return optname, Options.directive_defaults[optname]
 760         elif directivetype is bool:
 761             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 762                 raise PostParseError(pos,
 763                     'The %s directive takes one compile-time boolean argument' % optname)
 764             return (optname, args[0].value)
 765         elif directivetype is str:
 766             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 767                                                                               ExprNodes.UnicodeNode)):
 768                 raise PostParseError(pos,
 769                     'The %s directive takes one compile-time string argument' % optname)
 770             return (optname, str(args[0].value))
 771         elif directivetype is dict:
 772             if len(args) != 0:
 773                 raise PostParseError(pos,
 774                     'The %s directive takes no prepositional arguments' % optname)
 775             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 776         elif directivetype is list:
 777             if kwds and len(kwds) != 0:
 778                 raise PostParseError(pos,
 779                     'The %s directive takes no keyword arguments' % optname)
 780             return optname, [ str(arg.value) for arg in args ]
 781         else:
 782             assert False
 783
 784     def visit_with_directives(self, body, directives):
 785         olddirectives = self.directives
 786         newdirectives = copy.copy(olddirectives)
 787         newdirectives.update(directives)
 788         self.directives = newdirectives
 789         assert isinstance(body, Nodes.StatListNode), body
 790         retbody = self.visit_Node(body)
 791         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 792                                                  directives=newdirectives)
 793         self.directives = olddirectives
 794         return directive
 795
 796     # Handle decorators
 797     def visit_FuncDefNode(self, node):
 798         directives = self._extract_directives(node, 'function')
 799         if not directives:
 800             return self.visit_Node(node)
 801         body = Nodes.StatListNode(node.pos, stats=[node])
 802         return self.visit_with_directives(body, directives)
 803
 804     def visit_CVarDefNode(self, node):
 805         if not node.decorators:
 806             return node
 807         for dec in node.decorators:
 808             for directive in self.try_to_parse_directives(dec.decorator) or ():
 809                 if directive is not None and directive[0] == u'locals':
 810                     node.directive_locals = directive[1]
 811                 else:
 812                     self.context.nonfatal_error(PostParseError(dec.pos,
 813                         "Cdef functions can only take cython.locals() decorator."))
 814         return node
 815
 816     def visit_CClassDefNode(self, node):
 817         directives = self._extract_directives(node, 'cclass')
 818         if not directives:
 819             return self.visit_Node(node)
 820         body = Nodes.StatListNode(node.pos, stats=[node])
 821         return self.visit_with_directives(body, directives)
 822
 823     def visit_PyClassDefNode(self, node):
 824         directives = self._extract_directives(node, 'class')
 825         if not directives:
 826             return self.visit_Node(node)
 827         body = Nodes.StatListNode(node.pos, stats=[node])
 828         return self.visit_with_directives(body, directives)
 829
 830     def _extract_directives(self, node, scope_name):
 831         if not node.decorators:
 832             return {}
 833         # Split the decorators into two lists -- real decorators and directives
 834         directives = []
 835         realdecs = []
 836         for dec in node.decorators:
 837             new_directives = self.try_to_parse_directives(dec.decorator)
 838             if new_directives is not None:
 839                 for directive in new_directives:
 840                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 841                         directives.append(directive)
 842             else:
 843                 realdecs.append(dec)
 844         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 845             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 846         else:
 847             node.decorators = realdecs
 848         # merge or override repeated directives
 849         optdict = {}
 850         directives.reverse() # Decorators coming first take precedence
 851         for directive in directives:
 852             name, value = directive
 853             if name in optdict:
 854                 old_value = optdict[name]
 855                 # keywords and arg lists can be merged, everything
 856                 # else overrides completely
 857                 if isinstance(old_value, dict):
 858                     old_value.update(value)
 859                 elif isinstance(old_value, list):
 860                     old_value.extend(value)
 861                 else:
 862                     optdict[name] = value
 863             else:
 864                 optdict[name] = value
 865         return optdict
 866
 867     # Handle with statements
 868     def visit_WithStatNode(self, node):
 869         directive_dict = {}
 870         for directive in self.try_to_parse_directives(node.manager) or []:
 871             if directive is not None:
 872                 if node.target is not None:
 873                     self.context.nonfatal_error(
 874                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 875                 else:
 876                     name, value = directive
 877                     if name == 'nogil':
 878                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 879                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 880                         return self.visit_Node(node)
 881                     if self.check_directive_scope(node.pos, name, 'with statement'):
 882                         directive_dict[name] = value
 883         if directive_dict:
 884             return self.visit_with_directives(node.body, directive_dict)
 885         return self.visit_Node(node)
 886
 887 class WithTransform(CythonTransform, SkipDeclarations):
 888
 889     # EXCINFO is manually set to a variable that contains
 890     # the exc_info() tuple that can be generated by the enclosing except
 891     # statement.
 892     template_without_target = TreeFragment(u"""
 893         MGR = EXPR
 894         EXIT = MGR.__exit__
 895         MGR.__enter__()
 896         EXC = True
 897         try:
 898             try:
 899                 EXCINFO = None
 900                 BODY
 901             except:
 902                 EXC = False
 903                 if not EXIT(*EXCINFO):
 904                     raise
 905         finally:
 906             if EXC:
 907                 EXIT(None, None, None)
 908     """, temps=[u'MGR', u'EXC', u"EXIT"],
 909     pipeline=[NormalizeTree(None)])
 910
 911     template_with_target = TreeFragment(u"""
 912         MGR = EXPR
 913         EXIT = MGR.__exit__
 914         VALUE = MGR.__enter__()
 915         EXC = True
 916         try:
 917             try:
 918                 EXCINFO = None
 919                 TARGET = VALUE
 920                 BODY
 921             except:
 922                 EXC = False
 923                 if not EXIT(*EXCINFO):
 924                     raise
 925         finally:
 926             if EXC:
 927                 EXIT(None, None, None)
 928             MGR = EXIT = VALUE = EXC = None
 929
 930     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 931     pipeline=[NormalizeTree(None)])
 932
 933     def visit_WithStatNode(self, node):
 934         # TODO: Cleanup badly needed
 935         TemplateTransform.temp_name_counter += 1
 936         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 937
 938         self.visitchildren(node, ['body'])
 939         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 940         if node.target is not None:
 941             result = self.template_with_target.substitute({
 942                 u'EXPR' : node.manager,
 943                 u'BODY' : node.body,
 944                 u'TARGET' : node.target,
 945                 u'EXCINFO' : excinfo_temp
 946                 }, pos=node.pos)
 947         else:
 948             result = self.template_without_target.substitute({
 949                 u'EXPR' : node.manager,
 950                 u'BODY' : node.body,
 951                 u'EXCINFO' : excinfo_temp
 952                 }, pos=node.pos)
 953
 954         # Set except excinfo target to EXCINFO
 955         try_except = result.stats[-1].body.stats[-1]
 956         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 957 #            excinfo_temp.ref(node.pos))
 958
 959 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 960 #            node.pos, temps=[excinfo_temp], body=try_except)
 961
 962         return result
 963
 964     def visit_ExprNode(self, node):
 965         # With statements are never inside expressions.
 966         return node
 967
 968
 969 class DecoratorTransform(CythonTransform, SkipDeclarations):
 970
 971     def visit_DefNode(self, func_node):
 972         self.visitchildren(func_node)
 973         if not func_node.decorators:
 974             return func_node
 975         return self._handle_decorators(
 976             func_node, func_node.name)
 977
 978     def visit_CClassDefNode(self, class_node):
 979         # This doesn't currently work, so it's disabled.
 980         #
 981         # Problem: assignments to cdef class names do not work.  They
 982         # would require an additional check anyway, as the extension
 983         # type must not change its C type, so decorators cannot
 984         # replace an extension type, just alter it and return it.
 985
 986         self.visitchildren(class_node)
 987         if not class_node.decorators:
 988             return class_node
 989         error(class_node.pos,
 990               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 991         return class_node
 992         #return self._handle_decorators(
 993         #    class_node, class_node.class_name)
 994
 995     def visit_ClassDefNode(self, class_node):
 996         self.visitchildren(class_node)
 997         if not class_node.decorators:
 998             return class_node
 999         return self._handle_decorators(
1000             class_node, class_node.name)
1001
1002     def _handle_decorators(self, node, name):
1003         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1004         for decorator in node.decorators[::-1]:
1005             decorator_result = ExprNodes.SimpleCallNode(
1006                 decorator.pos,
1007                 function = decorator.decorator,
1008                 args = [decorator_result])
1009
1010         name_node = ExprNodes.NameNode(node.pos, name = name)
1011         reassignment = Nodes.SingleAssignmentNode(
1012             node.pos,
1013             lhs = name_node,
1014             rhs = decorator_result)
1015         return [node, reassignment]
1016
1017
1018 class AnalyseDeclarationsTransform(CythonTransform):
1019
1020     basic_property = TreeFragment(u"""
1021 property NAME:
1022     def __get__(self):
1023         return ATTR
1024     def __set__(self, value):
1025         ATTR = value
1026     """, level='c_class')
1027     basic_pyobject_property = TreeFragment(u"""
1028 property NAME:
1029     def __get__(self):
1030         return ATTR
1031     def __set__(self, value):
1032         ATTR = value
1033     def __del__(self):
1034         ATTR = None
1035     """, level='c_class')
1036     basic_property_ro = TreeFragment(u"""
1037 property NAME:
1038     def __get__(self):
1039         return ATTR
1040     """, level='c_class')
1041
1042     def __call__(self, root):
1043         self.env_stack = [root.scope]
1044         # needed to determine if a cdef var is declared after it's used.
1045         self.seen_vars_stack = []
1046         return super(AnalyseDeclarationsTransform, self).__call__(root)
1047
1048     def visit_NameNode(self, node):
1049         self.seen_vars_stack[-1].add(node.name)
1050         return node
1051
1052     def visit_ModuleNode(self, node):
1053         self.seen_vars_stack.append(cython.set())
1054         node.analyse_declarations(self.env_stack[-1])
1055         self.visitchildren(node)
1056         self.seen_vars_stack.pop()
1057         return node
1058
1059     def visit_LambdaNode(self, node):
1060         node.analyse_declarations(self.env_stack[-1])
1061         self.visitchildren(node)
1062         return node
1063
1064     def visit_ClassDefNode(self, node):
1065         self.env_stack.append(node.scope)
1066         self.visitchildren(node)
1067         self.env_stack.pop()
1068         return node
1069
1070     def visit_CClassDefNode(self, node):
1071         node = self.visit_ClassDefNode(node)
1072         if node.scope and node.scope.implemented:
1073             stats = []
1074             for entry in node.scope.var_entries:
1075                 if entry.needs_property:
1076                     property = self.create_Property(entry)
1077                     property.analyse_declarations(node.scope)
1078                     self.visit(property)
1079                     stats.append(property)
1080             if stats:
1081                 node.body.stats += stats
1082         return node
1083
1084     def visit_FuncDefNode(self, node):
1085         self.seen_vars_stack.append(cython.set())
1086         lenv = node.local_scope
1087         node.body.analyse_control_flow(lenv) # this will be totally refactored
1088         node.declare_arguments(lenv)
1089         for var, type_node in node.directive_locals.items():
1090             if not lenv.lookup_here(var):   # don't redeclare args
1091                 type = type_node.analyse_as_type(lenv)
1092                 if type:
1093                     lenv.declare_var(var, type, type_node.pos)
1094                 else:
1095                     error(type_node.pos, "Not a type")
1096         node.body.analyse_declarations(lenv)
1097         self.env_stack.append(lenv)
1098         self.visitchildren(node)
1099         self.env_stack.pop()
1100         self.seen_vars_stack.pop()
1101         return node
1102
1103     def visit_ScopedExprNode(self, node):
1104         env = self.env_stack[-1]
1105         node.analyse_declarations(env)
1106         # the node may or may not have a local scope
1107         if node.has_local_scope:
1108             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1109             self.env_stack.append(node.expr_scope)
1110             node.analyse_scoped_declarations(node.expr_scope)
1111             self.visitchildren(node)
1112             self.env_stack.pop()
1113             self.seen_vars_stack.pop()
1114         else:
1115             node.analyse_scoped_declarations(env)
1116             self.visitchildren(node)
1117         return node
1118
1119     def visit_TempResultFromStatNode(self, node):
1120         self.visitchildren(node)
1121         node.analyse_declarations(self.env_stack[-1])
1122         return node
1123
1124     # Some nodes are no longer needed after declaration
1125     # analysis and can be dropped. The analysis was performed
1126     # on these nodes in a seperate recursive process from the
1127     # enclosing function or module, so we can simply drop them.
1128     def visit_CDeclaratorNode(self, node):
1129         # necessary to ensure that all CNameDeclaratorNodes are visited.
1130         self.visitchildren(node)
1131         return node
1132
1133     def visit_CTypeDefNode(self, node):
1134         return node
1135
1136     def visit_CBaseTypeNode(self, node):
1137         return None
1138
1139     def visit_CEnumDefNode(self, node):
1140         if node.visibility == 'public':
1141             return node
1142         else:
1143             return None
1144
1145     def visit_CStructOrUnionDefNode(self, node):
1146         return None
1147
1148     def visit_CNameDeclaratorNode(self, node):
1149         if node.name in self.seen_vars_stack[-1]:
1150             entry = self.env_stack[-1].lookup(node.name)
1151             if entry is None or entry.visibility != 'extern':
1152                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1153         self.visitchildren(node)
1154         return node
1155
1156     def visit_CVarDefNode(self, node):
1157         # to ensure all CNameDeclaratorNodes are visited.
1158         self.visitchildren(node)
1159         return None
1160
1161     def create_Property(self, entry):
1162         if entry.visibility == 'public':
1163             if entry.type.is_pyobject:
1164                 template = self.basic_pyobject_property
1165             else:
1166                 template = self.basic_property
1167         elif entry.visibility == 'readonly':
1168             template = self.basic_property_ro
1169         property = template.substitute({
1170                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1171                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1172                                                  attribute=entry.name),
1173             }, pos=entry.pos).stats[0]
1174         property.name = entry.name
1175         # ---------------------------------------
1176         # XXX This should go to AutoDocTransforms
1177         # ---------------------------------------
1178         if (Options.docstrings and
1179             self.current_directives['embedsignature']):
1180             attr_name = entry.name
1181             type_name = entry.type.declaration_code("", for_display=1)
1182             default_value = ''
1183             if not entry.type.is_pyobject:
1184                 type_name = "'%s'" % type_name
1185             elif entry.type.is_extension_type:
1186                 type_name = entry.type.module_name + '.' + type_name
1187             if entry.init is not None:
1188                 default_value = ' = ' + entry.init
1189             elif entry.init_to_none:
1190                 default_value = ' = ' + repr(None)
1191             docstring = attr_name + ': ' + type_name + default_value
1192             property.doc = EncodedString(docstring)
1193         # ---------------------------------------
1194         return property
1195
1196 class AnalyseExpressionsTransform(CythonTransform):
1197
1198     def visit_ModuleNode(self, node):
1199         node.scope.infer_types()
1200         node.body.analyse_expressions(node.scope)
1201         self.visitchildren(node)
1202         return node
1203
1204     def visit_FuncDefNode(self, node):
1205         node.local_scope.infer_types()
1206         node.body.analyse_expressions(node.local_scope)
1207         self.visitchildren(node)
1208         return node
1209
1210     def visit_ScopedExprNode(self, node):
1211         if node.has_local_scope:
1212             node.expr_scope.infer_types()
1213             node.analyse_scoped_expressions(node.expr_scope)
1214         self.visitchildren(node)
1215         return node
1216
1217 class ExpandInplaceOperators(EnvTransform):
1218
1219     def visit_InPlaceAssignmentNode(self, node):
1220         lhs = node.lhs
1221         rhs = node.rhs
1222         if lhs.type.is_cpp_class:
1223             # No getting around this exact operator here.
1224             return node
1225         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1226             # There is code to handle this case.
1227             return node
1228
1229         env = self.current_env()
1230         def side_effect_free_reference(node, setting=False):
1231             if isinstance(node, ExprNodes.NameNode):
1232                 return node, []
1233             elif node.type.is_pyobject and not setting:
1234                 node = LetRefNode(node)
1235                 return node, [node]
1236             elif isinstance(node, ExprNodes.IndexNode):
1237                 if node.is_buffer_access:
1238                     raise ValueError, "Buffer access"
1239                 base, temps = side_effect_free_reference(node.base)
1240                 index = LetRefNode(node.index)
1241                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1242             elif isinstance(node, ExprNodes.AttributeNode):
1243                 obj, temps = side_effect_free_reference(node.obj)
1244                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1245             else:
1246                 node = LetRefNode(node)
1247                 return node, [node]
1248         try:
1249             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1250         except ValueError:
1251             return node
1252         dup = lhs.__class__(**lhs.__dict__)
1253         binop = ExprNodes.binop_node(node.pos,
1254                                      operator = node.operator,
1255                                      operand1 = dup,
1256                                      operand2 = rhs,
1257                                      inplace=True)
1258         # Manually analyse types for new node.
1259         lhs.analyse_target_types(env)
1260         dup.analyse_types(env)
1261         binop.analyse_operation(env)
1262         node = Nodes.SingleAssignmentNode(
1263             node.pos,
1264             lhs = lhs,
1265             rhs=binop.coerce_to(lhs.type, env))
1266         # Use LetRefNode to avoid side effects.
1267         let_ref_nodes.reverse()
1268         for t in let_ref_nodes:
1269             node = LetNode(t, node)
1270         return node
1271
1272     def visit_ExprNode(self, node):
1273         # In-place assignments can't happen within an expression.
1274         return node
1275
1276
1277 class AlignFunctionDefinitions(CythonTransform):
1278     """
1279     This class takes the signatures from a .pxd file and applies them to
1280     the def methods in a .py file.
1281     """
1282
1283     def visit_ModuleNode(self, node):
1284         self.scope = node.scope
1285         self.directives = node.directives
1286         self.visitchildren(node)
1287         return node
1288
1289     def visit_PyClassDefNode(self, node):
1290         pxd_def = self.scope.lookup(node.name)
1291         if pxd_def:
1292             if pxd_def.is_cclass:
1293                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1294             else:
1295                 error(node.pos, "'%s' redeclared" % node.name)
1296                 error(pxd_def.pos, "previous declaration here")
1297                 return None
1298         else:
1299             return node
1300
1301     def visit_CClassDefNode(self, node, pxd_def=None):
1302         if pxd_def is None:
1303             pxd_def = self.scope.lookup(node.class_name)
1304         if pxd_def:
1305             outer_scope = self.scope
1306             self.scope = pxd_def.type.scope
1307         self.visitchildren(node)
1308         if pxd_def:
1309             self.scope = outer_scope
1310         return node
1311
1312     def visit_DefNode(self, node):
1313         pxd_def = self.scope.lookup(node.name)
1314         if pxd_def:
1315             if not pxd_def.is_cfunction:
1316                 error(node.pos, "'%s' redeclared" % node.name)
1317                 error(pxd_def.pos, "previous declaration here")
1318                 return None
1319             node = node.as_cfunction(pxd_def)
1320         elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1321             node = node.as_cfunction(scope=self.scope)
1322         # Enable this when internal def functions are allowed.
1323         # self.visitchildren(node)
1324         return node
1325
1326
1327 class YieldNodeCollector(TreeVisitor):
1328
1329     def __init__(self):
1330         super(YieldNodeCollector, self).__init__()
1331         self.yields = []
1332         self.returns = []
1333         self.has_return_value = False
1334
1335     def visit_Node(self, node):
1336         return self.visitchildren(node)
1337
1338     def visit_YieldExprNode(self, node):
1339         if self.has_return_value:
1340             error(node.pos, "'yield' outside function")
1341         self.yields.append(node)
1342         self.visitchildren(node)
1343
1344     def visit_ReturnStatNode(self, node):
1345         if node.value:
1346             self.has_return_value = True
1347             if self.yields:
1348                 error(node.pos, "'return' with argument inside generator")
1349         self.returns.append(node)
1350
1351     def visit_ClassDefNode(self, node):
1352         pass
1353
1354     def visit_DefNode(self, node):
1355         pass
1356
1357     def visit_LambdaNode(self, node):
1358         pass
1359
1360     def visit_GeneratorExpressionNode(self, node):
1361         pass
1362
1363 class MarkClosureVisitor(CythonTransform):
1364
1365     def visit_ModuleNode(self, node):
1366         self.needs_closure = False
1367         self.visitchildren(node)
1368         return node
1369
1370     def visit_FuncDefNode(self, node):
1371         self.needs_closure = False
1372         self.visitchildren(node)
1373         node.needs_closure = self.needs_closure
1374         self.needs_closure = True
1375
1376         collector = YieldNodeCollector()
1377         collector.visitchildren(node)
1378
1379         if collector.yields:
1380             for i, yield_expr in enumerate(collector.yields):
1381                 yield_expr.label_num = i + 1
1382
1383             gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
1384                                                name=node.name,
1385                                                body=node.body)
1386             generator = Nodes.GeneratorDefNode(pos=node.pos,
1387                                                name=node.name,
1388                                                args=node.args,
1389                                                star_arg=node.star_arg,
1390                                                starstar_arg=node.starstar_arg,
1391                                                doc=node.doc,
1392                                                decorators=node.decorators,
1393                                                gbody=gbody,
1394                                                lambda_name=node.lambda_name)
1395             return generator
1396         return node
1397
1398     def visit_CFuncDefNode(self, node):
1399         self.visit_FuncDefNode(node)
1400         if node.needs_closure:
1401             error(node.pos, "closures inside cdef functions not yet supported")
1402         return node
1403
1404     def visit_LambdaNode(self, node):
1405         self.needs_closure = False
1406         self.visitchildren(node)
1407         node.needs_closure = self.needs_closure
1408         self.needs_closure = True
1409         return node
1410
1411     def visit_ClassDefNode(self, node):
1412         self.visitchildren(node)
1413         self.needs_closure = True
1414         return node
1415
1416 class CreateClosureClasses(CythonTransform):
1417     # Output closure classes in module scope for all functions
1418     # that really need it.
1419
1420     def __init__(self, context):
1421         super(CreateClosureClasses, self).__init__(context)
1422         self.path = []
1423         self.in_lambda = False
1424         self.generator_class = None
1425
1426     def visit_ModuleNode(self, node):
1427         self.module_scope = node.scope
1428         self.visitchildren(node)
1429         return node
1430
1431     def create_generator_class(self, target_module_scope, pos):
1432         if self.generator_class:
1433             return self.generator_class
1434         # XXX: make generator class creation cleaner
1435         entry = target_module_scope.declare_c_class(name='__pyx_Generator',
1436                     objstruct_cname='__pyx_Generator_object',
1437                     typeobj_cname='__pyx_Generator_type',
1438                     pos=pos, defining=True, implementing=True)
1439         klass = entry.type.scope
1440         klass.is_internal = True
1441         klass.directives = {'final': True}
1442
1443         body_type = PyrexTypes.create_typedef_type('generator_body',
1444                                                    PyrexTypes.c_void_ptr_type,
1445                                                    '__pyx_generator_body_t')
1446         klass.declare_var(pos=pos, name='body', cname='body',
1447                           type=body_type, is_cdef=True)
1448         klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
1449                           is_cdef=True)
1450         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
1451                           is_cdef=True)
1452
1453         import TypeSlots
1454         e = klass.declare_pyfunction('send', pos)
1455         e.func_cname = '__Pyx_Generator_Send'
1456         e.signature = TypeSlots.binaryfunc
1457
1458         e = klass.declare_pyfunction('close', pos)
1459         e.func_cname = '__Pyx_Generator_Close'
1460         e.signature = TypeSlots.unaryfunc
1461
1462         e = klass.declare_pyfunction('throw', pos)
1463         e.func_cname = '__Pyx_Generator_Throw'
1464         e.signature = TypeSlots.pyfunction_signature
1465
1466         e = klass.declare_var('__iter__', PyrexTypes.py_object_type, pos, visibility='public')
1467         e.func_cname = 'PyObject_SelfIter'
1468
1469         e = klass.declare_var('__next__', PyrexTypes.py_object_type, pos, visibility='public')
1470         e.func_cname = '__Pyx_Generator_Next'
1471
1472         self.generator_class = entry.type
1473         return self.generator_class
1474
1475     def get_scope_use(self, node):
1476         from_closure = []
1477         in_closure = []
1478         for name, entry in node.local_scope.entries.items():
1479             if entry.from_closure:
1480                 from_closure.append((name, entry))
1481             elif entry.in_closure and not entry.from_closure:
1482                 in_closure.append((name, entry))
1483         return from_closure, in_closure
1484
1485     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1486         # skip generator body
1487         if node.is_generator_body:
1488             return
1489         # move local variables into closure
1490         if node.is_generator:
1491             for entry in node.local_scope.entries.values():
1492                 if not entry.from_closure:
1493                     entry.in_closure = True
1494
1495         from_closure, in_closure = self.get_scope_use(node)
1496         in_closure.sort()
1497
1498         # Now from the begining
1499         node.needs_closure = False
1500         node.needs_outer_scope = False
1501
1502         func_scope = node.local_scope
1503         cscope = node.entry.scope
1504         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1505             cscope = cscope.outer_scope
1506
1507         if not from_closure and (self.path or inner_node):
1508             if not inner_node:
1509                 if not node.assmt:
1510                     raise InternalError, "DefNode does not have assignment node"
1511                 inner_node = node.assmt.rhs
1512             inner_node.needs_self_code = False
1513             node.needs_outer_scope = False
1514
1515         if node.is_generator:
1516             generator_class = self.create_generator_class(target_module_scope, node.pos)
1517         elif not in_closure and not from_closure:
1518             return
1519         elif not in_closure:
1520             func_scope.is_passthrough = True
1521             func_scope.scope_class = cscope.scope_class
1522             node.needs_outer_scope = True
1523             return
1524
1525         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1526
1527         if node.is_generator:
1528             entry = target_module_scope.declare_c_class(name = as_name,
1529                         pos = node.pos, defining = True, implementing = True, base_type=generator_class)
1530         else:
1531             entry = target_module_scope.declare_c_class(name = as_name,
1532                         pos = node.pos, defining = True, implementing = True)
1533         func_scope.scope_class = entry
1534         class_scope = entry.type.scope
1535         class_scope.is_internal = True
1536         class_scope.directives = {'final': True}
1537
1538         if from_closure:
1539             assert cscope.is_closure_scope
1540             class_scope.declare_var(pos=node.pos,
1541                                     name=Naming.outer_scope_cname,
1542                                     cname=Naming.outer_scope_cname,
1543                                     type=cscope.scope_class.type,
1544                                     is_cdef=True)
1545             node.needs_outer_scope = True
1546         for name, entry in in_closure:
1547             class_scope.declare_var(pos=entry.pos,
1548                                     name=entry.name,
1549                                     cname=entry.cname,
1550                                     type=entry.type,
1551                                     is_cdef=True)
1552         node.needs_closure = True
1553         # Do it here because other classes are already checked
1554         target_module_scope.check_c_class(func_scope.scope_class)
1555
1556     def visit_LambdaNode(self, node):
1557         was_in_lambda = self.in_lambda
1558         self.in_lambda = True
1559         self.create_class_from_scope(node.def_node, self.module_scope, node)
1560         self.visitchildren(node)
1561         self.in_lambda = was_in_lambda
1562         return node
1563
1564     def visit_FuncDefNode(self, node):
1565         if self.in_lambda:
1566             self.visitchildren(node)
1567             return node
1568         if node.needs_closure or self.path:
1569             self.create_class_from_scope(node, self.module_scope)
1570             self.path.append(node)
1571             self.visitchildren(node)
1572             self.path.pop()
1573         return node
1574
1575
1576 class GilCheck(VisitorTransform):
1577     """
1578     Call `node.gil_check(env)` on each node to make sure we hold the
1579     GIL when we need it.  Raise an error when on Python operations
1580     inside a `nogil` environment.
1581     """
1582     def __call__(self, root):
1583         self.env_stack = [root.scope]
1584         self.nogil = False
1585         return super(GilCheck, self).__call__(root)
1586
1587     def visit_FuncDefNode(self, node):
1588         self.env_stack.append(node.local_scope)
1589         was_nogil = self.nogil
1590         self.nogil = node.local_scope.nogil
1591         if self.nogil and node.nogil_check:
1592             node.nogil_check(node.local_scope)
1593         self.visitchildren(node)
1594         self.env_stack.pop()
1595         self.nogil = was_nogil
1596         return node
1597
1598     def visit_GILStatNode(self, node):
1599         env = self.env_stack[-1]
1600         if self.nogil and node.nogil_check: node.nogil_check()
1601         was_nogil = self.nogil
1602         self.nogil = (node.state == 'nogil')
1603         self.visitchildren(node)
1604         self.nogil = was_nogil
1605         return node
1606
1607     def visit_Node(self, node):
1608         if self.env_stack and self.nogil and node.nogil_check:
1609             node.nogil_check(self.env_stack[-1])
1610         self.visitchildren(node)
1611         return node
1612
1613
1614 class TransformBuiltinMethods(EnvTransform):
1615
1616     def visit_SingleAssignmentNode(self, node):
1617         if node.declaration_only:
1618             return None
1619         else:
1620             self.visitchildren(node)
1621             return node
1622
1623     def visit_AttributeNode(self, node):
1624         self.visitchildren(node)
1625         return self.visit_cython_attribute(node)
1626
1627     def visit_NameNode(self, node):
1628         return self.visit_cython_attribute(node)
1629
1630     def visit_cython_attribute(self, node):
1631         attribute = node.as_cython_attribute()
1632         if attribute:
1633             if attribute == u'compiled':
1634                 node = ExprNodes.BoolNode(node.pos, value=True)
1635             elif attribute == u'NULL':
1636                 node = ExprNodes.NullNode(node.pos)
1637             elif attribute in (u'set', u'frozenset'):
1638                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1639                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1640             elif not PyrexTypes.parse_basic_type(attribute):
1641                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1642         return node
1643
1644     def visit_SimpleCallNode(self, node):
1645
1646         # locals builtin
1647         if isinstance(node.function, ExprNodes.NameNode):
1648             if node.function.name == 'locals':
1649                 lenv = self.current_env()
1650                 entry = lenv.lookup_here('locals')
1651                 if entry:
1652                     # not the builtin 'locals'
1653                     return node
1654                 if len(node.args) > 0:
1655                     error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1656                     return node
1657                 pos = node.pos
1658                 items = [ ExprNodes.DictItemNode(pos,
1659                                                  key=ExprNodes.StringNode(pos, value=var),
1660                                                  value=ExprNodes.NameNode(pos, name=var))
1661                           for var in lenv.entries ]
1662                 return ExprNodes.DictNode(pos, key_value_pairs=items)
1663
1664         # cython.foo
1665         function = node.function.as_cython_attribute()
1666         if function:
1667             if function in InterpretCompilerDirectives.unop_method_nodes:
1668                 if len(node.args) != 1:
1669                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1670                 else:
1671                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1672             elif function in InterpretCompilerDirectives.binop_method_nodes:
1673                 if len(node.args) != 2:
1674                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1675                 else:
1676                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1677             elif function == u'cast':
1678                 if len(node.args) != 2:
1679                     error(node.function.pos, u"cast() takes exactly two arguments")
1680                 else:
1681                     type = node.args[0].analyse_as_type(self.current_env())
1682                     if type:
1683                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1684                     else:
1685                         error(node.args[0].pos, "Not a type")
1686             elif function == u'sizeof':
1687                 if len(node.args) != 1:
1688                     error(node.function.pos, u"sizeof() takes exactly one argument")
1689                 else:
1690                     type = node.args[0].analyse_as_type(self.current_env())
1691                     if type:
1692                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1693                     else:
1694                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1695             elif function == 'cmod':
1696                 if len(node.args) != 2:
1697                     error(node.function.pos, u"cmod() takes exactly two arguments")
1698                 else:
1699                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1700                     node.cdivision = True
1701             elif function == 'cdiv':
1702                 if len(node.args) != 2:
1703                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1704                 else:
1705                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1706                     node.cdivision = True
1707             elif function == u'set':
1708                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1709             else:
1710                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1711
1712         self.visitchildren(node)
1713         return node
1714
1715
1716 class DebugTransform(CythonTransform):
1717     """
1718     Create debug information and all functions' visibility to extern in order
1719     to enable debugging.
1720     """
1721
1722     def __init__(self, context, options, result):
1723         super(DebugTransform, self).__init__(context)
1724         self.visited = cython.set()
1725         # our treebuilder and debug output writer
1726         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1727         self.tb = self.context.gdb_debug_outputwriter
1728         #self.c_output_file = options.output_file
1729         self.c_output_file = result.c_file
1730
1731         # Closure support, basically treat nested functions as if the AST were
1732         # never nested
1733         self.nested_funcdefs = []
1734
1735         # tells visit_NameNode whether it should register step-into functions
1736         self.register_stepinto = False
1737
1738     def visit_ModuleNode(self, node):
1739         self.tb.module_name = node.full_module_name
1740         attrs = dict(
1741             module_name=node.full_module_name,
1742             filename=node.pos[0].filename,
1743             c_filename=self.c_output_file)
1744
1745         self.tb.start('Module', attrs)
1746
1747         # serialize functions
1748         self.tb.start('Functions')
1749         # First, serialize functions normally...
1750         self.visitchildren(node)
1751
1752         # ... then, serialize nested functions
1753         for nested_funcdef in self.nested_funcdefs:
1754             self.visit_FuncDefNode(nested_funcdef)
1755
1756         self.register_stepinto = True
1757         self.serialize_modulenode_as_function(node)
1758         self.register_stepinto = False
1759         self.tb.end('Functions')
1760
1761         # 2.3 compatibility. Serialize global variables
1762         self.tb.start('Globals')
1763         entries = {}
1764
1765         for k, v in node.scope.entries.iteritems():
1766             if (v.qualified_name not in self.visited and not
1767                 v.name.startswith('__pyx_') and not
1768                 v.type.is_cfunction and not
1769                 v.type.is_extension_type):
1770                 entries[k]= v
1771
1772         self.serialize_local_variables(entries)
1773         self.tb.end('Globals')
1774         # self.tb.end('Module') # end Module after the line number mapping in
1775         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1776         return node
1777
1778     def visit_FuncDefNode(self, node):
1779         self.visited.add(node.local_scope.qualified_name)
1780
1781         if getattr(node, 'is_wrapper', False):
1782             return node
1783
1784         if self.register_stepinto:
1785             self.nested_funcdefs.append(node)
1786             return node
1787
1788         # node.entry.visibility = 'extern'
1789         if node.py_func is None:
1790             pf_cname = ''
1791         else:
1792             pf_cname = node.py_func.entry.func_cname
1793
1794         attrs = dict(
1795             name=node.entry.name,
1796             cname=node.entry.func_cname,
1797             pf_cname=pf_cname,
1798             qualified_name=node.local_scope.qualified_name,
1799             lineno=str(node.pos[1]))
1800
1801         self.tb.start('Function', attrs=attrs)
1802
1803         self.tb.start('Locals')
1804         self.serialize_local_variables(node.local_scope.entries)
1805         self.tb.end('Locals')
1806
1807         self.tb.start('Arguments')
1808         for arg in node.local_scope.arg_entries:
1809             self.tb.start(arg.name)
1810             self.tb.end(arg.name)
1811         self.tb.end('Arguments')
1812
1813         self.tb.start('StepIntoFunctions')
1814         self.register_stepinto = True
1815         self.visitchildren(node)
1816         self.register_stepinto = False
1817         self.tb.end('StepIntoFunctions')
1818         self.tb.end('Function')
1819
1820         return node
1821
1822     def visit_NameNode(self, node):
1823         if (self.register_stepinto and
1824             node.type.is_cfunction and
1825             getattr(node, 'is_called', False) and
1826             node.entry.func_cname is not None):
1827             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1828             # declared functions are not 'in_cinclude'.
1829             # This means we will list called 'cdef' functions as
1830             # "step into functions", but this is not an issue as they will be
1831             # recognized as Cython functions anyway.
1832             attrs = dict(name=node.entry.func_cname)
1833             self.tb.start('StepIntoFunction', attrs=attrs)
1834             self.tb.end('StepIntoFunction')
1835
1836         self.visitchildren(node)
1837         return node
1838
1839     def serialize_modulenode_as_function(self, node):
1840         """
1841         Serialize the module-level code as a function so the debugger will know
1842         it's a "relevant frame" and it will know where to set the breakpoint
1843         for 'break modulename'.
1844         """
1845         name = node.full_module_name.rpartition('.')[-1]
1846
1847         cname_py2 = 'init' + name
1848         cname_py3 = 'PyInit_' + name
1849
1850         py2_attrs = dict(
1851             name=name,
1852             cname=cname_py2,
1853             pf_cname='',
1854             # Ignore the qualified_name, breakpoints should be set using
1855             # `cy break modulename:lineno` for module-level breakpoints.
1856             qualified_name='',
1857             lineno='1',
1858             is_initmodule_function="True",
1859         )
1860
1861         py3_attrs = dict(py2_attrs, cname=cname_py3)
1862
1863         self._serialize_modulenode_as_function(node, py2_attrs)
1864         self._serialize_modulenode_as_function(node, py3_attrs)
1865
1866     def _serialize_modulenode_as_function(self, node, attrs):
1867         self.tb.start('Function', attrs=attrs)
1868
1869         self.tb.start('Locals')
1870         self.serialize_local_variables(node.scope.entries)
1871         self.tb.end('Locals')
1872
1873         self.tb.start('Arguments')
1874         self.tb.end('Arguments')
1875
1876         self.tb.start('StepIntoFunctions')
1877         self.register_stepinto = True
1878         self.visitchildren(node)
1879         self.register_stepinto = False
1880         self.tb.end('StepIntoFunctions')
1881
1882         self.tb.end('Function')
1883
1884     def serialize_local_variables(self, entries):
1885         for entry in entries.values():
1886             if entry.type.is_pyobject:
1887                 vartype = 'PythonObject'
1888             else:
1889                 vartype = 'CObject'
1890
1891             if entry.from_closure:
1892                 # We're dealing with a closure where a variable from an outer
1893                 # scope is accessed, get it from the scope object.
1894                 cname = '%s->%s' % (Naming.cur_scope_cname,
1895                                     entry.outer_entry.cname)
1896
1897                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1898                                       entry.scope.name,
1899                                       entry.name)
1900             elif entry.in_closure:
1901                 cname = '%s->%s' % (Naming.cur_scope_cname,
1902                                     entry.cname)
1903                 qname = entry.qualified_name
1904             else:
1905                 cname = entry.cname
1906                 qname = entry.qualified_name
1907
1908             if not entry.pos:
1909                 # this happens for variables that are not in the user's code,
1910                 # e.g. for the global __builtins__, __doc__, etc. We can just
1911                 # set the lineno to 0 for those.
1912                 lineno = '0'
1913             else:
1914                 lineno = str(entry.pos[1])
1915
1916             attrs = dict(
1917                 name=entry.name,
1918                 cname=cname,
1919                 qualified_name=qname,
1920                 type=vartype,
1921                 lineno=lineno)
1922
1923             self.tb.start('LocalVar', attrs)
1924             self.tb.end('LocalVar')
1925