Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67
  68 class NormalizeTree(CythonTransform):
  69     """
  70     This transform fixes up a few things after parsing
  71     in order to make the parse tree more suitable for
  72     transforms.
  73
  74     a) After parsing, blocks with only one statement will
  75     be represented by that statement, not by a StatListNode.
  76     When doing transforms this is annoying and inconsistent,
  77     as one cannot in general remove a statement in a consistent
  78     way and so on. This transform wraps any single statements
  79     in a StatListNode containing a single statement.
  80
  81     b) The PassStatNode is a noop and serves no purpose beyond
  82     plugging such one-statement blocks; i.e., once parsed a
  83 `    "pass" can just as well be represented using an empty
  84     StatListNode. This means less special cases to worry about
  85     in subsequent transforms (one always checks to see if a
  86     StatListNode has no children to see if the block is empty).
  87     """
  88
  89     def __init__(self, context):
  90         super(NormalizeTree, self).__init__(context)
  91         self.is_in_statlist = False
  92         self.is_in_expr = False
  93
  94     def visit_ExprNode(self, node):
  95         stacktmp = self.is_in_expr
  96         self.is_in_expr = True
  97         self.visitchildren(node)
  98         self.is_in_expr = stacktmp
  99         return node
 100
 101     def visit_StatNode(self, node, is_listcontainer=False):
 102         stacktmp = self.is_in_statlist
 103         self.is_in_statlist = is_listcontainer
 104         self.visitchildren(node)
 105         self.is_in_statlist = stacktmp
 106         if not self.is_in_statlist and not self.is_in_expr:
 107             return Nodes.StatListNode(pos=node.pos, stats=[node])
 108         else:
 109             return node
 110
 111     def visit_StatListNode(self, node):
 112         self.is_in_statlist = True
 113         self.visitchildren(node)
 114         self.is_in_statlist = False
 115         return node
 116
 117     def visit_ParallelAssignmentNode(self, node):
 118         return self.visit_StatNode(node, True)
 119
 120     def visit_CEnumDefNode(self, node):
 121         return self.visit_StatNode(node, True)
 122
 123     def visit_CStructOrUnionDefNode(self, node):
 124         return self.visit_StatNode(node, True)
 125
 126     # Eliminate PassStatNode
 127     def visit_PassStatNode(self, node):
 128         if not self.is_in_statlist:
 129             return Nodes.StatListNode(pos=node.pos, stats=[])
 130         else:
 131             return []
 132
 133     def visit_CDeclaratorNode(self, node):
 134         return node
 135
 136
 137 class PostParseError(CompileError): pass
 138
 139 # error strings checked by unit tests, so define them
 140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 143 class PostParse(ScopeTrackingTransform):
 144     """
 145     Basic interpretation of the parse tree, as well as validity
 146     checking that can be done on a very basic level on the parse
 147     tree (while still not being a problem with the basic syntax,
 148     as such).
 149
 150     Specifically:
 151     - Default values to cdef assignments are turned into single
 152     assignments following the declaration (everywhere but in class
 153     bodies, where they raise a compile error)
 154
 155     - Interpret some node structures into Python runtime values.
 156     Some nodes take compile-time arguments (currently:
 157     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 158     which should be interpreted. This happens in a general way
 159     and other steps should be taken to ensure validity.
 160
 161     Type arguments cannot be interpreted in this way.
 162
 163     - For __cythonbufferdefaults__ the arguments are checked for
 164     validity.
 165
 166     TemplatedTypeNode has its directives interpreted:
 167     Any first positional argument goes into the "dtype" attribute,
 168     any "ndim" keyword argument goes into the "ndim" attribute and
 169     so on. Also it is checked that the directive combination is valid.
 170     - __cythonbufferdefaults__ attributes are parsed and put into the
 171     type information.
 172
 173     Note: Currently Parsing.py does a lot of interpretation and
 174     reorganization that can be refactored into this transform
 175     if a more pure Abstract Syntax Tree is wanted.
 176     """
 177
 178     def __init__(self, context):
 179         super(PostParse, self).__init__(context)
 180         self.specialattribute_handlers = {
 181             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 182         }
 183
 184     def visit_ModuleNode(self, node):
 185         self.lambda_counter = 1
 186         return super(PostParse, self).visit_ModuleNode(node)
 187
 188     def visit_LambdaNode(self, node):
 189         # unpack a lambda expression into the corresponding DefNode
 190         lambda_id = self.lambda_counter
 191         self.lambda_counter += 1
 192         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 193
 194         body = Nodes.ReturnStatNode(
 195             node.result_expr.pos, value = node.result_expr)
 196         node.def_node = Nodes.DefNode(
 197             node.pos, name=node.name, lambda_name=node.lambda_name,
 198             args=node.args, star_arg=node.star_arg,
 199             starstar_arg=node.starstar_arg,
 200             body=body)
 201         self.visitchildren(node)
 202         return node
 203
 204     # cdef variables
 205     def handle_bufferdefaults(self, decl):
 206         if not isinstance(decl.default, ExprNodes.DictNode):
 207             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 208         self.scope_node.buffer_defaults_node = decl.default
 209         self.scope_node.buffer_defaults_pos = decl.pos
 210
 211     def visit_CVarDefNode(self, node):
 212         # This assumes only plain names and pointers are assignable on
 213         # declaration. Also, it makes use of the fact that a cdef decl
 214         # must appear before the first use, so we don't have to deal with
 215         # "i = 3; cdef int i = i" and can simply move the nodes around.
 216         try:
 217             self.visitchildren(node)
 218             stats = [node]
 219             newdecls = []
 220             for decl in node.declarators:
 221                 declbase = decl
 222                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 223                     declbase = declbase.base
 224                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 225                     if declbase.default is not None:
 226                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 227                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 228                                 handler = self.specialattribute_handlers.get(decl.name)
 229                                 if handler:
 230                                     if decl is not declbase:
 231                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 232                                     handler(decl)
 233                                     continue # Remove declaration
 234                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 235                         first_assignment = self.scope_type != 'module'
 236                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 237                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 238                             rhs=declbase.default, first=first_assignment))
 239                         declbase.default = None
 240                 newdecls.append(decl)
 241             node.declarators = newdecls
 242             return stats
 243         except PostParseError, e:
 244             # An error in a cdef clause is ok, simply remove the declaration
 245             # and try to move on to report more errors
 246             self.context.nonfatal_error(e)
 247             return None
 248
 249     # Split parallel assignments (a,b = b,a) into separate partial
 250     # assignments that are executed rhs-first using temps.  This
 251     # restructuring must be applied before type analysis so that known
 252     # types on rhs and lhs can be matched directly.  It is required in
 253     # the case that the types cannot be coerced to a Python type in
 254     # order to assign from a tuple.
 255
 256     def visit_SingleAssignmentNode(self, node):
 257         self.visitchildren(node)
 258         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 259
 260     def visit_CascadedAssignmentNode(self, node):
 261         self.visitchildren(node)
 262         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 263
 264     def _visit_assignment_node(self, node, expr_list):
 265         """Flatten parallel assignments into separate single
 266         assignments or cascaded assignments.
 267         """
 268         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 269             # no parallel assignments => nothing to do
 270             return node
 271
 272         expr_list_list = []
 273         flatten_parallel_assignments(expr_list, expr_list_list)
 274         temp_refs = []
 275         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 276
 277         nodes = []
 278         for expr_list in expr_list_list:
 279             lhs_list = expr_list[:-1]
 280             rhs = expr_list[-1]
 281             if len(lhs_list) == 1:
 282                 node = Nodes.SingleAssignmentNode(rhs.pos,
 283                     lhs = lhs_list[0], rhs = rhs)
 284             else:
 285                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 286                     lhs_list = lhs_list, rhs = rhs)
 287             nodes.append(node)
 288
 289         if len(nodes) == 1:
 290             assign_node = nodes[0]
 291         else:
 292             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 293
 294         if temp_refs:
 295             duplicates_and_temps = [ (temp.expression, temp)
 296                                      for temp in temp_refs ]
 297             sort_common_subsequences(duplicates_and_temps)
 298             for _, temp_ref in duplicates_and_temps[::-1]:
 299                 assign_node = LetNode(temp_ref, assign_node)
 300
 301         return assign_node
 302
 303 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 304     """Replace rhs items by LetRefNodes if they appear more than once.
 305     Creates a sequence of LetRefNodes that set up the required temps
 306     and appends them to ref_node_sequence.  The input list is modified
 307     in-place.
 308     """
 309     seen_nodes = cython.set()
 310     ref_nodes = {}
 311     def find_duplicates(node):
 312         if node.is_literal or node.is_name:
 313             # no need to replace those; can't include attributes here
 314             # as their access is not necessarily side-effect free
 315             return
 316         if node in seen_nodes:
 317             if node not in ref_nodes:
 318                 ref_node = LetRefNode(node)
 319                 ref_nodes[node] = ref_node
 320                 ref_node_sequence.append(ref_node)
 321         else:
 322             seen_nodes.add(node)
 323             if node.is_sequence_constructor:
 324                 for item in node.args:
 325                     find_duplicates(item)
 326
 327     for expr_list in expr_list_list:
 328         rhs = expr_list[-1]
 329         find_duplicates(rhs)
 330     if not ref_nodes:
 331         return
 332
 333     def substitute_nodes(node):
 334         if node in ref_nodes:
 335             return ref_nodes[node]
 336         elif node.is_sequence_constructor:
 337             node.args = list(map(substitute_nodes, node.args))
 338         return node
 339
 340     # replace nodes inside of the common subexpressions
 341     for node in ref_nodes:
 342         if node.is_sequence_constructor:
 343             node.args = list(map(substitute_nodes, node.args))
 344
 345     # replace common subexpressions on all rhs items
 346     for expr_list in expr_list_list:
 347         expr_list[-1] = substitute_nodes(expr_list[-1])
 348
 349 def sort_common_subsequences(items):
 350     """Sort items/subsequences so that all items and subsequences that
 351     an item contains appear before the item itself.  This is needed
 352     because each rhs item must only be evaluated once, so its value
 353     must be evaluated first and then reused when packing sequences
 354     that contain it.
 355
 356     This implies a partial order, and the sort must be stable to
 357     preserve the original order as much as possible, so we use a
 358     simple insertion sort (which is very fast for short sequences, the
 359     normal case in practice).
 360     """
 361     def contains(seq, x):
 362         for item in seq:
 363             if item is x:
 364                 return True
 365             elif item.is_sequence_constructor and contains(item.args, x):
 366                 return True
 367         return False
 368     def lower_than(a,b):
 369         return b.is_sequence_constructor and contains(b.args, a)
 370
 371     for pos, item in enumerate(items):
 372         key = item[1] # the ResultRefNode which has already been injected into the sequences
 373         new_pos = pos
 374         for i in xrange(pos-1, -1, -1):
 375             if lower_than(key, items[i][0]):
 376                 new_pos = i
 377         if new_pos != pos:
 378             for i in xrange(pos, new_pos, -1):
 379                 items[i] = items[i-1]
 380             items[new_pos] = item
 381
 382 def flatten_parallel_assignments(input, output):
 383     #  The input is a list of expression nodes, representing the LHSs
 384     #  and RHS of one (possibly cascaded) assignment statement.  For
 385     #  sequence constructors, rearranges the matching parts of both
 386     #  sides into a list of equivalent assignments between the
 387     #  individual elements.  This transformation is applied
 388     #  recursively, so that nested structures get matched as well.
 389     rhs = input[-1]
 390     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 391         output.append(input)
 392         return
 393
 394     complete_assignments = []
 395
 396     rhs_size = len(rhs.args)
 397     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 398     starred_assignments = []
 399     for lhs in input[:-1]:
 400         if not lhs.is_sequence_constructor:
 401             if lhs.is_starred:
 402                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 403             complete_assignments.append(lhs)
 404             continue
 405         lhs_size = len(lhs.args)
 406         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 407         if starred_targets > 1:
 408             error(lhs.pos, "more than 1 starred expression in assignment")
 409             output.append([lhs,rhs])
 410             continue
 411         elif lhs_size - starred_targets > rhs_size:
 412             error(lhs.pos, "need more than %d value%s to unpack"
 413                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 414             output.append([lhs,rhs])
 415             continue
 416         elif starred_targets:
 417             map_starred_assignment(lhs_targets, starred_assignments,
 418                                    lhs.args, rhs.args)
 419         elif lhs_size < rhs_size:
 420             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 421                   % (lhs_size, rhs_size))
 422             output.append([lhs,rhs])
 423             continue
 424         else:
 425             for targets, expr in zip(lhs_targets, lhs.args):
 426                 targets.append(expr)
 427
 428     if complete_assignments:
 429         complete_assignments.append(rhs)
 430         output.append(complete_assignments)
 431
 432     # recursively flatten partial assignments
 433     for cascade, rhs in zip(lhs_targets, rhs.args):
 434         if cascade:
 435             cascade.append(rhs)
 436             flatten_parallel_assignments(cascade, output)
 437
 438     # recursively flatten starred assignments
 439     for cascade in starred_assignments:
 440         if cascade[0].is_sequence_constructor:
 441             flatten_parallel_assignments(cascade, output)
 442         else:
 443             output.append(cascade)
 444
 445 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 446     # Appends the fixed-position LHS targets to the target list that
 447     # appear left and right of the starred argument.
 448     #
 449     # The starred_assignments list receives a new tuple
 450     # (lhs_target, rhs_values_list) that maps the remaining arguments
 451     # (those that match the starred target) to a list.
 452
 453     # left side of the starred target
 454     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 455         if expr.is_starred:
 456             starred = i
 457             lhs_remaining = len(lhs_args) - i - 1
 458             break
 459         targets.append(expr)
 460     else:
 461         raise InternalError("no starred arg found when splitting starred assignment")
 462
 463     # right side of the starred target
 464     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 465                                             lhs_args[-lhs_remaining:])):
 466         targets.append(expr)
 467
 468     # the starred target itself, must be assigned a (potentially empty) list
 469     target = lhs_args[starred].target # unpack starred node
 470     starred_rhs = rhs_args[starred:]
 471     if lhs_remaining:
 472         starred_rhs = starred_rhs[:-lhs_remaining]
 473     if starred_rhs:
 474         pos = starred_rhs[0].pos
 475     else:
 476         pos = target.pos
 477     starred_assignments.append([
 478         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 479
 480
 481 class PxdPostParse(CythonTransform, SkipDeclarations):
 482     """
 483     Basic interpretation/validity checking that should only be
 484     done on pxd trees.
 485
 486     A lot of this checking currently happens in the parser; but
 487     what is listed below happens here.
 488
 489     - "def" functions are let through only if they fill the
 490     getbuffer/releasebuffer slots
 491
 492     - cdef functions are let through only if they are on the
 493     top level and are declared "inline"
 494     """
 495     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 496     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 497
 498     def __call__(self, node):
 499         self.scope_type = 'pxd'
 500         return super(PxdPostParse, self).__call__(node)
 501
 502     def visit_CClassDefNode(self, node):
 503         old = self.scope_type
 504         self.scope_type = 'cclass'
 505         self.visitchildren(node)
 506         self.scope_type = old
 507         return node
 508
 509     def visit_FuncDefNode(self, node):
 510         # FuncDefNode always come with an implementation (without
 511         # an imp they are CVarDefNodes..)
 512         err = self.ERR_INLINE_ONLY
 513
 514         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 515             and node.name in ('__getbuffer__', '__releasebuffer__')):
 516             err = None # allow these slots
 517
 518         if isinstance(node, Nodes.CFuncDefNode):
 519             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 520                 node.inline_in_pxd = True
 521                 if node.visibility != 'private':
 522                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 523                 elif node.api:
 524                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 525                 else:
 526                     err = None # allow inline function
 527             else:
 528                 err = self.ERR_INLINE_ONLY
 529
 530         if err:
 531             self.context.nonfatal_error(PostParseError(node.pos, err))
 532             return None
 533         else:
 534             return node
 535
 536 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 537     """
 538     After parsing, directives can be stored in a number of places:
 539     - #cython-comments at the top of the file (stored in ModuleNode)
 540     - Command-line arguments overriding these
 541     - @cython.directivename decorators
 542     - with cython.directivename: statements
 543
 544     This transform is responsible for interpreting these various sources
 545     and store the directive in two ways:
 546     - Set the directives attribute of the ModuleNode for global directives.
 547     - Use a CompilerDirectivesNode to override directives for a subtree.
 548
 549     (The first one is primarily to not have to modify with the tree
 550     structure, so that ModuleNode stay on top.)
 551
 552     The directives are stored in dictionaries from name to value in effect.
 553     Each such dictionary is always filled in for all possible directives,
 554     using default values where no value is given by the user.
 555
 556     The available directives are controlled in Options.py.
 557
 558     Note that we have to run this prior to analysis, and so some minor
 559     duplication of functionality has to occur: We manually track cimports
 560     and which names the "cython" module may have been imported to.
 561     """
 562     unop_method_nodes = {
 563         'typeof': ExprNodes.TypeofNode,
 564
 565         'operator.address': ExprNodes.AmpersandNode,
 566         'operator.dereference': ExprNodes.DereferenceNode,
 567         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 568         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 569         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 570         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 571
 572         # For backwards compatability.
 573         'address': ExprNodes.AmpersandNode,
 574     }
 575
 576     binop_method_nodes = {
 577         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 578     }
 579
 580     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 581                                   'cast', 'pointer', 'compiled', 'NULL'])
 582     special_methods.update(unop_method_nodes.keys())
 583
 584     def __init__(self, context, compilation_directive_defaults):
 585         super(InterpretCompilerDirectives, self).__init__(context)
 586         self.compilation_directive_defaults = {}
 587         for key, value in compilation_directive_defaults.items():
 588             self.compilation_directive_defaults[unicode(key)] = value
 589         self.cython_module_names = cython.set()
 590         self.directive_names = {}
 591
 592     def check_directive_scope(self, pos, directive, scope):
 593         legal_scopes = Options.directive_scopes.get(directive, None)
 594         if legal_scopes and scope not in legal_scopes:
 595             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 596                                         'is not allowed in %s scope' % (directive, scope)))
 597             return False
 598         else:
 599             return True
 600
 601     # Set up processing and handle the cython: comments.
 602     def visit_ModuleNode(self, node):
 603         for key, value in node.directive_comments.items():
 604             if not self.check_directive_scope(node.pos, key, 'module'):
 605                 self.wrong_scope_error(node.pos, key, 'module')
 606                 del node.directive_comments[key]
 607
 608         directives = copy.copy(Options.directive_defaults)
 609         directives.update(self.compilation_directive_defaults)
 610         directives.update(node.directive_comments)
 611         self.directives = directives
 612         node.directives = directives
 613         self.visitchildren(node)
 614         node.cython_module_names = self.cython_module_names
 615         return node
 616
 617     # The following four functions track imports and cimports that
 618     # begin with "cython"
 619     def is_cython_directive(self, name):
 620         return (name in Options.directive_types or
 621                 name in self.special_methods or
 622                 PyrexTypes.parse_basic_type(name))
 623
 624     def visit_CImportStatNode(self, node):
 625         if node.module_name == u"cython":
 626             self.cython_module_names.add(node.as_name or u"cython")
 627         elif node.module_name.startswith(u"cython."):
 628             if node.as_name:
 629                 self.directive_names[node.as_name] = node.module_name[7:]
 630             else:
 631                 self.cython_module_names.add(u"cython")
 632             # if this cimport was a compiler directive, we don't
 633             # want to leave the cimport node sitting in the tree
 634             return None
 635         return node
 636
 637     def visit_FromCImportStatNode(self, node):
 638         if (node.module_name == u"cython") or \
 639                node.module_name.startswith(u"cython."):
 640             submodule = (node.module_name + u".")[7:]
 641             newimp = []
 642             for pos, name, as_name, kind in node.imported_names:
 643                 full_name = submodule + name
 644                 if self.is_cython_directive(full_name):
 645                     if as_name is None:
 646                         as_name = full_name
 647                     self.directive_names[as_name] = full_name
 648                     if kind is not None:
 649                         self.context.nonfatal_error(PostParseError(pos,
 650                             "Compiler directive imports must be plain imports"))
 651                 else:
 652                     newimp.append((pos, name, as_name, kind))
 653             if not newimp:
 654                 return None
 655             node.imported_names = newimp
 656         return node
 657
 658     def visit_FromImportStatNode(self, node):
 659         if (node.module.module_name.value == u"cython") or \
 660                node.module.module_name.value.startswith(u"cython."):
 661             submodule = (node.module.module_name.value + u".")[7:]
 662             newimp = []
 663             for name, name_node in node.items:
 664                 full_name = submodule + name
 665                 if self.is_cython_directive(full_name):
 666                     self.directive_names[name_node.name] = full_name
 667                 else:
 668                     newimp.append((name, name_node))
 669             if not newimp:
 670                 return None
 671             node.items = newimp
 672         return node
 673
 674     def visit_SingleAssignmentNode(self, node):
 675         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 676                 node.rhs.module_name.value == u'cython'):
 677             node = Nodes.CImportStatNode(node.pos,
 678                                          module_name = u'cython',
 679                                          as_name = node.lhs.name)
 680             self.visit_CImportStatNode(node)
 681         else:
 682             self.visitchildren(node)
 683         return node
 684
 685     def visit_NameNode(self, node):
 686         if node.name in self.cython_module_names:
 687             node.is_cython_module = True
 688         else:
 689             node.cython_attribute = self.directive_names.get(node.name)
 690         return node
 691
 692     def try_to_parse_directives(self, node):
 693         # If node is the contents of an directive (in a with statement or
 694         # decorator), returns a list of (directivename, value) pairs.
 695         # Otherwise, returns None
 696         if isinstance(node, ExprNodes.CallNode):
 697             self.visit(node.function)
 698             optname = node.function.as_cython_attribute()
 699             if optname:
 700                 directivetype = Options.directive_types.get(optname)
 701                 if directivetype:
 702                     args, kwds = node.explicit_args_kwds()
 703                     directives = []
 704                     key_value_pairs = []
 705                     if kwds is not None and directivetype is not dict:
 706                         for keyvalue in kwds.key_value_pairs:
 707                             key, value = keyvalue
 708                             sub_optname = "%s.%s" % (optname, key.value)
 709                             if Options.directive_types.get(sub_optname):
 710                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 711                             else:
 712                                 key_value_pairs.append(keyvalue)
 713                         if not key_value_pairs:
 714                             kwds = None
 715                         else:
 716                             kwds.key_value_pairs = key_value_pairs
 717                         if directives and not kwds and not args:
 718                             return directives
 719                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 720                     return directives
 721         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 722             self.visit(node)
 723             optname = node.as_cython_attribute()
 724             if optname:
 725                 directivetype = Options.directive_types.get(optname)
 726                 if directivetype is bool:
 727                     return [(optname, True)]
 728                 elif directivetype is None:
 729                     return [(optname, None)]
 730                 else:
 731                     raise PostParseError(
 732                         node.pos, "The '%s' directive should be used as a function call." % optname)
 733         return None
 734
 735     def try_to_parse_directive(self, optname, args, kwds, pos):
 736         directivetype = Options.directive_types.get(optname)
 737         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 738             return optname, Options.directive_defaults[optname]
 739         elif directivetype is bool:
 740             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 741                 raise PostParseError(pos,
 742                     'The %s directive takes one compile-time boolean argument' % optname)
 743             return (optname, args[0].value)
 744         elif directivetype is str:
 745             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 746                                                                               ExprNodes.UnicodeNode)):
 747                 raise PostParseError(pos,
 748                     'The %s directive takes one compile-time string argument' % optname)
 749             return (optname, str(args[0].value))
 750         elif directivetype is dict:
 751             if len(args) != 0:
 752                 raise PostParseError(pos,
 753                     'The %s directive takes no prepositional arguments' % optname)
 754             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 755         elif directivetype is list:
 756             if kwds and len(kwds) != 0:
 757                 raise PostParseError(pos,
 758                     'The %s directive takes no keyword arguments' % optname)
 759             return optname, [ str(arg.value) for arg in args ]
 760         else:
 761             assert False
 762
 763     def visit_with_directives(self, body, directives):
 764         olddirectives = self.directives
 765         newdirectives = copy.copy(olddirectives)
 766         newdirectives.update(directives)
 767         self.directives = newdirectives
 768         assert isinstance(body, Nodes.StatListNode), body
 769         retbody = self.visit_Node(body)
 770         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 771                                                  directives=newdirectives)
 772         self.directives = olddirectives
 773         return directive
 774
 775     # Handle decorators
 776     def visit_FuncDefNode(self, node):
 777         directives = self._extract_directives(node, 'function')
 778         if not directives:
 779             return self.visit_Node(node)
 780         body = Nodes.StatListNode(node.pos, stats=[node])
 781         return self.visit_with_directives(body, directives)
 782
 783     def visit_CVarDefNode(self, node):
 784         if not node.decorators:
 785             return node
 786         for dec in node.decorators:
 787             for directive in self.try_to_parse_directives(dec.decorator) or ():
 788                 if directive is not None and directive[0] == u'locals':
 789                     node.directive_locals = directive[1]
 790                 else:
 791                     self.context.nonfatal_error(PostParseError(dec.pos,
 792                         "Cdef functions can only take cython.locals() decorator."))
 793         return node
 794
 795     def visit_CClassDefNode(self, node):
 796         directives = self._extract_directives(node, 'cclass')
 797         if not directives:
 798             return self.visit_Node(node)
 799         body = Nodes.StatListNode(node.pos, stats=[node])
 800         return self.visit_with_directives(body, directives)
 801
 802     def visit_PyClassDefNode(self, node):
 803         directives = self._extract_directives(node, 'class')
 804         if not directives:
 805             return self.visit_Node(node)
 806         body = Nodes.StatListNode(node.pos, stats=[node])
 807         return self.visit_with_directives(body, directives)
 808
 809     def _extract_directives(self, node, scope_name):
 810         if not node.decorators:
 811             return {}
 812         # Split the decorators into two lists -- real decorators and directives
 813         directives = []
 814         realdecs = []
 815         for dec in node.decorators:
 816             new_directives = self.try_to_parse_directives(dec.decorator)
 817             if new_directives is not None:
 818                 for directive in new_directives:
 819                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 820                         directives.append(directive)
 821             else:
 822                 realdecs.append(dec)
 823         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 824             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 825         else:
 826             node.decorators = realdecs
 827         # merge or override repeated directives
 828         optdict = {}
 829         directives.reverse() # Decorators coming first take precedence
 830         for directive in directives:
 831             name, value = directive
 832             if name in optdict:
 833                 old_value = optdict[name]
 834                 # keywords and arg lists can be merged, everything
 835                 # else overrides completely
 836                 if isinstance(old_value, dict):
 837                     old_value.update(value)
 838                 elif isinstance(old_value, list):
 839                     old_value.extend(value)
 840                 else:
 841                     optdict[name] = value
 842             else:
 843                 optdict[name] = value
 844         return optdict
 845
 846     # Handle with statements
 847     def visit_WithStatNode(self, node):
 848         directive_dict = {}
 849         for directive in self.try_to_parse_directives(node.manager) or []:
 850             if directive is not None:
 851                 if node.target is not None:
 852                     self.context.nonfatal_error(
 853                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 854                 else:
 855                     name, value = directive
 856                     if name == 'nogil':
 857                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 858                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 859                         return self.visit_Node(node)
 860                     if self.check_directive_scope(node.pos, name, 'with statement'):
 861                         directive_dict[name] = value
 862         if directive_dict:
 863             return self.visit_with_directives(node.body, directive_dict)
 864         return self.visit_Node(node)
 865
 866 class WithTransform(CythonTransform, SkipDeclarations):
 867
 868     # EXCINFO is manually set to a variable that contains
 869     # the exc_info() tuple that can be generated by the enclosing except
 870     # statement.
 871     template_without_target = TreeFragment(u"""
 872         MGR = EXPR
 873         EXIT = MGR.__exit__
 874         MGR.__enter__()
 875         EXC = True
 876         try:
 877             try:
 878                 EXCINFO = None
 879                 BODY
 880             except:
 881                 EXC = False
 882                 if not EXIT(*EXCINFO):
 883                     raise
 884         finally:
 885             if EXC:
 886                 EXIT(None, None, None)
 887     """, temps=[u'MGR', u'EXC', u"EXIT"],
 888     pipeline=[NormalizeTree(None)])
 889
 890     template_with_target = TreeFragment(u"""
 891         MGR = EXPR
 892         EXIT = MGR.__exit__
 893         VALUE = MGR.__enter__()
 894         EXC = True
 895         try:
 896             try:
 897                 EXCINFO = None
 898                 TARGET = VALUE
 899                 BODY
 900             except:
 901                 EXC = False
 902                 if not EXIT(*EXCINFO):
 903                     raise
 904         finally:
 905             if EXC:
 906                 EXIT(None, None, None)
 907             MGR = EXIT = VALUE = EXC = None
 908
 909     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 910     pipeline=[NormalizeTree(None)])
 911
 912     def visit_WithStatNode(self, node):
 913         # TODO: Cleanup badly needed
 914         TemplateTransform.temp_name_counter += 1
 915         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 916
 917         self.visitchildren(node, ['body'])
 918         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 919         if node.target is not None:
 920             result = self.template_with_target.substitute({
 921                 u'EXPR' : node.manager,
 922                 u'BODY' : node.body,
 923                 u'TARGET' : node.target,
 924                 u'EXCINFO' : excinfo_temp
 925                 }, pos=node.pos)
 926         else:
 927             result = self.template_without_target.substitute({
 928                 u'EXPR' : node.manager,
 929                 u'BODY' : node.body,
 930                 u'EXCINFO' : excinfo_temp
 931                 }, pos=node.pos)
 932
 933         # Set except excinfo target to EXCINFO
 934         try_except = result.stats[-1].body.stats[-1]
 935         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 936 #            excinfo_temp.ref(node.pos))
 937
 938 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 939 #            node.pos, temps=[excinfo_temp], body=try_except)
 940
 941         return result
 942
 943     def visit_ExprNode(self, node):
 944         # With statements are never inside expressions.
 945         return node
 946
 947
 948 class DecoratorTransform(CythonTransform, SkipDeclarations):
 949
 950     def visit_DefNode(self, func_node):
 951         self.visitchildren(func_node)
 952         if not func_node.decorators:
 953             return func_node
 954         return self._handle_decorators(
 955             func_node, func_node.name)
 956
 957     def visit_CClassDefNode(self, class_node):
 958         # This doesn't currently work, so it's disabled.
 959         #
 960         # Problem: assignments to cdef class names do not work.  They
 961         # would require an additional check anyway, as the extension
 962         # type must not change its C type, so decorators cannot
 963         # replace an extension type, just alter it and return it.
 964
 965         self.visitchildren(class_node)
 966         if not class_node.decorators:
 967             return class_node
 968         error(class_node.pos,
 969               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 970         return class_node
 971         #return self._handle_decorators(
 972         #    class_node, class_node.class_name)
 973
 974     def visit_ClassDefNode(self, class_node):
 975         self.visitchildren(class_node)
 976         if not class_node.decorators:
 977             return class_node
 978         return self._handle_decorators(
 979             class_node, class_node.name)
 980
 981     def _handle_decorators(self, node, name):
 982         decorator_result = ExprNodes.NameNode(node.pos, name = name)
 983         for decorator in node.decorators[::-1]:
 984             decorator_result = ExprNodes.SimpleCallNode(
 985                 decorator.pos,
 986                 function = decorator.decorator,
 987                 args = [decorator_result])
 988
 989         name_node = ExprNodes.NameNode(node.pos, name = name)
 990         reassignment = Nodes.SingleAssignmentNode(
 991             node.pos,
 992             lhs = name_node,
 993             rhs = decorator_result)
 994         return [node, reassignment]
 995
 996
 997 class AnalyseDeclarationsTransform(CythonTransform):
 998
 999     basic_property = TreeFragment(u"""
1000 property NAME:
1001     def __get__(self):
1002         return ATTR
1003     def __set__(self, value):
1004         ATTR = value
1005     """, level='c_class')
1006     basic_pyobject_property = TreeFragment(u"""
1007 property NAME:
1008     def __get__(self):
1009         return ATTR
1010     def __set__(self, value):
1011         ATTR = value
1012     def __del__(self):
1013         ATTR = None
1014     """, level='c_class')
1015     basic_property_ro = TreeFragment(u"""
1016 property NAME:
1017     def __get__(self):
1018         return ATTR
1019     """, level='c_class')
1020
1021     def __call__(self, root):
1022         self.env_stack = [root.scope]
1023         # needed to determine if a cdef var is declared after it's used.
1024         self.seen_vars_stack = []
1025         return super(AnalyseDeclarationsTransform, self).__call__(root)
1026
1027     def visit_NameNode(self, node):
1028         self.seen_vars_stack[-1].add(node.name)
1029         return node
1030
1031     def visit_ModuleNode(self, node):
1032         self.seen_vars_stack.append(cython.set())
1033         node.analyse_declarations(self.env_stack[-1])
1034         self.visitchildren(node)
1035         self.seen_vars_stack.pop()
1036         return node
1037
1038     def visit_LambdaNode(self, node):
1039         node.analyse_declarations(self.env_stack[-1])
1040         self.visitchildren(node)
1041         return node
1042
1043     def visit_ClassDefNode(self, node):
1044         self.env_stack.append(node.scope)
1045         self.visitchildren(node)
1046         self.env_stack.pop()
1047         return node
1048
1049     def visit_CClassDefNode(self, node):
1050         node = self.visit_ClassDefNode(node)
1051         if node.scope and node.scope.implemented:
1052             stats = []
1053             for entry in node.scope.var_entries:
1054                 if entry.needs_property:
1055                     property = self.create_Property(entry)
1056                     property.analyse_declarations(node.scope)
1057                     self.visit(property)
1058                     stats.append(property)
1059             if stats:
1060                 node.body.stats += stats
1061         return node
1062
1063     def visit_FuncDefNode(self, node):
1064         self.seen_vars_stack.append(cython.set())
1065         lenv = node.local_scope
1066         node.body.analyse_control_flow(lenv) # this will be totally refactored
1067         node.declare_arguments(lenv)
1068         for var, type_node in node.directive_locals.items():
1069             if not lenv.lookup_here(var):   # don't redeclare args
1070                 type = type_node.analyse_as_type(lenv)
1071                 if type:
1072                     lenv.declare_var(var, type, type_node.pos)
1073                 else:
1074                     error(type_node.pos, "Not a type")
1075         node.body.analyse_declarations(lenv)
1076         self.env_stack.append(lenv)
1077         self.visitchildren(node)
1078         self.env_stack.pop()
1079         self.seen_vars_stack.pop()
1080         return node
1081
1082     def visit_ScopedExprNode(self, node):
1083         env = self.env_stack[-1]
1084         node.analyse_declarations(env)
1085         # the node may or may not have a local scope
1086         if node.has_local_scope:
1087             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1088             self.env_stack.append(node.expr_scope)
1089             node.analyse_scoped_declarations(node.expr_scope)
1090             self.visitchildren(node)
1091             self.env_stack.pop()
1092             self.seen_vars_stack.pop()
1093         else:
1094             node.analyse_scoped_declarations(env)
1095             self.visitchildren(node)
1096         return node
1097
1098     def visit_TempResultFromStatNode(self, node):
1099         self.visitchildren(node)
1100         node.analyse_declarations(self.env_stack[-1])
1101         return node
1102
1103     # Some nodes are no longer needed after declaration
1104     # analysis and can be dropped. The analysis was performed
1105     # on these nodes in a seperate recursive process from the
1106     # enclosing function or module, so we can simply drop them.
1107     def visit_CDeclaratorNode(self, node):
1108         # necessary to ensure that all CNameDeclaratorNodes are visited.
1109         self.visitchildren(node)
1110         return node
1111
1112     def visit_CTypeDefNode(self, node):
1113         return node
1114
1115     def visit_CBaseTypeNode(self, node):
1116         return None
1117
1118     def visit_CEnumDefNode(self, node):
1119         if node.visibility == 'public':
1120             return node
1121         else:
1122             return None
1123
1124     def visit_CStructOrUnionDefNode(self, node):
1125         return None
1126
1127     def visit_CNameDeclaratorNode(self, node):
1128         if node.name in self.seen_vars_stack[-1]:
1129             entry = self.env_stack[-1].lookup(node.name)
1130             if entry is None or not entry.c_source.extern:
1131                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1132         self.visitchildren(node)
1133         return node
1134
1135     def visit_CVarDefNode(self, node):
1136         # to ensure all CNameDeclaratorNodes are visited.
1137         self.visitchildren(node)
1138         return None
1139
1140     def create_Property(self, entry):
1141         if entry.python_binding.visibility == 'public':
1142             if entry.type.is_pyobject:
1143                 template = self.basic_pyobject_property
1144             else:
1145                 template = self.basic_property
1146         elif entry.python_binding.visibility == 'readonly':
1147             template = self.basic_property_ro
1148         else:
1149             raise NotImplementedError('private python methods')
1150         property = template.substitute({
1151                 u"ATTR": ExprNodes.AttributeNode(
1152                     pos=entry.pos,
1153                     obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1154                     attribute=entry.python_binding.name),
1155             }, pos=entry.pos).stats[0]
1156         property.name = entry.python_binding.name
1157         # ---------------------------------------
1158         # XXX This should go to AutoDocTransforms
1159         # ---------------------------------------
1160         if (Options.docstrings and
1161             self.current_directives['embedsignature']):
1162             attr_name = entry.python_binding.name
1163             type_name = entry.type.declaration_code("", for_display=1)
1164             default_value = ''
1165             if not entry.type.is_pyobject:
1166                 type_name = "'%s'" % type_name
1167             elif entry.type.is_extension_type:
1168                 type_name = entry.type.module_name + '.' + type_name
1169             if entry.init is not None:
1170                 default_value = ' = ' + entry.init
1171             elif entry.init_to_none:
1172                 default_value = ' = ' + repr(None)
1173             docstring = attr_name + ': ' + type_name + default_value
1174             property.doc = EncodedString(docstring)
1175         # ---------------------------------------
1176         return property
1177
1178 class AnalyseExpressionsTransform(CythonTransform):
1179
1180     def visit_ModuleNode(self, node):
1181         node.scope.infer_types()
1182         node.body.analyse_expressions(node.scope)
1183         self.visitchildren(node)
1184         return node
1185
1186     def visit_FuncDefNode(self, node):
1187         node.local_scope.infer_types()
1188         node.body.analyse_expressions(node.local_scope)
1189         self.visitchildren(node)
1190         return node
1191
1192     def visit_ScopedExprNode(self, node):
1193         if node.has_local_scope:
1194             node.expr_scope.infer_types()
1195             node.analyse_scoped_expressions(node.expr_scope)
1196         self.visitchildren(node)
1197         return node
1198
1199 class ExpandInplaceOperators(EnvTransform):
1200
1201     def visit_InPlaceAssignmentNode(self, node):
1202         lhs = node.lhs
1203         rhs = node.rhs
1204         if lhs.type.is_cpp_class:
1205             # No getting around this exact operator here.
1206             return node
1207         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1208             # There is code to handle this case.
1209             return node
1210
1211         env = self.current_env()
1212         def side_effect_free_reference(node, setting=False):
1213             if isinstance(node, ExprNodes.NameNode):
1214                 return node, []
1215             elif node.type.is_pyobject and not setting:
1216                 node = LetRefNode(node)
1217                 return node, [node]
1218             elif isinstance(node, ExprNodes.IndexNode):
1219                 if node.is_buffer_access:
1220                     raise ValueError, "Buffer access"
1221                 base, temps = side_effect_free_reference(node.base)
1222                 index = LetRefNode(node.index)
1223                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1224             elif isinstance(node, ExprNodes.AttributeNode):
1225                 obj, temps = side_effect_free_reference(node.obj)
1226                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1227             else:
1228                 node = LetRefNode(node)
1229                 return node, [node]
1230         try:
1231             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1232         except ValueError:
1233             return node
1234         dup = lhs.__class__(**lhs.__dict__)
1235         binop = ExprNodes.binop_node(node.pos,
1236                                      operator = node.operator,
1237                                      operand1 = dup,
1238                                      operand2 = rhs,
1239                                      inplace=True)
1240         # Manually analyse types for new node.
1241         lhs.analyse_target_types(env)
1242         dup.analyse_types(env)
1243         binop.analyse_operation(env)
1244         node = Nodes.SingleAssignmentNode(
1245             node.pos,
1246             lhs = lhs,
1247             rhs=binop.coerce_to(lhs.type, env))
1248         # Use LetRefNode to avoid side effects.
1249         let_ref_nodes.reverse()
1250         for t in let_ref_nodes:
1251             node = LetNode(t, node)
1252         return node
1253
1254     def visit_ExprNode(self, node):
1255         # In-place assignments can't happen within an expression.
1256         return node
1257
1258
1259 class AlignFunctionDefinitions(CythonTransform):
1260     """
1261     This class takes the signatures from a .pxd file and applies them to
1262     the def methods in a .py file.
1263     """
1264
1265     def visit_ModuleNode(self, node):
1266         self.scope = node.scope
1267         self.directives = node.directives
1268         self.visitchildren(node)
1269         return node
1270
1271     def visit_PyClassDefNode(self, node):
1272         pxd_def = self.scope.lookup(node.name)
1273         if pxd_def:
1274             if pxd_def.is_cclass:
1275                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1276             else:
1277                 error(node.pos, "'%s' redeclared" % node.name)
1278                 error(pxd_def.pos, "previous declaration here")
1279                 return None
1280         else:
1281             return node
1282
1283     def visit_CClassDefNode(self, node, pxd_def=None):
1284         if pxd_def is None:
1285             pxd_def = self.scope.lookup(node.class_name)
1286         if pxd_def:
1287             outer_scope = self.scope
1288             self.scope = pxd_def.type.scope
1289         self.visitchildren(node)
1290         if pxd_def:
1291             self.scope = outer_scope
1292         return node
1293
1294     def visit_DefNode(self, node):
1295         pxd_def = self.scope.lookup(node.name)
1296         if pxd_def:
1297             if not pxd_def.is_cfunction:
1298                 error(node.pos, "'%s' redeclared" % node.name)
1299                 error(pxd_def.pos, "previous declaration here")
1300                 return None
1301             node = node.as_cfunction(pxd_def)
1302         elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1303             node = node.as_cfunction(scope=self.scope)
1304         # Enable this when internal def functions are allowed.
1305         # self.visitchildren(node)
1306         return node
1307
1308
1309 class MarkClosureVisitor(CythonTransform):
1310
1311     def visit_ModuleNode(self, node):
1312         self.needs_closure = False
1313         self.visitchildren(node)
1314         return node
1315
1316     def visit_FuncDefNode(self, node):
1317         self.needs_closure = False
1318         self.visitchildren(node)
1319         node.needs_closure = self.needs_closure
1320         self.needs_closure = True
1321         return node
1322
1323     def visit_CFuncDefNode(self, node):
1324         self.visit_FuncDefNode(node)
1325         if node.needs_closure:
1326             error(node.pos, "closures inside cdef functions not yet supported")
1327         return node
1328
1329     def visit_LambdaNode(self, node):
1330         self.needs_closure = False
1331         self.visitchildren(node)
1332         node.needs_closure = self.needs_closure
1333         self.needs_closure = True
1334         return node
1335
1336     def visit_ClassDefNode(self, node):
1337         self.visitchildren(node)
1338         self.needs_closure = True
1339         return node
1340
1341
1342 class CreateClosureClasses(CythonTransform):
1343     # Output closure classes in module scope for all functions
1344     # that really need it.
1345
1346     def __init__(self, context):
1347         super(CreateClosureClasses, self).__init__(context)
1348         self.path = []
1349         self.in_lambda = False
1350
1351     def visit_ModuleNode(self, node):
1352         self.module_scope = node.scope
1353         self.visitchildren(node)
1354         return node
1355
1356     def get_scope_use(self, node):
1357         from_closure = []
1358         in_closure = []
1359         for name, entry in node.local_scope.entries.items():
1360             if entry.from_closure:
1361                 from_closure.append((name, entry))
1362             elif entry.in_closure and not entry.from_closure:
1363                 in_closure.append((name, entry))
1364         return from_closure, in_closure
1365
1366     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1367         from_closure, in_closure = self.get_scope_use(node)
1368         in_closure.sort()
1369
1370         # Now from the begining
1371         node.needs_closure = False
1372         node.needs_outer_scope = False
1373
1374         func_scope = node.local_scope
1375         cscope = node.entry.scope
1376         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1377             cscope = cscope.outer_scope
1378
1379         if not from_closure and (self.path or inner_node):
1380             if not inner_node:
1381                 if not node.assmt:
1382                     raise InternalError, "DefNode does not have assignment node"
1383                 inner_node = node.assmt.rhs
1384             inner_node.needs_self_code = False
1385             node.needs_outer_scope = False
1386         # Simple cases
1387         if not in_closure and not from_closure:
1388             return
1389         elif not in_closure:
1390             func_scope.is_passthrough = True
1391             func_scope.scope_class = cscope.scope_class
1392             node.needs_outer_scope = True
1393             return
1394
1395         as_name = '%s_%s' % (
1396             target_module_scope.next_id(Naming.closure_class_prefix),
1397             node.entry.c_binding.name)
1398
1399         entry = target_module_scope.declare_c_class(name = as_name,
1400             pos = node.pos, defining = True, implementing = True)
1401         func_scope.scope_class = entry
1402         class_scope = entry.type.scope
1403         class_scope.is_internal = True
1404         class_scope.directives = {'final': True}
1405
1406         if from_closure:
1407             assert cscope.is_closure_scope
1408             class_scope.declare_var(pos=node.pos,
1409                                     name=Naming.outer_scope_cname,
1410                                     cname=Naming.outer_scope_cname,
1411                                     type=cscope.scope_class.type,
1412                                     is_cdef=True)
1413             node.needs_outer_scope = True
1414         for name, entry in in_closure:
1415             class_scope.declare_var(pos=entry.pos,
1416                                     name=entry.python_binding.name,
1417                                     cname=entry.c_binding.name,
1418                                     type=entry.type,
1419                                     is_cdef=True)
1420         node.needs_closure = True
1421         # Do it here because other classes are already checked
1422         target_module_scope.check_c_class(func_scope.scope_class)
1423
1424     def visit_LambdaNode(self, node):
1425         was_in_lambda = self.in_lambda
1426         self.in_lambda = True
1427         self.create_class_from_scope(node.def_node, self.module_scope, node)
1428         self.visitchildren(node)
1429         self.in_lambda = was_in_lambda
1430         return node
1431
1432     def visit_FuncDefNode(self, node):
1433         if self.in_lambda:
1434             self.visitchildren(node)
1435             return node
1436         if node.needs_closure or self.path:
1437             self.create_class_from_scope(node, self.module_scope)
1438             self.path.append(node)
1439             self.visitchildren(node)
1440             self.path.pop()
1441         return node
1442
1443
1444 class GilCheck(VisitorTransform):
1445     """
1446     Call `node.gil_check(env)` on each node to make sure we hold the
1447     GIL when we need it.  Raise an error when on Python operations
1448     inside a `nogil` environment.
1449     """
1450     def __call__(self, root):
1451         self.env_stack = [root.scope]
1452         self.nogil = False
1453         return super(GilCheck, self).__call__(root)
1454
1455     def visit_FuncDefNode(self, node):
1456         self.env_stack.append(node.local_scope)
1457         was_nogil = self.nogil
1458         self.nogil = node.local_scope.nogil
1459         if self.nogil and node.nogil_check:
1460             node.nogil_check(node.local_scope)
1461         self.visitchildren(node)
1462         self.env_stack.pop()
1463         self.nogil = was_nogil
1464         return node
1465
1466     def visit_GILStatNode(self, node):
1467         env = self.env_stack[-1]
1468         if self.nogil and node.nogil_check: node.nogil_check()
1469         was_nogil = self.nogil
1470         self.nogil = (node.state == 'nogil')
1471         self.visitchildren(node)
1472         self.nogil = was_nogil
1473         return node
1474
1475     def visit_Node(self, node):
1476         if self.env_stack and self.nogil and node.nogil_check:
1477             node.nogil_check(self.env_stack[-1])
1478         self.visitchildren(node)
1479         return node
1480
1481
1482 class TransformBuiltinMethods(EnvTransform):
1483
1484     def visit_SingleAssignmentNode(self, node):
1485         if node.declaration_only:
1486             return None
1487         else:
1488             self.visitchildren(node)
1489             return node
1490
1491     def visit_AttributeNode(self, node):
1492         self.visitchildren(node)
1493         return self.visit_cython_attribute(node)
1494
1495     def visit_NameNode(self, node):
1496         return self.visit_cython_attribute(node)
1497
1498     def visit_cython_attribute(self, node):
1499         attribute = node.as_cython_attribute()
1500         if attribute:
1501             if attribute == u'compiled':
1502                 node = ExprNodes.BoolNode(node.pos, value=True)
1503             elif attribute == u'NULL':
1504                 node = ExprNodes.NullNode(node.pos)
1505             elif attribute in (u'set', u'frozenset'):
1506                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1507                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1508             elif not PyrexTypes.parse_basic_type(attribute):
1509                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1510         return node
1511
1512     def visit_SimpleCallNode(self, node):
1513
1514         # locals builtin
1515         if isinstance(node.function, ExprNodes.NameNode):
1516             if node.function.name == 'locals':
1517                 lenv = self.current_env()
1518                 entry = lenv.lookup_here('locals')
1519                 if entry:
1520                     # not the builtin 'locals'
1521                     return node
1522                 if len(node.args) > 0:
1523                     error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1524                     return node
1525                 pos = node.pos
1526                 items = [ ExprNodes.DictItemNode(pos,
1527                                                  key=ExprNodes.StringNode(pos, value=var),
1528                                                  value=ExprNodes.NameNode(pos, name=var))
1529                           for var in lenv.entries ]
1530                 return ExprNodes.DictNode(pos, key_value_pairs=items)
1531
1532         # cython.foo
1533         function = node.function.as_cython_attribute()
1534         if function:
1535             if function in InterpretCompilerDirectives.unop_method_nodes:
1536                 if len(node.args) != 1:
1537                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1538                 else:
1539                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1540             elif function in InterpretCompilerDirectives.binop_method_nodes:
1541                 if len(node.args) != 2:
1542                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1543                 else:
1544                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1545             elif function == u'cast':
1546                 if len(node.args) != 2:
1547                     error(node.function.pos, u"cast() takes exactly two arguments")
1548                 else:
1549                     type = node.args[0].analyse_as_type(self.current_env())
1550                     if type:
1551                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1552                     else:
1553                         error(node.args[0].pos, "Not a type")
1554             elif function == u'sizeof':
1555                 if len(node.args) != 1:
1556                     error(node.function.pos, u"sizeof() takes exactly one argument")
1557                 else:
1558                     type = node.args[0].analyse_as_type(self.current_env())
1559                     if type:
1560                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1561                     else:
1562                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1563             elif function == 'cmod':
1564                 if len(node.args) != 2:
1565                     error(node.function.pos, u"cmod() takes exactly two arguments")
1566                 else:
1567                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1568                     node.cdivision = True
1569             elif function == 'cdiv':
1570                 if len(node.args) != 2:
1571                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1572                 else:
1573                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1574                     node.cdivision = True
1575             elif function == u'set':
1576                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1577             else:
1578                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1579
1580         self.visitchildren(node)
1581         return node
1582
1583
1584 class DebugTransform(CythonTransform):
1585     """
1586     Create debug information and all functions' visibility to extern in order
1587     to enable debugging.
1588     """
1589
1590     def __init__(self, context, options, result):
1591         super(DebugTransform, self).__init__(context)
1592         self.visited = cython.set()
1593         # our treebuilder and debug output writer
1594         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1595         self.tb = self.context.gdb_debug_outputwriter
1596         #self.c_output_file = options.output_file
1597         self.c_output_file = result.c_file
1598
1599         # Closure support, basically treat nested functions as if the AST were
1600         # never nested
1601         self.nested_funcdefs = []
1602
1603         # tells visit_NameNode whether it should register step-into functions
1604         self.register_stepinto = False
1605
1606     def visit_ModuleNode(self, node):
1607         self.tb.module_name = node.full_module_name
1608         attrs = dict(
1609             module_name=node.full_module_name,
1610             filename=node.pos[0].filename,
1611             c_filename=self.c_output_file)
1612
1613         self.tb.start('Module', attrs)
1614
1615         # serialize functions
1616         self.tb.start('Functions')
1617         # First, serialize functions normally...
1618         self.visitchildren(node)
1619
1620         # ... then, serialize nested functions
1621         for nested_funcdef in self.nested_funcdefs:
1622             self.visit_FuncDefNode(nested_funcdef)
1623
1624         self.register_stepinto = True
1625         self.serialize_modulenode_as_function(node)
1626         self.register_stepinto = False
1627         self.tb.end('Functions')
1628
1629         # 2.3 compatibility. Serialize global variables
1630         self.tb.start('Globals')
1631         entries = {}
1632
1633         for k, v in node.scope.entries.iteritems():
1634             if (v.qualified_name not in self.visited and not
1635                 v.name.startswith('__pyx_') and not
1636                 v.type.is_cfunction and not
1637                 v.type.is_extension_type):
1638                 entries[k]= v
1639
1640         self.serialize_local_variables(entries)
1641         self.tb.end('Globals')
1642         # self.tb.end('Module') # end Module after the line number mapping in
1643         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1644         return node
1645
1646     def visit_FuncDefNode(self, node):
1647         self.visited.add(node.local_scope.qualified_name)
1648
1649         if getattr(node, 'is_wrapper', False):
1650             return node
1651
1652         if self.register_stepinto:
1653             self.nested_funcdefs.append(node)
1654             return node
1655
1656         # node.entry.c_source.extern
1657         if node.py_func is None:
1658             pf_cname = ''
1659         else:
1660             pf_cname = node.py_func.entry.func_cname
1661
1662         attrs = dict(
1663             name=node.entry.python_binding.name,
1664             cname=node.entry.func_cname,
1665             pf_cname=pf_cname,
1666             qualified_name=node.local_scope.qualified_name,
1667             lineno=str(node.pos[1]))
1668
1669         self.tb.start('Function', attrs=attrs)
1670
1671         self.tb.start('Locals')
1672         self.serialize_local_variables(node.local_scope.entries)
1673         self.tb.end('Locals')
1674
1675         self.tb.start('Arguments')
1676         for arg in node.local_scope.arg_entries:
1677             self.tb.start(arg.name)
1678             self.tb.end(arg.name)
1679         self.tb.end('Arguments')
1680
1681         self.tb.start('StepIntoFunctions')
1682         self.register_stepinto = True
1683         self.visitchildren(node)
1684         self.register_stepinto = False
1685         self.tb.end('StepIntoFunctions')
1686         self.tb.end('Function')
1687
1688         return node
1689
1690     def visit_NameNode(self, node):
1691         if (self.register_stepinto and
1692             node.type.is_cfunction and
1693             getattr(node, 'is_called', False) and
1694             node.entry.func_cname is not None):
1695             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1696             # declared functions are not 'in_cinclude'.
1697             # This means we will list called 'cdef' functions as
1698             # "step into functions", but this is not an issue as they will be
1699             # recognized as Cython functions anyway.
1700             attrs = dict(name=node.entry.func_cname)
1701             self.tb.start('StepIntoFunction', attrs=attrs)
1702             self.tb.end('StepIntoFunction')
1703
1704         self.visitchildren(node)
1705         return node
1706
1707     def serialize_modulenode_as_function(self, node):
1708         """
1709         Serialize the module-level code as a function so the debugger will know
1710         it's a "relevant frame" and it will know where to set the breakpoint
1711         for 'break modulename'.
1712         """
1713         name = node.full_module_name.rpartition('.')[-1]
1714
1715         cname_py2 = 'init' + name
1716         cname_py3 = 'PyInit_' + name
1717
1718         py2_attrs = dict(
1719             name=name,
1720             cname=cname_py2,
1721             pf_cname='',
1722             # Ignore the qualified_name, breakpoints should be set using
1723             # `cy break modulename:lineno` for module-level breakpoints.
1724             qualified_name='',
1725             lineno='1',
1726             is_initmodule_function="True",
1727         )
1728
1729         py3_attrs = dict(py2_attrs, cname=cname_py3)
1730
1731         self._serialize_modulenode_as_function(node, py2_attrs)
1732         self._serialize_modulenode_as_function(node, py3_attrs)
1733
1734     def _serialize_modulenode_as_function(self, node, attrs):
1735         self.tb.start('Function', attrs=attrs)
1736
1737         self.tb.start('Locals')
1738         self.serialize_local_variables(node.scope.entries)
1739         self.tb.end('Locals')
1740
1741         self.tb.start('Arguments')
1742         self.tb.end('Arguments')
1743
1744         self.tb.start('StepIntoFunctions')
1745         self.register_stepinto = True
1746         self.visitchildren(node)
1747         self.register_stepinto = False
1748         self.tb.end('StepIntoFunctions')
1749
1750         self.tb.end('Function')
1751
1752     def serialize_local_variables(self, entries):
1753         for entry in entries.values():
1754             if entry.type.is_pyobject:
1755                 vartype = 'PythonObject'
1756             else:
1757                 vartype = 'CObject'
1758
1759             if entry.from_closure:
1760                 # We're dealing with a closure where a variable from an outer
1761                 # scope is accessed, get it from the scope object.
1762                 cname = '%s->%s' % (Naming.cur_scope_cname,
1763                                     entry.outer_entry.c_binding.name)
1764
1765                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1766                                       entry.scope.name,
1767                                       entry.python_binding.name)
1768             elif entry.in_closure:
1769                 cname = '%s->%s' % (Naming.cur_scope_cname,
1770                                     entry.c_binding.name)
1771                 qname = entry.qualified_name
1772             else:
1773                 cname = entry.c_binding.name
1774                 qname = entry.qualified_name
1775
1776             if not entry.pos:
1777                 # this happens for variables that are not in the user's code,
1778                 # e.g. for the global __builtins__, __doc__, etc. We can just
1779                 # set the lineno to 0 for those.
1780                 lineno = '0'
1781             else:
1782                 lineno = str(entry.pos[1])
1783
1784             attrs = dict(
1785                 name=entry.python_binding.name,
1786                 cname=cname,
1787                 qualified_name=qname,
1788                 type=vartype,
1789                 lineno=lineno)
1790
1791             self.tb.start('LocalVar', attrs)
1792             self.tb.end('LocalVar')
1793