Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 import PyrexTypes
  10 import Naming
  11 import ExprNodes
  12 import Nodes
  13 import Options
  14
  15 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  16 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  17 from Cython.Compiler.ModuleNode import ModuleNode
  18 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  19 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  20 from Cython.Compiler.StringEncoding import EncodedString
  21 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  22
  23 import copy
  24
  25
  26 class NameNodeCollector(TreeVisitor):
  27     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  28     attribute.
  29     """
  30     def __init__(self):
  31         super(NameNodeCollector, self).__init__()
  32         self.name_nodes = []
  33
  34     def visit_NameNode(self, node):
  35         self.name_nodes.append(node)
  36
  37     def visit_Node(self, node):
  38         self._visitchildren(node, None)
  39
  40
  41 class SkipDeclarations(object):
  42     """
  43     Variable and function declarations can often have a deep tree structure,
  44     and yet most transformations don't need to descend to this depth.
  45
  46     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  47     is no need to use this for transformations after that point.
  48     """
  49     def visit_CTypeDefNode(self, node):
  50         return node
  51
  52     def visit_CVarDefNode(self, node):
  53         return node
  54
  55     def visit_CDeclaratorNode(self, node):
  56         return node
  57
  58     def visit_CBaseTypeNode(self, node):
  59         return node
  60
  61     def visit_CEnumDefNode(self, node):
  62         return node
  63
  64     def visit_CStructOrUnionDefNode(self, node):
  65         return node
  66
  67
  68 class NormalizeTree(CythonTransform):
  69     """
  70     This transform fixes up a few things after parsing
  71     in order to make the parse tree more suitable for
  72     transforms.
  73
  74     a) After parsing, blocks with only one statement will
  75     be represented by that statement, not by a StatListNode.
  76     When doing transforms this is annoying and inconsistent,
  77     as one cannot in general remove a statement in a consistent
  78     way and so on. This transform wraps any single statements
  79     in a StatListNode containing a single statement.
  80
  81     b) The PassStatNode is a noop and serves no purpose beyond
  82     plugging such one-statement blocks; i.e., once parsed a
  83 `    "pass" can just as well be represented using an empty
  84     StatListNode. This means less special cases to worry about
  85     in subsequent transforms (one always checks to see if a
  86     StatListNode has no children to see if the block is empty).
  87     """
  88
  89     def __init__(self, context):
  90         super(NormalizeTree, self).__init__(context)
  91         self.is_in_statlist = False
  92         self.is_in_expr = False
  93
  94     def visit_ExprNode(self, node):
  95         stacktmp = self.is_in_expr
  96         self.is_in_expr = True
  97         self.visitchildren(node)
  98         self.is_in_expr = stacktmp
  99         return node
 100
 101     def visit_StatNode(self, node, is_listcontainer=False):
 102         stacktmp = self.is_in_statlist
 103         self.is_in_statlist = is_listcontainer
 104         self.visitchildren(node)
 105         self.is_in_statlist = stacktmp
 106         if not self.is_in_statlist and not self.is_in_expr:
 107             return Nodes.StatListNode(pos=node.pos, stats=[node])
 108         else:
 109             return node
 110
 111     def visit_StatListNode(self, node):
 112         self.is_in_statlist = True
 113         self.visitchildren(node)
 114         self.is_in_statlist = False
 115         return node
 116
 117     def visit_ParallelAssignmentNode(self, node):
 118         return self.visit_StatNode(node, True)
 119
 120     def visit_CEnumDefNode(self, node):
 121         return self.visit_StatNode(node, True)
 122
 123     def visit_CStructOrUnionDefNode(self, node):
 124         return self.visit_StatNode(node, True)
 125
 126     # Eliminate PassStatNode
 127     def visit_PassStatNode(self, node):
 128         if not self.is_in_statlist:
 129             return Nodes.StatListNode(pos=node.pos, stats=[])
 130         else:
 131             return []
 132
 133     def visit_CDeclaratorNode(self, node):
 134         return node
 135
 136
 137 class PostParseError(CompileError): pass
 138
 139 # error strings checked by unit tests, so define them
 140 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 141 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 142 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 143 class PostParse(ScopeTrackingTransform):
 144     """
 145     Basic interpretation of the parse tree, as well as validity
 146     checking that can be done on a very basic level on the parse
 147     tree (while still not being a problem with the basic syntax,
 148     as such).
 149
 150     Specifically:
 151     - Default values to cdef assignments are turned into single
 152     assignments following the declaration (everywhere but in class
 153     bodies, where they raise a compile error)
 154
 155     - Interpret some node structures into Python runtime values.
 156     Some nodes take compile-time arguments (currently:
 157     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 158     which should be interpreted. This happens in a general way
 159     and other steps should be taken to ensure validity.
 160
 161     Type arguments cannot be interpreted in this way.
 162
 163     - For __cythonbufferdefaults__ the arguments are checked for
 164     validity.
 165
 166     TemplatedTypeNode has its directives interpreted:
 167     Any first positional argument goes into the "dtype" attribute,
 168     any "ndim" keyword argument goes into the "ndim" attribute and
 169     so on. Also it is checked that the directive combination is valid.
 170     - __cythonbufferdefaults__ attributes are parsed and put into the
 171     type information.
 172
 173     Note: Currently Parsing.py does a lot of interpretation and
 174     reorganization that can be refactored into this transform
 175     if a more pure Abstract Syntax Tree is wanted.
 176     """
 177
 178     def __init__(self, context):
 179         super(PostParse, self).__init__(context)
 180         self.specialattribute_handlers = {
 181             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 182         }
 183
 184     def visit_ModuleNode(self, node):
 185         self.lambda_counter = 1
 186         return super(PostParse, self).visit_ModuleNode(node)
 187
 188     def visit_LambdaNode(self, node):
 189         # unpack a lambda expression into the corresponding DefNode
 190         lambda_id = self.lambda_counter
 191         self.lambda_counter += 1
 192         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 193
 194         body = Nodes.ReturnStatNode(
 195             node.result_expr.pos, value = node.result_expr)
 196         node.def_node = Nodes.DefNode(
 197             node.pos, name=node.name, lambda_name=node.lambda_name,
 198             args=node.args, star_arg=node.star_arg,
 199             starstar_arg=node.starstar_arg,
 200             body=body)
 201         self.visitchildren(node)
 202         return node
 203
 204     # cdef variables
 205     def handle_bufferdefaults(self, decl):
 206         if not isinstance(decl.default, ExprNodes.DictNode):
 207             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 208         self.scope_node.buffer_defaults_node = decl.default
 209         self.scope_node.buffer_defaults_pos = decl.pos
 210
 211     def visit_CVarDefNode(self, node):
 212         # This assumes only plain names and pointers are assignable on
 213         # declaration. Also, it makes use of the fact that a cdef decl
 214         # must appear before the first use, so we don't have to deal with
 215         # "i = 3; cdef int i = i" and can simply move the nodes around.
 216         try:
 217             self.visitchildren(node)
 218             stats = [node]
 219             newdecls = []
 220             for decl in node.declarators:
 221                 declbase = decl
 222                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 223                     declbase = declbase.base
 224                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 225                     if declbase.default is not None:
 226                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 227                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 228                                 handler = self.specialattribute_handlers.get(decl.name)
 229                                 if handler:
 230                                     if decl is not declbase:
 231                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 232                                     handler(decl)
 233                                     continue # Remove declaration
 234                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 235                         first_assignment = self.scope_type != 'module'
 236                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 237                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 238                             rhs=declbase.default, first=first_assignment))
 239                         declbase.default = None
 240                 newdecls.append(decl)
 241             node.declarators = newdecls
 242             return stats
 243         except PostParseError, e:
 244             # An error in a cdef clause is ok, simply remove the declaration
 245             # and try to move on to report more errors
 246             self.context.nonfatal_error(e)
 247             return None
 248
 249     # Split parallel assignments (a,b = b,a) into separate partial
 250     # assignments that are executed rhs-first using temps.  This
 251     # restructuring must be applied before type analysis so that known
 252     # types on rhs and lhs can be matched directly.  It is required in
 253     # the case that the types cannot be coerced to a Python type in
 254     # order to assign from a tuple.
 255
 256     def visit_SingleAssignmentNode(self, node):
 257         self.visitchildren(node)
 258         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 259
 260     def visit_CascadedAssignmentNode(self, node):
 261         self.visitchildren(node)
 262         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 263
 264     def _visit_assignment_node(self, node, expr_list):
 265         """Flatten parallel assignments into separate single
 266         assignments or cascaded assignments.
 267         """
 268         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 269             # no parallel assignments => nothing to do
 270             return node
 271
 272         expr_list_list = []
 273         flatten_parallel_assignments(expr_list, expr_list_list)
 274         temp_refs = []
 275         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 276
 277         nodes = []
 278         for expr_list in expr_list_list:
 279             lhs_list = expr_list[:-1]
 280             rhs = expr_list[-1]
 281             if len(lhs_list) == 1:
 282                 node = Nodes.SingleAssignmentNode(rhs.pos,
 283                     lhs = lhs_list[0], rhs = rhs)
 284             else:
 285                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 286                     lhs_list = lhs_list, rhs = rhs)
 287             nodes.append(node)
 288
 289         if len(nodes) == 1:
 290             assign_node = nodes[0]
 291         else:
 292             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 293
 294         if temp_refs:
 295             duplicates_and_temps = [ (temp.expression, temp)
 296                                      for temp in temp_refs ]
 297             sort_common_subsequences(duplicates_and_temps)
 298             for _, temp_ref in duplicates_and_temps[::-1]:
 299                 assign_node = LetNode(temp_ref, assign_node)
 300
 301         return assign_node
 302
 303 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 304     """Replace rhs items by LetRefNodes if they appear more than once.
 305     Creates a sequence of LetRefNodes that set up the required temps
 306     and appends them to ref_node_sequence.  The input list is modified
 307     in-place.
 308     """
 309     seen_nodes = cython.set()
 310     ref_nodes = {}
 311     def find_duplicates(node):
 312         if node.is_literal or node.is_name:
 313             # no need to replace those; can't include attributes here
 314             # as their access is not necessarily side-effect free
 315             return
 316         if node in seen_nodes:
 317             if node not in ref_nodes:
 318                 ref_node = LetRefNode(node)
 319                 ref_nodes[node] = ref_node
 320                 ref_node_sequence.append(ref_node)
 321         else:
 322             seen_nodes.add(node)
 323             if node.is_sequence_constructor:
 324                 for item in node.args:
 325                     find_duplicates(item)
 326
 327     for expr_list in expr_list_list:
 328         rhs = expr_list[-1]
 329         find_duplicates(rhs)
 330     if not ref_nodes:
 331         return
 332
 333     def substitute_nodes(node):
 334         if node in ref_nodes:
 335             return ref_nodes[node]
 336         elif node.is_sequence_constructor:
 337             node.args = list(map(substitute_nodes, node.args))
 338         return node
 339
 340     # replace nodes inside of the common subexpressions
 341     for node in ref_nodes:
 342         if node.is_sequence_constructor:
 343             node.args = list(map(substitute_nodes, node.args))
 344
 345     # replace common subexpressions on all rhs items
 346     for expr_list in expr_list_list:
 347         expr_list[-1] = substitute_nodes(expr_list[-1])
 348
 349 def sort_common_subsequences(items):
 350     """Sort items/subsequences so that all items and subsequences that
 351     an item contains appear before the item itself.  This is needed
 352     because each rhs item must only be evaluated once, so its value
 353     must be evaluated first and then reused when packing sequences
 354     that contain it.
 355
 356     This implies a partial order, and the sort must be stable to
 357     preserve the original order as much as possible, so we use a
 358     simple insertion sort (which is very fast for short sequences, the
 359     normal case in practice).
 360     """
 361     def contains(seq, x):
 362         for item in seq:
 363             if item is x:
 364                 return True
 365             elif item.is_sequence_constructor and contains(item.args, x):
 366                 return True
 367         return False
 368     def lower_than(a,b):
 369         return b.is_sequence_constructor and contains(b.args, a)
 370
 371     for pos, item in enumerate(items):
 372         key = item[1] # the ResultRefNode which has already been injected into the sequences
 373         new_pos = pos
 374         for i in xrange(pos-1, -1, -1):
 375             if lower_than(key, items[i][0]):
 376                 new_pos = i
 377         if new_pos != pos:
 378             for i in xrange(pos, new_pos, -1):
 379                 items[i] = items[i-1]
 380             items[new_pos] = item
 381
 382 def flatten_parallel_assignments(input, output):
 383     #  The input is a list of expression nodes, representing the LHSs
 384     #  and RHS of one (possibly cascaded) assignment statement.  For
 385     #  sequence constructors, rearranges the matching parts of both
 386     #  sides into a list of equivalent assignments between the
 387     #  individual elements.  This transformation is applied
 388     #  recursively, so that nested structures get matched as well.
 389     rhs = input[-1]
 390     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 391         output.append(input)
 392         return
 393
 394     complete_assignments = []
 395
 396     rhs_size = len(rhs.args)
 397     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 398     starred_assignments = []
 399     for lhs in input[:-1]:
 400         if not lhs.is_sequence_constructor:
 401             if lhs.is_starred:
 402                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 403             complete_assignments.append(lhs)
 404             continue
 405         lhs_size = len(lhs.args)
 406         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 407         if starred_targets > 1:
 408             error(lhs.pos, "more than 1 starred expression in assignment")
 409             output.append([lhs,rhs])
 410             continue
 411         elif lhs_size - starred_targets > rhs_size:
 412             error(lhs.pos, "need more than %d value%s to unpack"
 413                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 414             output.append([lhs,rhs])
 415             continue
 416         elif starred_targets:
 417             map_starred_assignment(lhs_targets, starred_assignments,
 418                                    lhs.args, rhs.args)
 419         elif lhs_size < rhs_size:
 420             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 421                   % (lhs_size, rhs_size))
 422             output.append([lhs,rhs])
 423             continue
 424         else:
 425             for targets, expr in zip(lhs_targets, lhs.args):
 426                 targets.append(expr)
 427
 428     if complete_assignments:
 429         complete_assignments.append(rhs)
 430         output.append(complete_assignments)
 431
 432     # recursively flatten partial assignments
 433     for cascade, rhs in zip(lhs_targets, rhs.args):
 434         if cascade:
 435             cascade.append(rhs)
 436             flatten_parallel_assignments(cascade, output)
 437
 438     # recursively flatten starred assignments
 439     for cascade in starred_assignments:
 440         if cascade[0].is_sequence_constructor:
 441             flatten_parallel_assignments(cascade, output)
 442         else:
 443             output.append(cascade)
 444
 445 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 446     # Appends the fixed-position LHS targets to the target list that
 447     # appear left and right of the starred argument.
 448     #
 449     # The starred_assignments list receives a new tuple
 450     # (lhs_target, rhs_values_list) that maps the remaining arguments
 451     # (those that match the starred target) to a list.
 452
 453     # left side of the starred target
 454     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 455         if expr.is_starred:
 456             starred = i
 457             lhs_remaining = len(lhs_args) - i - 1
 458             break
 459         targets.append(expr)
 460     else:
 461         raise InternalError("no starred arg found when splitting starred assignment")
 462
 463     # right side of the starred target
 464     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 465                                             lhs_args[-lhs_remaining:])):
 466         targets.append(expr)
 467
 468     # the starred target itself, must be assigned a (potentially empty) list
 469     target = lhs_args[starred].target # unpack starred node
 470     starred_rhs = rhs_args[starred:]
 471     if lhs_remaining:
 472         starred_rhs = starred_rhs[:-lhs_remaining]
 473     if starred_rhs:
 474         pos = starred_rhs[0].pos
 475     else:
 476         pos = target.pos
 477     starred_assignments.append([
 478         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 479
 480
 481 class PxdPostParse(CythonTransform, SkipDeclarations):
 482     """
 483     Basic interpretation/validity checking that should only be
 484     done on pxd trees.
 485
 486     A lot of this checking currently happens in the parser; but
 487     what is listed below happens here.
 488
 489     - "def" functions are let through only if they fill the
 490     getbuffer/releasebuffer slots
 491
 492     - cdef functions are let through only if they are on the
 493     top level and are declared "inline"
 494     """
 495     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 496     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 497
 498     def __call__(self, node):
 499         self.scope_type = 'pxd'
 500         return super(PxdPostParse, self).__call__(node)
 501
 502     def visit_CClassDefNode(self, node):
 503         old = self.scope_type
 504         self.scope_type = 'cclass'
 505         self.visitchildren(node)
 506         self.scope_type = old
 507         return node
 508
 509     def visit_FuncDefNode(self, node):
 510         # FuncDefNode always come with an implementation (without
 511         # an imp they are CVarDefNodes..)
 512         err = self.ERR_INLINE_ONLY
 513
 514         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 515             and node.name in ('__getbuffer__', '__releasebuffer__')):
 516             err = None # allow these slots
 517
 518         if isinstance(node, Nodes.CFuncDefNode):
 519             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 520                 node.inline_in_pxd = True
 521                 if node.visibility != 'private':
 522                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 523                 elif node.api:
 524                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 525                 else:
 526                     err = None # allow inline function
 527             else:
 528                 err = self.ERR_INLINE_ONLY
 529
 530         if err:
 531             self.context.nonfatal_error(PostParseError(node.pos, err))
 532             return None
 533         else:
 534             return node
 535
 536 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 537     """
 538     After parsing, directives can be stored in a number of places:
 539     - #cython-comments at the top of the file (stored in ModuleNode)
 540     - Command-line arguments overriding these
 541     - @cython.directivename decorators
 542     - with cython.directivename: statements
 543
 544     This transform is responsible for interpreting these various sources
 545     and store the directive in two ways:
 546     - Set the directives attribute of the ModuleNode for global directives.
 547     - Use a CompilerDirectivesNode to override directives for a subtree.
 548
 549     (The first one is primarily to not have to modify with the tree
 550     structure, so that ModuleNode stay on top.)
 551
 552     The directives are stored in dictionaries from name to value in effect.
 553     Each such dictionary is always filled in for all possible directives,
 554     using default values where no value is given by the user.
 555
 556     The available directives are controlled in Options.py.
 557
 558     Note that we have to run this prior to analysis, and so some minor
 559     duplication of functionality has to occur: We manually track cimports
 560     and which names the "cython" module may have been imported to.
 561     """
 562     unop_method_nodes = {
 563         'typeof': ExprNodes.TypeofNode,
 564
 565         'operator.address': ExprNodes.AmpersandNode,
 566         'operator.dereference': ExprNodes.DereferenceNode,
 567         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 568         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 569         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 570         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 571
 572         # For backwards compatability.
 573         'address': ExprNodes.AmpersandNode,
 574     }
 575
 576     binop_method_nodes = {
 577         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 578     }
 579
 580     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 581                                   'cast', 'pointer', 'compiled', 'NULL'])
 582     special_methods.update(unop_method_nodes.keys())
 583
 584     def __init__(self, context, compilation_directive_defaults):
 585         super(InterpretCompilerDirectives, self).__init__(context)
 586         self.compilation_directive_defaults = {}
 587         for key, value in compilation_directive_defaults.items():
 588             self.compilation_directive_defaults[unicode(key)] = value
 589         self.cython_module_names = cython.set()
 590         self.directive_names = {}
 591
 592     def check_directive_scope(self, pos, directive, scope):
 593         legal_scopes = Options.directive_scopes.get(directive, None)
 594         if legal_scopes and scope not in legal_scopes:
 595             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 596                                         'is not allowed in %s scope' % (directive, scope)))
 597             return False
 598         else:
 599             return True
 600
 601     # Set up processing and handle the cython: comments.
 602     def visit_ModuleNode(self, node):
 603         for key, value in node.directive_comments.items():
 604             if not self.check_directive_scope(node.pos, key, 'module'):
 605                 self.wrong_scope_error(node.pos, key, 'module')
 606                 del node.directive_comments[key]
 607
 608         directives = copy.copy(Options.directive_defaults)
 609         directives.update(self.compilation_directive_defaults)
 610         directives.update(node.directive_comments)
 611         self.directives = directives
 612         node.directives = directives
 613         self.visitchildren(node)
 614         node.cython_module_names = self.cython_module_names
 615         return node
 616
 617     # The following four functions track imports and cimports that
 618     # begin with "cython"
 619     def is_cython_directive(self, name):
 620         return (name in Options.directive_types or
 621                 name in self.special_methods or
 622                 PyrexTypes.parse_basic_type(name))
 623
 624     def visit_CImportStatNode(self, node):
 625         if node.module_name == u"cython":
 626             self.cython_module_names.add(node.as_name or u"cython")
 627         elif node.module_name.startswith(u"cython."):
 628             if node.as_name:
 629                 self.directive_names[node.as_name] = node.module_name[7:]
 630             else:
 631                 self.cython_module_names.add(u"cython")
 632             # if this cimport was a compiler directive, we don't
 633             # want to leave the cimport node sitting in the tree
 634             return None
 635         return node
 636
 637     def visit_FromCImportStatNode(self, node):
 638         if (node.module_name == u"cython") or \
 639                node.module_name.startswith(u"cython."):
 640             submodule = (node.module_name + u".")[7:]
 641             newimp = []
 642             for pos, name, as_name, kind in node.imported_names:
 643                 full_name = submodule + name
 644                 if self.is_cython_directive(full_name):
 645                     if as_name is None:
 646                         as_name = full_name
 647                     self.directive_names[as_name] = full_name
 648                     if kind is not None:
 649                         self.context.nonfatal_error(PostParseError(pos,
 650                             "Compiler directive imports must be plain imports"))
 651                 else:
 652                     newimp.append((pos, name, as_name, kind))
 653             if not newimp:
 654                 return None
 655             node.imported_names = newimp
 656         return node
 657
 658     def visit_FromImportStatNode(self, node):
 659         if (node.module.module_name.value == u"cython") or \
 660                node.module.module_name.value.startswith(u"cython."):
 661             submodule = (node.module.module_name.value + u".")[7:]
 662             newimp = []
 663             for name, name_node in node.items:
 664                 full_name = submodule + name
 665                 if self.is_cython_directive(full_name):
 666                     self.directive_names[name_node.name] = full_name
 667                 else:
 668                     newimp.append((name, name_node))
 669             if not newimp:
 670                 return None
 671             node.items = newimp
 672         return node
 673
 674     def visit_SingleAssignmentNode(self, node):
 675         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 676                 node.rhs.module_name.value == u'cython'):
 677             node = Nodes.CImportStatNode(node.pos,
 678                                          module_name = u'cython',
 679                                          as_name = node.lhs.name)
 680             self.visit_CImportStatNode(node)
 681         else:
 682             self.visitchildren(node)
 683         return node
 684
 685     def visit_NameNode(self, node):
 686         if node.name in self.cython_module_names:
 687             node.is_cython_module = True
 688         else:
 689             node.cython_attribute = self.directive_names.get(node.name)
 690         return node
 691
 692     def try_to_parse_directives(self, node):
 693         # If node is the contents of an directive (in a with statement or
 694         # decorator), returns a list of (directivename, value) pairs.
 695         # Otherwise, returns None
 696         if isinstance(node, ExprNodes.CallNode):
 697             self.visit(node.function)
 698             optname = node.function.as_cython_attribute()
 699             if optname:
 700                 directivetype = Options.directive_types.get(optname)
 701                 if directivetype:
 702                     args, kwds = node.explicit_args_kwds()
 703                     directives = []
 704                     key_value_pairs = []
 705                     if kwds is not None and directivetype is not dict:
 706                         for keyvalue in kwds.key_value_pairs:
 707                             key, value = keyvalue
 708                             sub_optname = "%s.%s" % (optname, key.value)
 709                             if Options.directive_types.get(sub_optname):
 710                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 711                             else:
 712                                 key_value_pairs.append(keyvalue)
 713                         if not key_value_pairs:
 714                             kwds = None
 715                         else:
 716                             kwds.key_value_pairs = key_value_pairs
 717                         if directives and not kwds and not args:
 718                             return directives
 719                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 720                     return directives
 721         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 722             self.visit(node)
 723             optname = node.as_cython_attribute()
 724             if optname:
 725                 directivetype = Options.directive_types.get(optname)
 726                 if directivetype is bool:
 727                     return [(optname, True)]
 728                 elif directivetype is None:
 729                     return [(optname, None)]
 730                 else:
 731                     raise PostParseError(
 732                         node.pos, "The '%s' directive should be used as a function call." % optname)
 733         return None
 734
 735     def try_to_parse_directive(self, optname, args, kwds, pos):
 736         directivetype = Options.directive_types.get(optname)
 737         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 738             return optname, Options.directive_defaults[optname]
 739         elif directivetype is bool:
 740             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 741                 raise PostParseError(pos,
 742                     'The %s directive takes one compile-time boolean argument' % optname)
 743             return (optname, args[0].value)
 744         elif directivetype is str:
 745             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 746                                                                               ExprNodes.UnicodeNode)):
 747                 raise PostParseError(pos,
 748                     'The %s directive takes one compile-time string argument' % optname)
 749             return (optname, str(args[0].value))
 750         elif directivetype is dict:
 751             if len(args) != 0:
 752                 raise PostParseError(pos,
 753                     'The %s directive takes no prepositional arguments' % optname)
 754             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 755         elif directivetype is list:
 756             if kwds and len(kwds) != 0:
 757                 raise PostParseError(pos,
 758                     'The %s directive takes no keyword arguments' % optname)
 759             return optname, [ str(arg.value) for arg in args ]
 760         else:
 761             assert False
 762
 763     def visit_with_directives(self, body, directives):
 764         olddirectives = self.directives
 765         newdirectives = copy.copy(olddirectives)
 766         newdirectives.update(directives)
 767         self.directives = newdirectives
 768         assert isinstance(body, Nodes.StatListNode), body
 769         retbody = self.visit_Node(body)
 770         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 771                                                  directives=newdirectives)
 772         self.directives = olddirectives
 773         return directive
 774
 775     # Handle decorators
 776     def visit_FuncDefNode(self, node):
 777         directives = self._extract_directives(node, 'function')
 778         if not directives:
 779             return self.visit_Node(node)
 780         body = Nodes.StatListNode(node.pos, stats=[node])
 781         return self.visit_with_directives(body, directives)
 782
 783     def visit_CVarDefNode(self, node):
 784         if not node.decorators:
 785             return node
 786         for dec in node.decorators:
 787             for directive in self.try_to_parse_directives(dec.decorator) or ():
 788                 if directive is not None and directive[0] == u'locals':
 789                     node.directive_locals = directive[1]
 790                 else:
 791                     self.context.nonfatal_error(PostParseError(dec.pos,
 792                         "Cdef functions can only take cython.locals() decorator."))
 793         return node
 794
 795     def visit_CClassDefNode(self, node):
 796         directives = self._extract_directives(node, 'cclass')
 797         if not directives:
 798             return self.visit_Node(node)
 799         body = Nodes.StatListNode(node.pos, stats=[node])
 800         return self.visit_with_directives(body, directives)
 801
 802     def visit_PyClassDefNode(self, node):
 803         directives = self._extract_directives(node, 'class')
 804         if not directives:
 805             return self.visit_Node(node)
 806         body = Nodes.StatListNode(node.pos, stats=[node])
 807         return self.visit_with_directives(body, directives)
 808
 809     def _extract_directives(self, node, scope_name):
 810         if not node.decorators:
 811             return {}
 812         # Split the decorators into two lists -- real decorators and directives
 813         directives = []
 814         realdecs = []
 815         for dec in node.decorators:
 816             new_directives = self.try_to_parse_directives(dec.decorator)
 817             if new_directives is not None:
 818                 for directive in new_directives:
 819                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 820                         directives.append(directive)
 821             else:
 822                 realdecs.append(dec)
 823         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 824             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 825         else:
 826             node.decorators = realdecs
 827         # merge or override repeated directives
 828         optdict = {}
 829         directives.reverse() # Decorators coming first take precedence
 830         for directive in directives:
 831             name, value = directive
 832             if name in optdict:
 833                 old_value = optdict[name]
 834                 # keywords and arg lists can be merged, everything
 835                 # else overrides completely
 836                 if isinstance(old_value, dict):
 837                     old_value.update(value)
 838                 elif isinstance(old_value, list):
 839                     old_value.extend(value)
 840                 else:
 841                     optdict[name] = value
 842             else:
 843                 optdict[name] = value
 844         return optdict
 845
 846     # Handle with statements
 847     def visit_WithStatNode(self, node):
 848         directive_dict = {}
 849         for directive in self.try_to_parse_directives(node.manager) or []:
 850             if directive is not None:
 851                 if node.target is not None:
 852                     self.context.nonfatal_error(
 853                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 854                 else:
 855                     name, value = directive
 856                     if name == 'nogil':
 857                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 858                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 859                         return self.visit_Node(node)
 860                     if self.check_directive_scope(node.pos, name, 'with statement'):
 861                         directive_dict[name] = value
 862         if directive_dict:
 863             return self.visit_with_directives(node.body, directive_dict)
 864         return self.visit_Node(node)
 865
 866 class WithTransform(CythonTransform, SkipDeclarations):
 867
 868     # EXCINFO is manually set to a variable that contains
 869     # the exc_info() tuple that can be generated by the enclosing except
 870     # statement.
 871     template_without_target = TreeFragment(u"""
 872         MGR = EXPR
 873         EXIT = MGR.__exit__
 874         MGR.__enter__()
 875         EXC = True
 876         try:
 877             try:
 878                 EXCINFO = None
 879                 BODY
 880             except:
 881                 EXC = False
 882                 if not EXIT(*EXCINFO):
 883                     raise
 884         finally:
 885             if EXC:
 886                 EXIT(None, None, None)
 887     """, temps=[u'MGR', u'EXC', u"EXIT"],
 888     pipeline=[NormalizeTree(None)])
 889
 890     template_with_target = TreeFragment(u"""
 891         MGR = EXPR
 892         EXIT = MGR.__exit__
 893         VALUE = MGR.__enter__()
 894         EXC = True
 895         try:
 896             try:
 897                 EXCINFO = None
 898                 TARGET = VALUE
 899                 BODY
 900             except:
 901                 EXC = False
 902                 if not EXIT(*EXCINFO):
 903                     raise
 904         finally:
 905             if EXC:
 906                 EXIT(None, None, None)
 907             MGR = EXIT = VALUE = EXC = None
 908
 909     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 910     pipeline=[NormalizeTree(None)])
 911
 912     def visit_WithStatNode(self, node):
 913         # TODO: Cleanup badly needed
 914         TemplateTransform.temp_name_counter += 1
 915         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 916
 917         self.visitchildren(node, ['body'])
 918         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 919         if node.target is not None:
 920             result = self.template_with_target.substitute({
 921                 u'EXPR' : node.manager,
 922                 u'BODY' : node.body,
 923                 u'TARGET' : node.target,
 924                 u'EXCINFO' : excinfo_temp
 925                 }, pos=node.pos)
 926         else:
 927             result = self.template_without_target.substitute({
 928                 u'EXPR' : node.manager,
 929                 u'BODY' : node.body,
 930                 u'EXCINFO' : excinfo_temp
 931                 }, pos=node.pos)
 932
 933         # Set except excinfo target to EXCINFO
 934         try_except = result.stats[-1].body.stats[-1]
 935         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 936 #            excinfo_temp.ref(node.pos))
 937
 938 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 939 #            node.pos, temps=[excinfo_temp], body=try_except)
 940
 941         return result
 942
 943     def visit_ExprNode(self, node):
 944         # With statements are never inside expressions.
 945         return node
 946
 947
 948 class DecoratorTransform(CythonTransform, SkipDeclarations):
 949
 950     def visit_DefNode(self, func_node):
 951         self.visitchildren(func_node)
 952         if not func_node.decorators:
 953             return func_node
 954         return self._handle_decorators(
 955             func_node, func_node.name)
 956
 957     def visit_CClassDefNode(self, class_node):
 958         # This doesn't currently work, so it's disabled.
 959         #
 960         # Problem: assignments to cdef class names do not work.  They
 961         # would require an additional check anyway, as the extension
 962         # type must not change its C type, so decorators cannot
 963         # replace an extension type, just alter it and return it.
 964
 965         self.visitchildren(class_node)
 966         if not class_node.decorators:
 967             return class_node
 968         error(class_node.pos,
 969               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 970         return class_node
 971         #return self._handle_decorators(
 972         #    class_node, class_node.class_name)
 973
 974     def visit_ClassDefNode(self, class_node):
 975         self.visitchildren(class_node)
 976         if not class_node.decorators:
 977             return class_node
 978         return self._handle_decorators(
 979             class_node, class_node.name)
 980
 981     def _handle_decorators(self, node, name):
 982         decorator_result = ExprNodes.NameNode(node.pos, name = name)
 983         for decorator in node.decorators[::-1]:
 984             decorator_result = ExprNodes.SimpleCallNode(
 985                 decorator.pos,
 986                 function = decorator.decorator,
 987                 args = [decorator_result])
 988
 989         name_node = ExprNodes.NameNode(node.pos, name = name)
 990         reassignment = Nodes.SingleAssignmentNode(
 991             node.pos,
 992             lhs = name_node,
 993             rhs = decorator_result)
 994         return [node, reassignment]
 995
 996
 997 class AnalyseDeclarationsTransform(CythonTransform):
 998
 999     basic_property = TreeFragment(u"""
1000 property NAME:
1001     def __get__(self):
1002         return ATTR
1003     def __set__(self, value):
1004         ATTR = value
1005     """, level='c_class')
1006     basic_pyobject_property = TreeFragment(u"""
1007 property NAME:
1008     def __get__(self):
1009         return ATTR
1010     def __set__(self, value):
1011         ATTR = value
1012     def __del__(self):
1013         ATTR = None
1014     """, level='c_class')
1015     basic_property_ro = TreeFragment(u"""
1016 property NAME:
1017     def __get__(self):
1018         return ATTR
1019     """, level='c_class')
1020
1021     def __call__(self, root):
1022         self.env_stack = [root.scope]
1023         # needed to determine if a cdef var is declared after it's used.
1024         self.seen_vars_stack = []
1025         return super(AnalyseDeclarationsTransform, self).__call__(root)
1026
1027     def visit_NameNode(self, node):
1028         self.seen_vars_stack[-1].add(node.name)
1029         return node
1030
1031     def visit_ModuleNode(self, node):
1032         self.seen_vars_stack.append(cython.set())
1033         node.analyse_declarations(self.env_stack[-1])
1034         self.visitchildren(node)
1035         self.seen_vars_stack.pop()
1036         return node
1037
1038     def visit_LambdaNode(self, node):
1039         node.analyse_declarations(self.env_stack[-1])
1040         self.visitchildren(node)
1041         return node
1042
1043     def visit_ClassDefNode(self, node):
1044         self.env_stack.append(node.scope)
1045         self.visitchildren(node)
1046         self.env_stack.pop()
1047         return node
1048
1049     def visit_CClassDefNode(self, node):
1050         node = self.visit_ClassDefNode(node)
1051         if node.scope and node.scope.implemented:
1052             stats = []
1053             for entry in node.scope.var_entries:
1054                 if entry.needs_property:
1055                     property = self.create_Property(entry)
1056                     property.analyse_declarations(node.scope)
1057                     self.visit(property)
1058                     stats.append(property)
1059             if stats:
1060                 node.body.stats += stats
1061         return node
1062
1063     def visit_FuncDefNode(self, node):
1064         self.seen_vars_stack.append(cython.set())
1065         lenv = node.local_scope
1066         node.body.analyse_control_flow(lenv) # this will be totally refactored
1067         node.declare_arguments(lenv)
1068         for var, type_node in node.directive_locals.items():
1069             if not lenv.lookup_here(var):   # don't redeclare args
1070                 type = type_node.analyse_as_type(lenv)
1071                 if type:
1072                     lenv.declare_var(var, type, type_node.pos)
1073                 else:
1074                     error(type_node.pos, "Not a type")
1075         node.body.analyse_declarations(lenv)
1076         self.env_stack.append(lenv)
1077         self.visitchildren(node)
1078         self.env_stack.pop()
1079         self.seen_vars_stack.pop()
1080         return node
1081
1082     def visit_ScopedExprNode(self, node):
1083         env = self.env_stack[-1]
1084         node.analyse_declarations(env)
1085         # the node may or may not have a local scope
1086         if node.has_local_scope:
1087             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1088             self.env_stack.append(node.expr_scope)
1089             node.analyse_scoped_declarations(node.expr_scope)
1090             self.visitchildren(node)
1091             self.env_stack.pop()
1092             self.seen_vars_stack.pop()
1093         else:
1094             node.analyse_scoped_declarations(env)
1095             self.visitchildren(node)
1096         return node
1097
1098     def visit_TempResultFromStatNode(self, node):
1099         self.visitchildren(node)
1100         node.analyse_declarations(self.env_stack[-1])
1101         return node
1102
1103     # Some nodes are no longer needed after declaration
1104     # analysis and can be dropped. The analysis was performed
1105     # on these nodes in a seperate recursive process from the
1106     # enclosing function or module, so we can simply drop them.
1107     def visit_CDeclaratorNode(self, node):
1108         # necessary to ensure that all CNameDeclaratorNodes are visited.
1109         self.visitchildren(node)
1110         return node
1111
1112     def visit_CTypeDefNode(self, node):
1113         return node
1114
1115     def visit_CBaseTypeNode(self, node):
1116         return None
1117
1118     def visit_CEnumDefNode(self, node):
1119         if node.visibility == 'public':
1120             return node
1121         else:
1122             return None
1123
1124     def visit_CStructOrUnionDefNode(self, node):
1125         return None
1126
1127     def visit_CNameDeclaratorNode(self, node):
1128         if node.name in self.seen_vars_stack[-1]:
1129             entry = self.env_stack[-1].lookup(node.name)
1130             if entry is None or entry.visibility != 'extern':
1131                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1132         self.visitchildren(node)
1133         return node
1134
1135     def visit_CVarDefNode(self, node):
1136         # to ensure all CNameDeclaratorNodes are visited.
1137         self.visitchildren(node)
1138         return None
1139
1140     def create_Property(self, entry):
1141         if entry.visibility == 'public':
1142             if entry.type.is_pyobject:
1143                 template = self.basic_pyobject_property
1144             else:
1145                 template = self.basic_property
1146         elif entry.visibility == 'readonly':
1147             template = self.basic_property_ro
1148         property = template.substitute({
1149                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1150                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1151                                                  attribute=entry.name),
1152             }, pos=entry.pos).stats[0]
1153         property.name = entry.name
1154         # ---------------------------------------
1155         # XXX This should go to AutoDocTransforms
1156         # ---------------------------------------
1157         if (Options.docstrings and
1158             self.current_directives['embedsignature']):
1159             attr_name = entry.name
1160             type_name = entry.type.declaration_code("", for_display=1)
1161             default_value = ''
1162             if not entry.type.is_pyobject:
1163                 type_name = "'%s'" % type_name
1164             elif entry.type.is_extension_type:
1165                 type_name = entry.type.module_name + '.' + type_name
1166             if entry.init is not None:
1167                 default_value = ' = ' + entry.init
1168             elif entry.init_to_none:
1169                 default_value = ' = ' + repr(None)
1170             docstring = attr_name + ': ' + type_name + default_value
1171             property.doc = EncodedString(docstring)
1172         # ---------------------------------------
1173         return property
1174
1175 class AnalyseExpressionsTransform(CythonTransform):
1176
1177     def visit_ModuleNode(self, node):
1178         node.scope.infer_types()
1179         node.body.analyse_expressions(node.scope)
1180         self.visitchildren(node)
1181         return node
1182
1183     def visit_FuncDefNode(self, node):
1184         node.local_scope.infer_types()
1185         node.body.analyse_expressions(node.local_scope)
1186         self.visitchildren(node)
1187         return node
1188
1189     def visit_ScopedExprNode(self, node):
1190         if node.has_local_scope:
1191             node.expr_scope.infer_types()
1192             node.analyse_scoped_expressions(node.expr_scope)
1193         self.visitchildren(node)
1194         return node
1195
1196 class ExpandInplaceOperators(EnvTransform):
1197
1198     def visit_InPlaceAssignmentNode(self, node):
1199         lhs = node.lhs
1200         rhs = node.rhs
1201         if lhs.type.is_cpp_class:
1202             # No getting around this exact operator here.
1203             return node
1204         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1205             # There is code to handle this case.
1206             return node
1207
1208         env = self.current_env()
1209         def side_effect_free_reference(node, setting=False):
1210             if isinstance(node, ExprNodes.NameNode):
1211                 return node, []
1212             elif node.type.is_pyobject and not setting:
1213                 node = LetRefNode(node)
1214                 return node, [node]
1215             elif isinstance(node, ExprNodes.IndexNode):
1216                 if node.is_buffer_access:
1217                     raise ValueError, "Buffer access"
1218                 base, temps = side_effect_free_reference(node.base)
1219                 index = LetRefNode(node.index)
1220                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1221             elif isinstance(node, ExprNodes.AttributeNode):
1222                 obj, temps = side_effect_free_reference(node.obj)
1223                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1224             else:
1225                 node = LetRefNode(node)
1226                 return node, [node]
1227         try:
1228             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1229         except ValueError:
1230             return node
1231         dup = lhs.__class__(**lhs.__dict__)
1232         binop = ExprNodes.binop_node(node.pos,
1233                                      operator = node.operator,
1234                                      operand1 = dup,
1235                                      operand2 = rhs,
1236                                      inplace=True)
1237         # Manually analyse types for new node.
1238         lhs.analyse_target_types(env)
1239         dup.analyse_types(env)
1240         binop.analyse_operation(env)
1241         node = Nodes.SingleAssignmentNode(
1242             node.pos,
1243             lhs = lhs,
1244             rhs=binop.coerce_to(lhs.type, env))
1245         # Use LetRefNode to avoid side effects.
1246         let_ref_nodes.reverse()
1247         for t in let_ref_nodes:
1248             node = LetNode(t, node)
1249         return node
1250
1251     def visit_ExprNode(self, node):
1252         # In-place assignments can't happen within an expression.
1253         return node
1254
1255
1256 class AlignFunctionDefinitions(CythonTransform):
1257     """
1258     This class takes the signatures from a .pxd file and applies them to
1259     the def methods in a .py file.
1260     """
1261
1262     def visit_ModuleNode(self, node):
1263         self.scope = node.scope
1264         self.directives = node.directives
1265         self.visitchildren(node)
1266         return node
1267
1268     def visit_PyClassDefNode(self, node):
1269         pxd_def = self.scope.lookup(node.name)
1270         if pxd_def:
1271             if pxd_def.is_cclass:
1272                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1273             else:
1274                 error(node.pos, "'%s' redeclared" % node.name)
1275                 error(pxd_def.pos, "previous declaration here")
1276                 return None
1277         else:
1278             return node
1279
1280     def visit_CClassDefNode(self, node, pxd_def=None):
1281         if pxd_def is None:
1282             pxd_def = self.scope.lookup(node.class_name)
1283         if pxd_def:
1284             outer_scope = self.scope
1285             self.scope = pxd_def.type.scope
1286         self.visitchildren(node)
1287         if pxd_def:
1288             self.scope = outer_scope
1289         return node
1290
1291     def visit_DefNode(self, node):
1292         pxd_def = self.scope.lookup(node.name)
1293         if pxd_def:
1294             if not pxd_def.is_cfunction:
1295                 error(node.pos, "'%s' redeclared" % node.name)
1296                 error(pxd_def.pos, "previous declaration here")
1297                 return None
1298             node = node.as_cfunction(pxd_def)
1299         elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1300             node = node.as_cfunction(scope=self.scope)
1301         # Enable this when internal def functions are allowed.
1302         # self.visitchildren(node)
1303         return node
1304
1305
1306 class MarkClosureVisitor(CythonTransform):
1307
1308     def visit_ModuleNode(self, node):
1309         self.needs_closure = False
1310         self.visitchildren(node)
1311         return node
1312
1313     def visit_FuncDefNode(self, node):
1314         self.needs_closure = False
1315         self.visitchildren(node)
1316         node.needs_closure = self.needs_closure
1317         self.needs_closure = True
1318         return node
1319
1320     def visit_CFuncDefNode(self, node):
1321         self.visit_FuncDefNode(node)
1322         if node.needs_closure:
1323             error(node.pos, "closures inside cdef functions not yet supported")
1324         return node
1325
1326     def visit_LambdaNode(self, node):
1327         self.needs_closure = False
1328         self.visitchildren(node)
1329         node.needs_closure = self.needs_closure
1330         self.needs_closure = True
1331         return node
1332
1333     def visit_ClassDefNode(self, node):
1334         self.visitchildren(node)
1335         self.needs_closure = True
1336         return node
1337
1338
1339 class CreateClosureClasses(CythonTransform):
1340     # Output closure classes in module scope for all functions
1341     # that really need it.
1342
1343     def __init__(self, context):
1344         super(CreateClosureClasses, self).__init__(context)
1345         self.path = []
1346         self.in_lambda = False
1347
1348     def visit_ModuleNode(self, node):
1349         self.module_scope = node.scope
1350         self.visitchildren(node)
1351         return node
1352
1353     def get_scope_use(self, node):
1354         from_closure = []
1355         in_closure = []
1356         for name, entry in node.local_scope.entries.items():
1357             if entry.from_closure:
1358                 from_closure.append((name, entry))
1359             elif entry.in_closure and not entry.from_closure:
1360                 in_closure.append((name, entry))
1361         return from_closure, in_closure
1362
1363     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1364         from_closure, in_closure = self.get_scope_use(node)
1365         in_closure.sort()
1366
1367         # Now from the begining
1368         node.needs_closure = False
1369         node.needs_outer_scope = False
1370
1371         func_scope = node.local_scope
1372         cscope = node.entry.scope
1373         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1374             cscope = cscope.outer_scope
1375
1376         if not from_closure and (self.path or inner_node):
1377             if not inner_node:
1378                 if not node.assmt:
1379                     raise InternalError, "DefNode does not have assignment node"
1380                 inner_node = node.assmt.rhs
1381             inner_node.needs_self_code = False
1382             node.needs_outer_scope = False
1383         # Simple cases
1384         if not in_closure and not from_closure:
1385             return
1386         elif not in_closure:
1387             func_scope.is_passthrough = True
1388             func_scope.scope_class = cscope.scope_class
1389             node.needs_outer_scope = True
1390             return
1391
1392         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1393
1394         entry = target_module_scope.declare_c_class(name = as_name,
1395             pos = node.pos, defining = True, implementing = True)
1396         func_scope.scope_class = entry
1397         class_scope = entry.type.scope
1398         class_scope.is_internal = True
1399         class_scope.directives = {'final': True}
1400
1401         if from_closure:
1402             assert cscope.is_closure_scope
1403             class_scope.declare_var(pos=node.pos,
1404                                     name=Naming.outer_scope_cname,
1405                                     cname=Naming.outer_scope_cname,
1406                                     type=cscope.scope_class.type,
1407                                     is_cdef=True)
1408             node.needs_outer_scope = True
1409         for name, entry in in_closure:
1410             class_scope.declare_var(pos=entry.pos,
1411                                     name=entry.name,
1412                                     cname=entry.cname,
1413                                     type=entry.type,
1414                                     is_cdef=True)
1415         node.needs_closure = True
1416         # Do it here because other classes are already checked
1417         target_module_scope.check_c_class(func_scope.scope_class)
1418
1419     def visit_LambdaNode(self, node):
1420         was_in_lambda = self.in_lambda
1421         self.in_lambda = True
1422         self.create_class_from_scope(node.def_node, self.module_scope, node)
1423         self.visitchildren(node)
1424         self.in_lambda = was_in_lambda
1425         return node
1426
1427     def visit_FuncDefNode(self, node):
1428         if self.in_lambda:
1429             self.visitchildren(node)
1430             return node
1431         if node.needs_closure or self.path:
1432             self.create_class_from_scope(node, self.module_scope)
1433             self.path.append(node)
1434             self.visitchildren(node)
1435             self.path.pop()
1436         return node
1437
1438
1439 class GilCheck(VisitorTransform):
1440     """
1441     Call `node.gil_check(env)` on each node to make sure we hold the
1442     GIL when we need it.  Raise an error when on Python operations
1443     inside a `nogil` environment.
1444     """
1445     def __call__(self, root):
1446         self.env_stack = [root.scope]
1447         self.nogil = False
1448         return super(GilCheck, self).__call__(root)
1449
1450     def visit_FuncDefNode(self, node):
1451         self.env_stack.append(node.local_scope)
1452         was_nogil = self.nogil
1453         self.nogil = node.local_scope.nogil
1454         if self.nogil and node.nogil_check:
1455             node.nogil_check(node.local_scope)
1456         self.visitchildren(node)
1457         self.env_stack.pop()
1458         self.nogil = was_nogil
1459         return node
1460
1461     def visit_GILStatNode(self, node):
1462         env = self.env_stack[-1]
1463         if self.nogil and node.nogil_check: node.nogil_check()
1464         was_nogil = self.nogil
1465         self.nogil = (node.state == 'nogil')
1466         self.visitchildren(node)
1467         self.nogil = was_nogil
1468         return node
1469
1470     def visit_Node(self, node):
1471         if self.env_stack and self.nogil and node.nogil_check:
1472             node.nogil_check(self.env_stack[-1])
1473         self.visitchildren(node)
1474         return node
1475
1476
1477 class TransformBuiltinMethods(EnvTransform):
1478
1479     def visit_SingleAssignmentNode(self, node):
1480         if node.declaration_only:
1481             return None
1482         else:
1483             self.visitchildren(node)
1484             return node
1485
1486     def visit_AttributeNode(self, node):
1487         self.visitchildren(node)
1488         return self.visit_cython_attribute(node)
1489
1490     def visit_NameNode(self, node):
1491         return self.visit_cython_attribute(node)
1492
1493     def visit_cython_attribute(self, node):
1494         attribute = node.as_cython_attribute()
1495         if attribute:
1496             if attribute == u'compiled':
1497                 node = ExprNodes.BoolNode(node.pos, value=True)
1498             elif attribute == u'NULL':
1499                 node = ExprNodes.NullNode(node.pos)
1500             elif attribute in (u'set', u'frozenset'):
1501                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1502                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1503             elif not PyrexTypes.parse_basic_type(attribute):
1504                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1505         return node
1506
1507     def visit_SimpleCallNode(self, node):
1508
1509         # locals builtin
1510         if isinstance(node.function, ExprNodes.NameNode):
1511             if node.function.name == 'locals':
1512                 lenv = self.current_env()
1513                 entry = lenv.lookup_here('locals')
1514                 if entry:
1515                     # not the builtin 'locals'
1516                     return node
1517                 if len(node.args) > 0:
1518                     error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1519                     return node
1520                 pos = node.pos
1521                 items = [ ExprNodes.DictItemNode(pos,
1522                                                  key=ExprNodes.StringNode(pos, value=var),
1523                                                  value=ExprNodes.NameNode(pos, name=var))
1524                           for var in lenv.entries ]
1525                 return ExprNodes.DictNode(pos, key_value_pairs=items)
1526
1527         # cython.foo
1528         function = node.function.as_cython_attribute()
1529         if function:
1530             if function in InterpretCompilerDirectives.unop_method_nodes:
1531                 if len(node.args) != 1:
1532                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1533                 else:
1534                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1535             elif function in InterpretCompilerDirectives.binop_method_nodes:
1536                 if len(node.args) != 2:
1537                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1538                 else:
1539                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1540             elif function == u'cast':
1541                 if len(node.args) != 2:
1542                     error(node.function.pos, u"cast() takes exactly two arguments")
1543                 else:
1544                     type = node.args[0].analyse_as_type(self.current_env())
1545                     if type:
1546                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1547                     else:
1548                         error(node.args[0].pos, "Not a type")
1549             elif function == u'sizeof':
1550                 if len(node.args) != 1:
1551                     error(node.function.pos, u"sizeof() takes exactly one argument")
1552                 else:
1553                     type = node.args[0].analyse_as_type(self.current_env())
1554                     if type:
1555                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1556                     else:
1557                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1558             elif function == 'cmod':
1559                 if len(node.args) != 2:
1560                     error(node.function.pos, u"cmod() takes exactly two arguments")
1561                 else:
1562                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1563                     node.cdivision = True
1564             elif function == 'cdiv':
1565                 if len(node.args) != 2:
1566                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1567                 else:
1568                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1569                     node.cdivision = True
1570             elif function == u'set':
1571                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1572             else:
1573                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1574
1575         self.visitchildren(node)
1576         return node
1577
1578
1579 class DebugTransform(CythonTransform):
1580     """
1581     Create debug information and all functions' visibility to extern in order
1582     to enable debugging.
1583     """
1584
1585     def __init__(self, context, options, result):
1586         super(DebugTransform, self).__init__(context)
1587         self.visited = cython.set()
1588         # our treebuilder and debug output writer
1589         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1590         self.tb = self.context.gdb_debug_outputwriter
1591         #self.c_output_file = options.output_file
1592         self.c_output_file = result.c_file
1593
1594         # tells visit_NameNode whether it should register step-into functions
1595         self.register_stepinto = False
1596
1597     def visit_ModuleNode(self, node):
1598         self.tb.module_name = node.full_module_name
1599         attrs = dict(
1600             module_name=node.full_module_name,
1601             filename=node.pos[0].filename,
1602             c_filename=self.c_output_file)
1603
1604         self.tb.start('Module', attrs)
1605
1606         # serialize functions
1607         self.tb.start('Functions')
1608         self.visitchildren(node)
1609         self.tb.end('Functions')
1610
1611         # 2.3 compatibility. Serialize global variables
1612         self.tb.start('Globals')
1613         entries = {}
1614
1615         for k, v in node.scope.entries.iteritems():
1616             if (v.qualified_name not in self.visited and not
1617                 v.name.startswith('__pyx_') and not
1618                 v.type.is_cfunction and not
1619                 v.type.is_extension_type):
1620                 entries[k]= v
1621
1622         self.serialize_local_variables(entries)
1623         self.tb.end('Globals')
1624         # self.tb.end('Module') # end Module after the line number mapping in
1625         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1626         return node
1627
1628     def visit_FuncDefNode(self, node):
1629         self.visited.add(node.local_scope.qualified_name)
1630         # node.entry.visibility = 'extern'
1631         if node.py_func is None:
1632             pf_cname = ''
1633         else:
1634             pf_cname = node.py_func.entry.func_cname
1635
1636         attrs = dict(
1637             name=node.entry.name,
1638             cname=node.entry.func_cname,
1639             pf_cname=pf_cname,
1640             qualified_name=node.local_scope.qualified_name,
1641             lineno=str(node.pos[1]))
1642
1643         self.tb.start('Function', attrs=attrs)
1644
1645         self.tb.start('Locals')
1646         self.serialize_local_variables(node.local_scope.entries)
1647         self.tb.end('Locals')
1648
1649         self.tb.start('Arguments')
1650         for arg in node.local_scope.arg_entries:
1651             self.tb.start(arg.name)
1652             self.tb.end(arg.name)
1653         self.tb.end('Arguments')
1654
1655         self.tb.start('StepIntoFunctions')
1656         self.register_stepinto = True
1657         self.visitchildren(node)
1658         self.register_stepinto = False
1659         self.tb.end('StepIntoFunctions')
1660         self.tb.end('Function')
1661
1662         return node
1663
1664     def visit_NameNode(self, node):
1665         if (self.register_stepinto and
1666             node.type.is_cfunction and
1667             getattr(node, 'is_called', False) and
1668             node.entry.func_cname is not None):
1669             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1670             # declared functions are not 'in_cinclude'.
1671             # This means we will list called 'cdef' functions as
1672             # "step into functions", but this is not an issue as they will be
1673             # recognized as Cython functions anyway.
1674             attrs = dict(name=node.entry.func_cname)
1675             self.tb.start('StepIntoFunction', attrs=attrs)
1676             self.tb.end('StepIntoFunction')
1677
1678         self.visitchildren(node)
1679         return node
1680
1681     def serialize_local_variables(self, entries):
1682         for entry in entries.values():
1683             if entry.type.is_pyobject:
1684                 vartype = 'PythonObject'
1685             else:
1686                 vartype = 'CObject'
1687
1688             cname = entry.cname
1689             # if entry.type.is_extension_type:
1690                 # cname = entry.type.typeptr_cname
1691
1692             if not entry.pos:
1693                 # this happens for variables that are not in the user's code,
1694                 # e.g. for the global __builtins__, __doc__, etc. We can just
1695                 # set the lineno to 0 for those.
1696                 lineno = '0'
1697             else:
1698                 lineno = str(entry.pos[1])
1699
1700             attrs = dict(
1701                 name=entry.name,
1702                 cname=cname,
1703                 qualified_name=entry.qualified_name,
1704                 type=vartype,
1705                 lineno=lineno)
1706
1707             self.tb.start('LocalVar', attrs)
1708             self.tb.end('LocalVar')
1709