Cython/Compiler/ParseTreeTransforms.py

   1
   2 import cython
   3 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   4                Options=object, UtilNodes=object, ModuleNode=object,
   5                LetNode=object, LetRefNode=object, TreeFragment=object,
   6                TemplateTransform=object, EncodedString=object,
   7                error=object, warning=object, copy=object)
   8
   9 from Binding import Binding
  10 import PyrexTypes
  11 import Naming
  12 import ExprNodes
  13 import Nodes
  14 import Options
  15
  16 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  17 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  18 from Cython.Compiler.ModuleNode import ModuleNode
  19 from Cython.Compiler.UtilNodes import LetNode, LetRefNode
  20 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
  21 from Cython.Compiler.StringEncoding import EncodedString
  22 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  23
  24 import copy
  25
  26
  27 class NameNodeCollector(TreeVisitor):
  28     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  29     attribute.
  30     """
  31     def __init__(self):
  32         super(NameNodeCollector, self).__init__()
  33         self.name_nodes = []
  34
  35     def visit_NameNode(self, node):
  36         self.name_nodes.append(node)
  37
  38     def visit_Node(self, node):
  39         self._visitchildren(node, None)
  40
  41
  42 class SkipDeclarations(object):
  43     """
  44     Variable and function declarations can often have a deep tree structure,
  45     and yet most transformations don't need to descend to this depth.
  46
  47     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  48     is no need to use this for transformations after that point.
  49     """
  50     def visit_CTypeDefNode(self, node):
  51         return node
  52
  53     def visit_CVarDefNode(self, node):
  54         return node
  55
  56     def visit_CDeclaratorNode(self, node):
  57         return node
  58
  59     def visit_CBaseTypeNode(self, node):
  60         return node
  61
  62     def visit_CEnumDefNode(self, node):
  63         return node
  64
  65     def visit_CStructOrUnionDefNode(self, node):
  66         return node
  67
  68
  69 class NormalizeTree(CythonTransform):
  70     """
  71     This transform fixes up a few things after parsing
  72     in order to make the parse tree more suitable for
  73     transforms.
  74
  75     a) After parsing, blocks with only one statement will
  76     be represented by that statement, not by a StatListNode.
  77     When doing transforms this is annoying and inconsistent,
  78     as one cannot in general remove a statement in a consistent
  79     way and so on. This transform wraps any single statements
  80     in a StatListNode containing a single statement.
  81
  82     b) The PassStatNode is a noop and serves no purpose beyond
  83     plugging such one-statement blocks; i.e., once parsed a
  84 `    "pass" can just as well be represented using an empty
  85     StatListNode. This means less special cases to worry about
  86     in subsequent transforms (one always checks to see if a
  87     StatListNode has no children to see if the block is empty).
  88     """
  89
  90     def __init__(self, context):
  91         super(NormalizeTree, self).__init__(context)
  92         self.is_in_statlist = False
  93         self.is_in_expr = False
  94
  95     def visit_ExprNode(self, node):
  96         stacktmp = self.is_in_expr
  97         self.is_in_expr = True
  98         self.visitchildren(node)
  99         self.is_in_expr = stacktmp
 100         return node
 101
 102     def visit_StatNode(self, node, is_listcontainer=False):
 103         stacktmp = self.is_in_statlist
 104         self.is_in_statlist = is_listcontainer
 105         self.visitchildren(node)
 106         self.is_in_statlist = stacktmp
 107         if not self.is_in_statlist and not self.is_in_expr:
 108             return Nodes.StatListNode(pos=node.pos, stats=[node])
 109         else:
 110             return node
 111
 112     def visit_StatListNode(self, node):
 113         self.is_in_statlist = True
 114         self.visitchildren(node)
 115         self.is_in_statlist = False
 116         return node
 117
 118     def visit_ParallelAssignmentNode(self, node):
 119         return self.visit_StatNode(node, True)
 120
 121     def visit_CEnumDefNode(self, node):
 122         return self.visit_StatNode(node, True)
 123
 124     def visit_CStructOrUnionDefNode(self, node):
 125         return self.visit_StatNode(node, True)
 126
 127     # Eliminate PassStatNode
 128     def visit_PassStatNode(self, node):
 129         if not self.is_in_statlist:
 130             return Nodes.StatListNode(pos=node.pos, stats=[])
 131         else:
 132             return []
 133
 134     def visit_CDeclaratorNode(self, node):
 135         return node
 136
 137
 138 class PostParseError(CompileError): pass
 139
 140 # error strings checked by unit tests, so define them
 141 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 142 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 143 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 144 class PostParse(ScopeTrackingTransform):
 145     """
 146     Basic interpretation of the parse tree, as well as validity
 147     checking that can be done on a very basic level on the parse
 148     tree (while still not being a problem with the basic syntax,
 149     as such).
 150
 151     Specifically:
 152     - Default values to cdef assignments are turned into single
 153     assignments following the declaration (everywhere but in class
 154     bodies, where they raise a compile error)
 155
 156     - Interpret some node structures into Python runtime values.
 157     Some nodes take compile-time arguments (currently:
 158     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 159     which should be interpreted. This happens in a general way
 160     and other steps should be taken to ensure validity.
 161
 162     Type arguments cannot be interpreted in this way.
 163
 164     - For __cythonbufferdefaults__ the arguments are checked for
 165     validity.
 166
 167     TemplatedTypeNode has its directives interpreted:
 168     Any first positional argument goes into the "dtype" attribute,
 169     any "ndim" keyword argument goes into the "ndim" attribute and
 170     so on. Also it is checked that the directive combination is valid.
 171     - __cythonbufferdefaults__ attributes are parsed and put into the
 172     type information.
 173
 174     Note: Currently Parsing.py does a lot of interpretation and
 175     reorganization that can be refactored into this transform
 176     if a more pure Abstract Syntax Tree is wanted.
 177     """
 178
 179     def __init__(self, context):
 180         super(PostParse, self).__init__(context)
 181         self.specialattribute_handlers = {
 182             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 183         }
 184
 185     def visit_ModuleNode(self, node):
 186         self.lambda_counter = 1
 187         return super(PostParse, self).visit_ModuleNode(node)
 188
 189     def visit_LambdaNode(self, node):
 190         # unpack a lambda expression into the corresponding DefNode
 191         lambda_id = self.lambda_counter
 192         self.lambda_counter += 1
 193         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 194
 195         body = Nodes.ReturnStatNode(
 196             node.result_expr.pos, value = node.result_expr)
 197         node.def_node = Nodes.DefNode(
 198             node.pos, name=node.name, lambda_name=node.lambda_name,
 199             args=node.args, star_arg=node.star_arg,
 200             starstar_arg=node.starstar_arg,
 201             body=body)
 202         self.visitchildren(node)
 203         return node
 204
 205     # cdef variables
 206     def handle_bufferdefaults(self, decl):
 207         if not isinstance(decl.default, ExprNodes.DictNode):
 208             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 209         self.scope_node.buffer_defaults_node = decl.default
 210         self.scope_node.buffer_defaults_pos = decl.pos
 211
 212     def visit_CVarDefNode(self, node):
 213         # This assumes only plain names and pointers are assignable on
 214         # declaration. Also, it makes use of the fact that a cdef decl
 215         # must appear before the first use, so we don't have to deal with
 216         # "i = 3; cdef int i = i" and can simply move the nodes around.
 217         try:
 218             self.visitchildren(node)
 219             stats = [node]
 220             newdecls = []
 221             for decl in node.declarators:
 222                 declbase = decl
 223                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 224                     declbase = declbase.base
 225                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 226                     if declbase.default is not None:
 227                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 228                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 229                                 handler = self.specialattribute_handlers.get(decl.name)
 230                                 if handler:
 231                                     if decl is not declbase:
 232                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 233                                     handler(decl)
 234                                     continue # Remove declaration
 235                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 236                         first_assignment = self.scope_type != 'module'
 237                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 238                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 239                             rhs=declbase.default, first=first_assignment))
 240                         declbase.default = None
 241                 newdecls.append(decl)
 242             node.declarators = newdecls
 243             return stats
 244         except PostParseError, e:
 245             # An error in a cdef clause is ok, simply remove the declaration
 246             # and try to move on to report more errors
 247             self.context.nonfatal_error(e)
 248             return None
 249
 250     # Split parallel assignments (a,b = b,a) into separate partial
 251     # assignments that are executed rhs-first using temps.  This
 252     # restructuring must be applied before type analysis so that known
 253     # types on rhs and lhs can be matched directly.  It is required in
 254     # the case that the types cannot be coerced to a Python type in
 255     # order to assign from a tuple.
 256
 257     def visit_SingleAssignmentNode(self, node):
 258         self.visitchildren(node)
 259         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 260
 261     def visit_CascadedAssignmentNode(self, node):
 262         self.visitchildren(node)
 263         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 264
 265     def _visit_assignment_node(self, node, expr_list):
 266         """Flatten parallel assignments into separate single
 267         assignments or cascaded assignments.
 268         """
 269         if sum([ 1 for expr in expr_list if expr.is_sequence_constructor ]) < 2:
 270             # no parallel assignments => nothing to do
 271             return node
 272
 273         expr_list_list = []
 274         flatten_parallel_assignments(expr_list, expr_list_list)
 275         temp_refs = []
 276         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 277
 278         nodes = []
 279         for expr_list in expr_list_list:
 280             lhs_list = expr_list[:-1]
 281             rhs = expr_list[-1]
 282             if len(lhs_list) == 1:
 283                 node = Nodes.SingleAssignmentNode(rhs.pos,
 284                     lhs = lhs_list[0], rhs = rhs)
 285             else:
 286                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 287                     lhs_list = lhs_list, rhs = rhs)
 288             nodes.append(node)
 289
 290         if len(nodes) == 1:
 291             assign_node = nodes[0]
 292         else:
 293             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 294
 295         if temp_refs:
 296             duplicates_and_temps = [ (temp.expression, temp)
 297                                      for temp in temp_refs ]
 298             sort_common_subsequences(duplicates_and_temps)
 299             for _, temp_ref in duplicates_and_temps[::-1]:
 300                 assign_node = LetNode(temp_ref, assign_node)
 301
 302         return assign_node
 303
 304 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 305     """Replace rhs items by LetRefNodes if they appear more than once.
 306     Creates a sequence of LetRefNodes that set up the required temps
 307     and appends them to ref_node_sequence.  The input list is modified
 308     in-place.
 309     """
 310     seen_nodes = cython.set()
 311     ref_nodes = {}
 312     def find_duplicates(node):
 313         if node.is_literal or node.is_name:
 314             # no need to replace those; can't include attributes here
 315             # as their access is not necessarily side-effect free
 316             return
 317         if node in seen_nodes:
 318             if node not in ref_nodes:
 319                 ref_node = LetRefNode(node)
 320                 ref_nodes[node] = ref_node
 321                 ref_node_sequence.append(ref_node)
 322         else:
 323             seen_nodes.add(node)
 324             if node.is_sequence_constructor:
 325                 for item in node.args:
 326                     find_duplicates(item)
 327
 328     for expr_list in expr_list_list:
 329         rhs = expr_list[-1]
 330         find_duplicates(rhs)
 331     if not ref_nodes:
 332         return
 333
 334     def substitute_nodes(node):
 335         if node in ref_nodes:
 336             return ref_nodes[node]
 337         elif node.is_sequence_constructor:
 338             node.args = list(map(substitute_nodes, node.args))
 339         return node
 340
 341     # replace nodes inside of the common subexpressions
 342     for node in ref_nodes:
 343         if node.is_sequence_constructor:
 344             node.args = list(map(substitute_nodes, node.args))
 345
 346     # replace common subexpressions on all rhs items
 347     for expr_list in expr_list_list:
 348         expr_list[-1] = substitute_nodes(expr_list[-1])
 349
 350 def sort_common_subsequences(items):
 351     """Sort items/subsequences so that all items and subsequences that
 352     an item contains appear before the item itself.  This is needed
 353     because each rhs item must only be evaluated once, so its value
 354     must be evaluated first and then reused when packing sequences
 355     that contain it.
 356
 357     This implies a partial order, and the sort must be stable to
 358     preserve the original order as much as possible, so we use a
 359     simple insertion sort (which is very fast for short sequences, the
 360     normal case in practice).
 361     """
 362     def contains(seq, x):
 363         for item in seq:
 364             if item is x:
 365                 return True
 366             elif item.is_sequence_constructor and contains(item.args, x):
 367                 return True
 368         return False
 369     def lower_than(a,b):
 370         return b.is_sequence_constructor and contains(b.args, a)
 371
 372     for pos, item in enumerate(items):
 373         key = item[1] # the ResultRefNode which has already been injected into the sequences
 374         new_pos = pos
 375         for i in xrange(pos-1, -1, -1):
 376             if lower_than(key, items[i][0]):
 377                 new_pos = i
 378         if new_pos != pos:
 379             for i in xrange(pos, new_pos, -1):
 380                 items[i] = items[i-1]
 381             items[new_pos] = item
 382
 383 def flatten_parallel_assignments(input, output):
 384     #  The input is a list of expression nodes, representing the LHSs
 385     #  and RHS of one (possibly cascaded) assignment statement.  For
 386     #  sequence constructors, rearranges the matching parts of both
 387     #  sides into a list of equivalent assignments between the
 388     #  individual elements.  This transformation is applied
 389     #  recursively, so that nested structures get matched as well.
 390     rhs = input[-1]
 391     if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):
 392         output.append(input)
 393         return
 394
 395     complete_assignments = []
 396
 397     rhs_size = len(rhs.args)
 398     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 399     starred_assignments = []
 400     for lhs in input[:-1]:
 401         if not lhs.is_sequence_constructor:
 402             if lhs.is_starred:
 403                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 404             complete_assignments.append(lhs)
 405             continue
 406         lhs_size = len(lhs.args)
 407         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 408         if starred_targets > 1:
 409             error(lhs.pos, "more than 1 starred expression in assignment")
 410             output.append([lhs,rhs])
 411             continue
 412         elif lhs_size - starred_targets > rhs_size:
 413             error(lhs.pos, "need more than %d value%s to unpack"
 414                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 415             output.append([lhs,rhs])
 416             continue
 417         elif starred_targets:
 418             map_starred_assignment(lhs_targets, starred_assignments,
 419                                    lhs.args, rhs.args)
 420         elif lhs_size < rhs_size:
 421             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 422                   % (lhs_size, rhs_size))
 423             output.append([lhs,rhs])
 424             continue
 425         else:
 426             for targets, expr in zip(lhs_targets, lhs.args):
 427                 targets.append(expr)
 428
 429     if complete_assignments:
 430         complete_assignments.append(rhs)
 431         output.append(complete_assignments)
 432
 433     # recursively flatten partial assignments
 434     for cascade, rhs in zip(lhs_targets, rhs.args):
 435         if cascade:
 436             cascade.append(rhs)
 437             flatten_parallel_assignments(cascade, output)
 438
 439     # recursively flatten starred assignments
 440     for cascade in starred_assignments:
 441         if cascade[0].is_sequence_constructor:
 442             flatten_parallel_assignments(cascade, output)
 443         else:
 444             output.append(cascade)
 445
 446 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 447     # Appends the fixed-position LHS targets to the target list that
 448     # appear left and right of the starred argument.
 449     #
 450     # The starred_assignments list receives a new tuple
 451     # (lhs_target, rhs_values_list) that maps the remaining arguments
 452     # (those that match the starred target) to a list.
 453
 454     # left side of the starred target
 455     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 456         if expr.is_starred:
 457             starred = i
 458             lhs_remaining = len(lhs_args) - i - 1
 459             break
 460         targets.append(expr)
 461     else:
 462         raise InternalError("no starred arg found when splitting starred assignment")
 463
 464     # right side of the starred target
 465     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 466                                             lhs_args[-lhs_remaining:])):
 467         targets.append(expr)
 468
 469     # the starred target itself, must be assigned a (potentially empty) list
 470     target = lhs_args[starred].target # unpack starred node
 471     starred_rhs = rhs_args[starred:]
 472     if lhs_remaining:
 473         starred_rhs = starred_rhs[:-lhs_remaining]
 474     if starred_rhs:
 475         pos = starred_rhs[0].pos
 476     else:
 477         pos = target.pos
 478     starred_assignments.append([
 479         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 480
 481
 482 class PxdPostParse(CythonTransform, SkipDeclarations):
 483     """
 484     Basic interpretation/validity checking that should only be
 485     done on pxd trees.
 486
 487     A lot of this checking currently happens in the parser; but
 488     what is listed below happens here.
 489
 490     - "def" functions are let through only if they fill the
 491     getbuffer/releasebuffer slots
 492
 493     - cdef functions are let through only if they are on the
 494     top level and are declared "inline"
 495     """
 496     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 497     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 498
 499     def __call__(self, node):
 500         self.scope_type = 'pxd'
 501         return super(PxdPostParse, self).__call__(node)
 502
 503     def visit_CClassDefNode(self, node):
 504         old = self.scope_type
 505         self.scope_type = 'cclass'
 506         self.visitchildren(node)
 507         self.scope_type = old
 508         return node
 509
 510     def visit_FuncDefNode(self, node):
 511         # FuncDefNode always come with an implementation (without
 512         # an imp they are CVarDefNodes..)
 513         err = self.ERR_INLINE_ONLY
 514
 515         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 516             and node.name in ('__getbuffer__', '__releasebuffer__')):
 517             err = None # allow these slots
 518
 519         if isinstance(node, Nodes.CFuncDefNode):
 520             if u'inline' in node.modifiers and self.scope_type == 'pxd':
 521                 node.inline_in_pxd = True
 522                 if node.c_visibility != 'private':
 523                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 524                 elif node.api:
 525                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 526                 else:
 527                     err = None # allow inline function
 528             else:
 529                 err = self.ERR_INLINE_ONLY
 530
 531         if err:
 532             self.context.nonfatal_error(PostParseError(node.pos, err))
 533             return None
 534         else:
 535             return node
 536
 537 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 538     """
 539     After parsing, directives can be stored in a number of places:
 540     - #cython-comments at the top of the file (stored in ModuleNode)
 541     - Command-line arguments overriding these
 542     - @cython.directivename decorators
 543     - with cython.directivename: statements
 544
 545     This transform is responsible for interpreting these various sources
 546     and store the directive in two ways:
 547     - Set the directives attribute of the ModuleNode for global directives.
 548     - Use a CompilerDirectivesNode to override directives for a subtree.
 549
 550     (The first one is primarily to not have to modify with the tree
 551     structure, so that ModuleNode stay on top.)
 552
 553     The directives are stored in dictionaries from name to value in effect.
 554     Each such dictionary is always filled in for all possible directives,
 555     using default values where no value is given by the user.
 556
 557     The available directives are controlled in Options.py.
 558
 559     Note that we have to run this prior to analysis, and so some minor
 560     duplication of functionality has to occur: We manually track cimports
 561     and which names the "cython" module may have been imported to.
 562     """
 563     unop_method_nodes = {
 564         'typeof': ExprNodes.TypeofNode,
 565
 566         'operator.address': ExprNodes.AmpersandNode,
 567         'operator.dereference': ExprNodes.DereferenceNode,
 568         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 569         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 570         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 571         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 572
 573         # For backwards compatability.
 574         'address': ExprNodes.AmpersandNode,
 575     }
 576
 577     binop_method_nodes = {
 578         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 579     }
 580
 581     special_methods = cython.set(['declare', 'union', 'struct', 'typedef', 'sizeof',
 582                                   'cast', 'pointer', 'compiled', 'NULL'])
 583     special_methods.update(unop_method_nodes.keys())
 584
 585     def __init__(self, context, compilation_directive_defaults):
 586         super(InterpretCompilerDirectives, self).__init__(context)
 587         self.compilation_directive_defaults = {}
 588         for key, value in compilation_directive_defaults.items():
 589             self.compilation_directive_defaults[unicode(key)] = value
 590         self.cython_module_names = cython.set()
 591         self.directive_names = {}
 592
 593     def check_directive_scope(self, pos, directive, scope):
 594         legal_scopes = Options.directive_scopes.get(directive, None)
 595         if legal_scopes and scope not in legal_scopes:
 596             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 597                                         'is not allowed in %s scope' % (directive, scope)))
 598             return False
 599         else:
 600             return True
 601
 602     # Set up processing and handle the cython: comments.
 603     def visit_ModuleNode(self, node):
 604         for key, value in node.directive_comments.items():
 605             if not self.check_directive_scope(node.pos, key, 'module'):
 606                 self.wrong_scope_error(node.pos, key, 'module')
 607                 del node.directive_comments[key]
 608
 609         directives = copy.copy(Options.directive_defaults)
 610         directives.update(self.compilation_directive_defaults)
 611         directives.update(node.directive_comments)
 612         self.directives = directives
 613         node.directives = directives
 614         self.visitchildren(node)
 615         node.cython_module_names = self.cython_module_names
 616         return node
 617
 618     # The following four functions track imports and cimports that
 619     # begin with "cython"
 620     def is_cython_directive(self, name):
 621         return (name in Options.directive_types or
 622                 name in self.special_methods or
 623                 PyrexTypes.parse_basic_type(name))
 624
 625     def visit_CImportStatNode(self, node):
 626         if node.module_name == u"cython":
 627             self.cython_module_names.add(node.as_name or u"cython")
 628         elif node.module_name.startswith(u"cython."):
 629             if node.as_name:
 630                 self.directive_names[node.as_name] = node.module_name[7:]
 631             else:
 632                 self.cython_module_names.add(u"cython")
 633             # if this cimport was a compiler directive, we don't
 634             # want to leave the cimport node sitting in the tree
 635             return None
 636         return node
 637
 638     def visit_FromCImportStatNode(self, node):
 639         if (node.module_name == u"cython") or \
 640                node.module_name.startswith(u"cython."):
 641             submodule = (node.module_name + u".")[7:]
 642             newimp = []
 643             for pos, name, as_name, kind in node.imported_names:
 644                 full_name = submodule + name
 645                 if self.is_cython_directive(full_name):
 646                     if as_name is None:
 647                         as_name = full_name
 648                     self.directive_names[as_name] = full_name
 649                     if kind is not None:
 650                         self.context.nonfatal_error(PostParseError(pos,
 651                             "Compiler directive imports must be plain imports"))
 652                 else:
 653                     newimp.append((pos, name, as_name, kind))
 654             if not newimp:
 655                 return None
 656             node.imported_names = newimp
 657         return node
 658
 659     def visit_FromImportStatNode(self, node):
 660         if (node.module.module_name.value == u"cython") or \
 661                node.module.module_name.value.startswith(u"cython."):
 662             submodule = (node.module.module_name.value + u".")[7:]
 663             newimp = []
 664             for name, name_node in node.items:
 665                 full_name = submodule + name
 666                 if self.is_cython_directive(full_name):
 667                     self.directive_names[name_node.name] = full_name
 668                 else:
 669                     newimp.append((name, name_node))
 670             if not newimp:
 671                 return None
 672             node.items = newimp
 673         return node
 674
 675     def visit_SingleAssignmentNode(self, node):
 676         if (isinstance(node.rhs, ExprNodes.ImportNode) and
 677                 node.rhs.module_name.value == u'cython'):
 678             node = Nodes.CImportStatNode(node.pos,
 679                                          module_name = u'cython',
 680                                          as_name = node.lhs.name)
 681             self.visit_CImportStatNode(node)
 682         else:
 683             self.visitchildren(node)
 684         return node
 685
 686     def visit_NameNode(self, node):
 687         if node.name in self.cython_module_names:
 688             node.is_cython_module = True
 689         else:
 690             node.cython_attribute = self.directive_names.get(node.name)
 691         return node
 692
 693     def try_to_parse_directives(self, node):
 694         # If node is the contents of an directive (in a with statement or
 695         # decorator), returns a list of (directivename, value) pairs.
 696         # Otherwise, returns None
 697         if isinstance(node, ExprNodes.CallNode):
 698             self.visit(node.function)
 699             optname = node.function.as_cython_attribute()
 700             if optname:
 701                 directivetype = Options.directive_types.get(optname)
 702                 if directivetype:
 703                     args, kwds = node.explicit_args_kwds()
 704                     directives = []
 705                     key_value_pairs = []
 706                     if kwds is not None and directivetype is not dict:
 707                         for keyvalue in kwds.key_value_pairs:
 708                             key, value = keyvalue
 709                             sub_optname = "%s.%s" % (optname, key.value)
 710                             if Options.directive_types.get(sub_optname):
 711                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 712                             else:
 713                                 key_value_pairs.append(keyvalue)
 714                         if not key_value_pairs:
 715                             kwds = None
 716                         else:
 717                             kwds.key_value_pairs = key_value_pairs
 718                         if directives and not kwds and not args:
 719                             return directives
 720                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 721                     return directives
 722         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 723             self.visit(node)
 724             optname = node.as_cython_attribute()
 725             if optname:
 726                 directivetype = Options.directive_types.get(optname)
 727                 if directivetype is bool:
 728                     return [(optname, True)]
 729                 elif directivetype is None:
 730                     return [(optname, None)]
 731                 else:
 732                     raise PostParseError(
 733                         node.pos, "The '%s' directive should be used as a function call." % optname)
 734         return None
 735
 736     def try_to_parse_directive(self, optname, args, kwds, pos):
 737         directivetype = Options.directive_types.get(optname)
 738         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 739             return optname, Options.directive_defaults[optname]
 740         elif directivetype is bool:
 741             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 742                 raise PostParseError(pos,
 743                     'The %s directive takes one compile-time boolean argument' % optname)
 744             return (optname, args[0].value)
 745         elif directivetype is str:
 746             if kwds is not None or len(args) != 1 or not isinstance(args[0], (ExprNodes.StringNode,
 747                                                                               ExprNodes.UnicodeNode)):
 748                 raise PostParseError(pos,
 749                     'The %s directive takes one compile-time string argument' % optname)
 750             return (optname, str(args[0].value))
 751         elif directivetype is dict:
 752             if len(args) != 0:
 753                 raise PostParseError(pos,
 754                     'The %s directive takes no prepositional arguments' % optname)
 755             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 756         elif directivetype is list:
 757             if kwds and len(kwds) != 0:
 758                 raise PostParseError(pos,
 759                     'The %s directive takes no keyword arguments' % optname)
 760             return optname, [ str(arg.value) for arg in args ]
 761         else:
 762             assert False
 763
 764     def visit_with_directives(self, body, directives):
 765         olddirectives = self.directives
 766         newdirectives = copy.copy(olddirectives)
 767         newdirectives.update(directives)
 768         self.directives = newdirectives
 769         assert isinstance(body, Nodes.StatListNode), body
 770         retbody = self.visit_Node(body)
 771         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 772                                                  directives=newdirectives)
 773         self.directives = olddirectives
 774         return directive
 775
 776     # Handle decorators
 777     def visit_FuncDefNode(self, node):
 778         directives = self._extract_directives(node, 'function')
 779         if not directives:
 780             return self.visit_Node(node)
 781         body = Nodes.StatListNode(node.pos, stats=[node])
 782         return self.visit_with_directives(body, directives)
 783
 784     def visit_CVarDefNode(self, node):
 785         if not node.decorators:
 786             return node
 787         for dec in node.decorators:
 788             for directive in self.try_to_parse_directives(dec.decorator) or ():
 789                 if directive is not None and directive[0] == u'locals':
 790                     node.directive_locals = directive[1]
 791                 else:
 792                     self.context.nonfatal_error(PostParseError(dec.pos,
 793                         "Cdef functions can only take cython.locals() decorator."))
 794         return node
 795
 796     def visit_CClassDefNode(self, node):
 797         directives = self._extract_directives(node, 'cclass')
 798         if not directives:
 799             return self.visit_Node(node)
 800         body = Nodes.StatListNode(node.pos, stats=[node])
 801         return self.visit_with_directives(body, directives)
 802
 803     def visit_PyClassDefNode(self, node):
 804         directives = self._extract_directives(node, 'class')
 805         if not directives:
 806             return self.visit_Node(node)
 807         body = Nodes.StatListNode(node.pos, stats=[node])
 808         return self.visit_with_directives(body, directives)
 809
 810     def _extract_directives(self, node, scope_name):
 811         if not node.decorators:
 812             return {}
 813         # Split the decorators into two lists -- real decorators and directives
 814         directives = []
 815         realdecs = []
 816         for dec in node.decorators:
 817             new_directives = self.try_to_parse_directives(dec.decorator)
 818             if new_directives is not None:
 819                 for directive in new_directives:
 820                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 821                         directives.append(directive)
 822             else:
 823                 realdecs.append(dec)
 824         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode)):
 825             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 826         else:
 827             node.decorators = realdecs
 828         # merge or override repeated directives
 829         optdict = {}
 830         directives.reverse() # Decorators coming first take precedence
 831         for directive in directives:
 832             name, value = directive
 833             if name in optdict:
 834                 old_value = optdict[name]
 835                 # keywords and arg lists can be merged, everything
 836                 # else overrides completely
 837                 if isinstance(old_value, dict):
 838                     old_value.update(value)
 839                 elif isinstance(old_value, list):
 840                     old_value.extend(value)
 841                 else:
 842                     optdict[name] = value
 843             else:
 844                 optdict[name] = value
 845         return optdict
 846
 847     # Handle with statements
 848     def visit_WithStatNode(self, node):
 849         directive_dict = {}
 850         for directive in self.try_to_parse_directives(node.manager) or []:
 851             if directive is not None:
 852                 if node.target is not None:
 853                     self.context.nonfatal_error(
 854                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
 855                 else:
 856                     name, value = directive
 857                     if name == 'nogil':
 858                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
 859                         node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
 860                         return self.visit_Node(node)
 861                     if self.check_directive_scope(node.pos, name, 'with statement'):
 862                         directive_dict[name] = value
 863         if directive_dict:
 864             return self.visit_with_directives(node.body, directive_dict)
 865         return self.visit_Node(node)
 866
 867 class WithTransform(CythonTransform, SkipDeclarations):
 868
 869     # EXCINFO is manually set to a variable that contains
 870     # the exc_info() tuple that can be generated by the enclosing except
 871     # statement.
 872     template_without_target = TreeFragment(u"""
 873         MGR = EXPR
 874         EXIT = MGR.__exit__
 875         MGR.__enter__()
 876         EXC = True
 877         try:
 878             try:
 879                 EXCINFO = None
 880                 BODY
 881             except:
 882                 EXC = False
 883                 if not EXIT(*EXCINFO):
 884                     raise
 885         finally:
 886             if EXC:
 887                 EXIT(None, None, None)
 888     """, temps=[u'MGR', u'EXC', u"EXIT"],
 889     pipeline=[NormalizeTree(None)])
 890
 891     template_with_target = TreeFragment(u"""
 892         MGR = EXPR
 893         EXIT = MGR.__exit__
 894         VALUE = MGR.__enter__()
 895         EXC = True
 896         try:
 897             try:
 898                 EXCINFO = None
 899                 TARGET = VALUE
 900                 BODY
 901             except:
 902                 EXC = False
 903                 if not EXIT(*EXCINFO):
 904                     raise
 905         finally:
 906             if EXC:
 907                 EXIT(None, None, None)
 908             MGR = EXIT = VALUE = EXC = None
 909
 910     """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
 911     pipeline=[NormalizeTree(None)])
 912
 913     def visit_WithStatNode(self, node):
 914         # TODO: Cleanup badly needed
 915         TemplateTransform.temp_name_counter += 1
 916         handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 917
 918         self.visitchildren(node, ['body'])
 919         excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
 920         if node.target is not None:
 921             result = self.template_with_target.substitute({
 922                 u'EXPR' : node.manager,
 923                 u'BODY' : node.body,
 924                 u'TARGET' : node.target,
 925                 u'EXCINFO' : excinfo_temp
 926                 }, pos=node.pos)
 927         else:
 928             result = self.template_without_target.substitute({
 929                 u'EXPR' : node.manager,
 930                 u'BODY' : node.body,
 931                 u'EXCINFO' : excinfo_temp
 932                 }, pos=node.pos)
 933
 934         # Set except excinfo target to EXCINFO
 935         try_except = result.stats[-1].body.stats[-1]
 936         try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
 937 #            excinfo_temp.ref(node.pos))
 938
 939 #        result.stats[-1].body.stats[-1] = TempsBlockNode(
 940 #            node.pos, temps=[excinfo_temp], body=try_except)
 941
 942         return result
 943
 944     def visit_ExprNode(self, node):
 945         # With statements are never inside expressions.
 946         return node
 947
 948
 949 class DecoratorTransform(CythonTransform, SkipDeclarations):
 950
 951     def visit_DefNode(self, func_node):
 952         self.visitchildren(func_node)
 953         if not func_node.decorators:
 954             return func_node
 955         return self._handle_decorators(
 956             func_node, func_node.name)
 957
 958     def visit_CClassDefNode(self, class_node):
 959         # This doesn't currently work, so it's disabled.
 960         #
 961         # Problem: assignments to cdef class names do not work.  They
 962         # would require an additional check anyway, as the extension
 963         # type must not change its C type, so decorators cannot
 964         # replace an extension type, just alter it and return it.
 965
 966         self.visitchildren(class_node)
 967         if not class_node.decorators:
 968             return class_node
 969         error(class_node.pos,
 970               "Decorators not allowed on cdef classes (used on type '%s')" % class_node.class_name)
 971         return class_node
 972         #return self._handle_decorators(
 973         #    class_node, class_node.class_name)
 974
 975     def visit_ClassDefNode(self, class_node):
 976         self.visitchildren(class_node)
 977         if not class_node.decorators:
 978             return class_node
 979         return self._handle_decorators(
 980             class_node, class_node.name)
 981
 982     def _handle_decorators(self, node, name):
 983         decorator_result = ExprNodes.NameNode(node.pos, name = name)
 984         for decorator in node.decorators[::-1]:
 985             decorator_result = ExprNodes.SimpleCallNode(
 986                 decorator.pos,
 987                 function = decorator.decorator,
 988                 args = [decorator_result])
 989
 990         name_node = ExprNodes.NameNode(node.pos, name = name)
 991         reassignment = Nodes.SingleAssignmentNode(
 992             node.pos,
 993             lhs = name_node,
 994             rhs = decorator_result)
 995         return [node, reassignment]
 996
 997
 998 class AnalyseDeclarationsTransform(CythonTransform):
 999
1000     basic_property = TreeFragment(u"""
1001 property NAME:
1002     def __get__(self):
1003         return ATTR
1004     def __set__(self, value):
1005         ATTR = value
1006     """, level='c_class')
1007     basic_pyobject_property = TreeFragment(u"""
1008 property NAME:
1009     def __get__(self):
1010         return ATTR
1011     def __set__(self, value):
1012         ATTR = value
1013     def __del__(self):
1014         ATTR = None
1015     """, level='c_class')
1016     basic_property_ro = TreeFragment(u"""
1017 property NAME:
1018     def __get__(self):
1019         return ATTR
1020     """, level='c_class')
1021
1022     struct_or_union_wrapper = TreeFragment(u"""
1023 cdef class NAME:
1024     cdef TYPE value
1025     def __init__(self, MEMBER=None):
1026         cdef int count
1027         count = 0
1028         INIT_ASSIGNMENTS
1029         if IS_UNION and count > 1:
1030             raise ValueError, "At most one union member should be specified."
1031     def __str__(self):
1032         return STR_FORMAT % MEMBER_TUPLE
1033     def __repr__(self):
1034         return REPR_FORMAT % MEMBER_TUPLE
1035     """)
1036
1037     init_assignment = TreeFragment(u"""
1038 if VALUE is not None:
1039     ATTR = VALUE
1040     count += 1
1041     """)
1042
1043     def __call__(self, root):
1044         self.env_stack = [root.scope]
1045         # needed to determine if a cdef var is declared after it's used.
1046         self.seen_vars_stack = []
1047         return super(AnalyseDeclarationsTransform, self).__call__(root)
1048
1049     def visit_NameNode(self, node):
1050         self.seen_vars_stack[-1].add(node.name)
1051         return node
1052
1053     def visit_ModuleNode(self, node):
1054         self.seen_vars_stack.append(cython.set())
1055         node.analyse_declarations(self.env_stack[-1])
1056         self.visitchildren(node)
1057         self.seen_vars_stack.pop()
1058         return node
1059
1060     def visit_LambdaNode(self, node):
1061         node.analyse_declarations(self.env_stack[-1])
1062         self.visitchildren(node)
1063         return node
1064
1065     def visit_ClassDefNode(self, node):
1066         self.env_stack.append(node.scope)
1067         self.visitchildren(node)
1068         self.env_stack.pop()
1069         return node
1070
1071     def visit_CClassDefNode(self, node):
1072         node = self.visit_ClassDefNode(node)
1073         if node.scope and node.scope.implemented:
1074             stats = []
1075             for entry in node.scope.var_entries:
1076                 if entry.needs_property:
1077                     property = self.create_Property(entry)
1078                     property.analyse_declarations(node.scope)
1079                     self.visit(property)
1080                     stats.append(property)
1081             if stats:
1082                 node.body.stats += stats
1083         return node
1084
1085     def visit_FuncDefNode(self, node):
1086         self.seen_vars_stack.append(cython.set())
1087         lenv = node.local_scope
1088         node.body.analyse_control_flow(lenv) # this will be totally refactored
1089         node.declare_arguments(lenv)
1090         for var, type_node in node.directive_locals.items():
1091             if not lenv.lookup_here(var):   # don't redeclare args
1092                 type = type_node.analyse_as_type(lenv)
1093                 if type:
1094                     binding = Binding(name = var)
1095                     lenv.declare_var(binding, type = type, pos = type_node.pos)
1096                 else:
1097                     error(type_node.pos, "Not a type")
1098         node.body.analyse_declarations(lenv)
1099         self.env_stack.append(lenv)
1100         self.visitchildren(node)
1101         self.env_stack.pop()
1102         self.seen_vars_stack.pop()
1103         return node
1104
1105     def visit_ScopedExprNode(self, node):
1106         env = self.env_stack[-1]
1107         node.analyse_declarations(env)
1108         # the node may or may not have a local scope
1109         if node.has_local_scope:
1110             self.seen_vars_stack.append(cython.set(self.seen_vars_stack[-1]))
1111             self.env_stack.append(node.expr_scope)
1112             node.analyse_scoped_declarations(node.expr_scope)
1113             self.visitchildren(node)
1114             self.env_stack.pop()
1115             self.seen_vars_stack.pop()
1116         else:
1117             node.analyse_scoped_declarations(env)
1118             self.visitchildren(node)
1119         return node
1120
1121     def visit_TempResultFromStatNode(self, node):
1122         self.visitchildren(node)
1123         node.analyse_declarations(self.env_stack[-1])
1124         return node
1125
1126     def visit_CStructOrUnionDefNode(self, node):
1127         # Create a wrapper node if needed.
1128         # We want to use the struct type information (so it can't happen
1129         # before this phase) but also create new objects to be declared
1130         # (so it can't happen later).
1131         # Note that we don't return the original node, as it is
1132         # never used after this phase.
1133         if True: # private (default)
1134             return None
1135
1136         self_value = ExprNodes.AttributeNode(
1137             pos = node.pos,
1138             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1139             attribute = EncodedString(u"value"))
1140         var_entries = node.entry.type.scope.var_entries
1141         attributes = []
1142         for entry in var_entries:
1143             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1144                                                       obj = self_value,
1145                                                       attribute = entry.name))
1146         # __init__ assignments
1147         init_assignments = []
1148         for entry, attr in zip(var_entries, attributes):
1149             # TODO: branch on visibility
1150             init_assignments.append(self.init_assignment.substitute({
1151                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1152                     u"ATTR": attr,
1153                 }, pos = entry.pos))
1154
1155         # create the class
1156         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1157         wrapper_class = self.struct_or_union_wrapper.substitute({
1158             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1159             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1160             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1161             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1162             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1163         }, pos = node.pos).stats[0]
1164         wrapper_class.class_name = node.name
1165         wrapper_class.shadow = True
1166         class_body = wrapper_class.body.stats
1167
1168         # fix value type
1169         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1170         class_body[0].base_type.name = node.name
1171
1172         # fix __init__ arguments
1173         init_method = class_body[1]
1174         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1175         arg_template = init_method.args[1]
1176         if not node.entry.type.is_struct:
1177             arg_template.kw_only = True
1178         del init_method.args[1]
1179         for entry, attr in zip(var_entries, attributes):
1180             arg = copy.deepcopy(arg_template)
1181             arg.declarator.name = entry.name
1182             init_method.args.append(arg)
1183
1184         # setters/getters
1185         for entry, attr in zip(var_entries, attributes):
1186             # TODO: branch on visibility
1187             if entry.type.is_pyobject:
1188                 template = self.basic_pyobject_property
1189             else:
1190                 template = self.basic_property
1191             property = template.substitute({
1192                     u"ATTR": attr,
1193                 }, pos = entry.pos).stats[0]
1194             property.name = entry.name
1195             wrapper_class.body.stats.append(property)
1196
1197         wrapper_class.analyse_declarations(self.env_stack[-1])
1198         return self.visit_CClassDefNode(wrapper_class)
1199
1200     # Some nodes are no longer needed after declaration
1201     # analysis and can be dropped. The analysis was performed
1202     # on these nodes in a seperate recursive process from the
1203     # enclosing function or module, so we can simply drop them.
1204     def visit_CDeclaratorNode(self, node):
1205         # necessary to ensure that all CNameDeclaratorNodes are visited.
1206         self.visitchildren(node)
1207         return node
1208
1209     def visit_CTypeDefNode(self, node):
1210         return node
1211
1212     def visit_CBaseTypeNode(self, node):
1213         return None
1214
1215     def visit_CEnumDefNode(self, node):
1216         if node.visibility == 'public':
1217             return node
1218         else:
1219             return None
1220
1221     def visit_CNameDeclaratorNode(self, node):
1222         if node.name in self.seen_vars_stack[-1]:
1223             entry = self.env_stack[-1].lookup(node.name)
1224             if entry is None or entry.c_visibility != 'extern':
1225                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1226         self.visitchildren(node)
1227         return node
1228
1229     def visit_CVarDefNode(self, node):
1230         # to ensure all CNameDeclaratorNodes are visited.
1231         self.visitchildren(node)
1232         return None
1233
1234     def create_Property(self, entry):
1235         if entry.visibility == 'public':
1236             if entry.type.is_pyobject:
1237                 template = self.basic_pyobject_property
1238             else:
1239                 template = self.basic_property
1240         elif entry.visibility == 'readonly':
1241             template = self.basic_property_ro
1242         else:
1243             raise NotImplementedError('private python methods')
1244         property = template.substitute({
1245                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1246                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1247                                                  attribute=entry.name),
1248             }, pos=entry.pos).stats[0]
1249         property.name = entry.name
1250         # ---------------------------------------
1251         # XXX This should go to AutoDocTransforms
1252         # ---------------------------------------
1253         if (Options.docstrings and
1254             self.current_directives['embedsignature']):
1255             attr_name = entry.name
1256             type_name = entry.type.declaration_code("", for_display=1)
1257             default_value = ''
1258             if not entry.type.is_pyobject:
1259                 type_name = "'%s'" % type_name
1260             elif entry.type.is_extension_type:
1261                 type_name = entry.type.module_name + '.' + type_name
1262             if entry.init is not None:
1263                 default_value = ' = ' + entry.init
1264             elif entry.init_to_none:
1265                 default_value = ' = ' + repr(None)
1266             docstring = attr_name + ': ' + type_name + default_value
1267             property.doc = EncodedString(docstring)
1268         # ---------------------------------------
1269         return property
1270
1271 class AnalyseExpressionsTransform(CythonTransform):
1272
1273     def visit_ModuleNode(self, node):
1274         node.scope.infer_types()
1275         node.body.analyse_expressions(node.scope)
1276         self.visitchildren(node)
1277         return node
1278
1279     def visit_FuncDefNode(self, node):
1280         node.local_scope.infer_types()
1281         node.body.analyse_expressions(node.local_scope)
1282         self.visitchildren(node)
1283         return node
1284
1285     def visit_ScopedExprNode(self, node):
1286         if node.has_local_scope:
1287             node.expr_scope.infer_types()
1288             node.analyse_scoped_expressions(node.expr_scope)
1289         self.visitchildren(node)
1290         return node
1291
1292 class ExpandInplaceOperators(EnvTransform):
1293
1294     def visit_InPlaceAssignmentNode(self, node):
1295         lhs = node.lhs
1296         rhs = node.rhs
1297         if lhs.type.is_cpp_class:
1298             # No getting around this exact operator here.
1299             return node
1300         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1301             # There is code to handle this case.
1302             return node
1303
1304         env = self.current_env()
1305         def side_effect_free_reference(node, setting=False):
1306             if isinstance(node, ExprNodes.NameNode):
1307                 return node, []
1308             elif node.type.is_pyobject and not setting:
1309                 node = LetRefNode(node)
1310                 return node, [node]
1311             elif isinstance(node, ExprNodes.IndexNode):
1312                 if node.is_buffer_access:
1313                     raise ValueError, "Buffer access"
1314                 base, temps = side_effect_free_reference(node.base)
1315                 index = LetRefNode(node.index)
1316                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1317             elif isinstance(node, ExprNodes.AttributeNode):
1318                 obj, temps = side_effect_free_reference(node.obj)
1319                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1320             else:
1321                 node = LetRefNode(node)
1322                 return node, [node]
1323         try:
1324             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1325         except ValueError:
1326             return node
1327         dup = lhs.__class__(**lhs.__dict__)
1328         binop = ExprNodes.binop_node(node.pos,
1329                                      operator = node.operator,
1330                                      operand1 = dup,
1331                                      operand2 = rhs,
1332                                      inplace=True)
1333         # Manually analyse types for new node.
1334         lhs.analyse_target_types(env)
1335         dup.analyse_types(env)
1336         binop.analyse_operation(env)
1337         node = Nodes.SingleAssignmentNode(
1338             node.pos,
1339             lhs = lhs,
1340             rhs=binop.coerce_to(lhs.type, env))
1341         # Use LetRefNode to avoid side effects.
1342         let_ref_nodes.reverse()
1343         for t in let_ref_nodes:
1344             node = LetNode(t, node)
1345         return node
1346
1347     def visit_ExprNode(self, node):
1348         # In-place assignments can't happen within an expression.
1349         return node
1350
1351
1352 class AlignFunctionDefinitions(CythonTransform):
1353     """
1354     This class takes the signatures from a .pxd file and applies them to
1355     the def methods in a .py file.
1356     """
1357
1358     def visit_ModuleNode(self, node):
1359         self.scope = node.scope
1360         self.directives = node.directives
1361         self.visitchildren(node)
1362         return node
1363
1364     def visit_PyClassDefNode(self, node):
1365         pxd_def = self.scope.lookup(node.name)
1366         if pxd_def:
1367             if pxd_def.is_cclass:
1368                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
1369             else:
1370                 error(node.pos, "'%s' redeclared" % node.name)
1371                 error(pxd_def.pos, "previous declaration here")
1372                 return None
1373         else:
1374             return node
1375
1376     def visit_CClassDefNode(self, node, pxd_def=None):
1377         if pxd_def is None:
1378             pxd_def = self.scope.lookup(node.class_name)
1379         if pxd_def:
1380             outer_scope = self.scope
1381             self.scope = pxd_def.type.scope
1382         self.visitchildren(node)
1383         if pxd_def:
1384             self.scope = outer_scope
1385         return node
1386
1387     def visit_DefNode(self, node):
1388         pxd_def = self.scope.lookup(node.name)
1389         if pxd_def:
1390             if not pxd_def.is_cfunction:
1391                 error(node.pos, "'%s' redeclared" % node.name)
1392                 error(pxd_def.pos, "previous declaration here")
1393                 return None
1394             node = node.as_cfunction(pxd_def)
1395         elif self.scope.is_module_scope and self.directives['auto_cpdef']:
1396             node = node.as_cfunction(scope=self.scope)
1397         # Enable this when internal def functions are allowed.
1398         # self.visitchildren(node)
1399         return node
1400
1401
1402 class MarkClosureVisitor(CythonTransform):
1403
1404     def visit_ModuleNode(self, node):
1405         self.needs_closure = False
1406         self.visitchildren(node)
1407         return node
1408
1409     def visit_FuncDefNode(self, node):
1410         self.needs_closure = False
1411         self.visitchildren(node)
1412         node.needs_closure = self.needs_closure
1413         self.needs_closure = True
1414         return node
1415
1416     def visit_CFuncDefNode(self, node):
1417         self.visit_FuncDefNode(node)
1418         if node.needs_closure:
1419             error(node.pos, "closures inside cdef functions not yet supported")
1420         return node
1421
1422     def visit_LambdaNode(self, node):
1423         self.needs_closure = False
1424         self.visitchildren(node)
1425         node.needs_closure = self.needs_closure
1426         self.needs_closure = True
1427         return node
1428
1429     def visit_ClassDefNode(self, node):
1430         self.visitchildren(node)
1431         self.needs_closure = True
1432         return node
1433
1434
1435 class CreateClosureClasses(CythonTransform):
1436     # Output closure classes in module scope for all functions
1437     # that really need it.
1438
1439     def __init__(self, context):
1440         super(CreateClosureClasses, self).__init__(context)
1441         self.path = []
1442         self.in_lambda = False
1443
1444     def visit_ModuleNode(self, node):
1445         self.module_scope = node.scope
1446         self.visitchildren(node)
1447         return node
1448
1449     def get_scope_use(self, node):
1450         from_closure = []
1451         in_closure = []
1452         for name, entry in node.local_scope.entries.items():
1453             if entry.from_closure:
1454                 from_closure.append((name, entry))
1455             elif entry.in_closure and not entry.from_closure:
1456                 in_closure.append((name, entry))
1457         return from_closure, in_closure
1458
1459     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
1460         from_closure, in_closure = self.get_scope_use(node)
1461         in_closure.sort()
1462
1463         # Now from the begining
1464         node.needs_closure = False
1465         node.needs_outer_scope = False
1466
1467         func_scope = node.local_scope
1468         cscope = node.entry.scope
1469         while cscope.is_py_class_scope or cscope.is_c_class_scope:
1470             cscope = cscope.outer_scope
1471
1472         if not from_closure and (self.path or inner_node):
1473             if not inner_node:
1474                 if not node.assmt:
1475                     raise InternalError, "DefNode does not have assignment node"
1476                 inner_node = node.assmt.rhs
1477             inner_node.needs_self_code = False
1478             node.needs_outer_scope = False
1479         # Simple cases
1480         if not in_closure and not from_closure:
1481             return
1482         elif not in_closure:
1483             func_scope.is_passthrough = True
1484             func_scope.scope_class = cscope.scope_class
1485             node.needs_outer_scope = True
1486             return
1487
1488         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
1489
1490         binding = Binding(name = as_name)
1491         entry = target_module_scope.declare_c_class(
1492             binding, defining = True, implementing = True, pos = node.pos)
1493         func_scope.scope_class = entry
1494         class_scope = entry.type.scope
1495         class_scope.is_internal = True
1496         class_scope.directives = {'final': True}
1497
1498         if from_closure:
1499             assert cscope.is_closure_scope
1500             binding = Binding(
1501                 name=Naming.outer_scope_cname, cname=Naming.outer_scope_cname)
1502             class_scope.declare_var(
1503                 binding, type=cscope.scope_class.type, is_cdef=True,
1504                 pos=node.pos)
1505             node.needs_outer_scope = True
1506         for name, entry in in_closure:
1507             binding = Binding(name=entry.name, cname=entry.cname)
1508             class_scope.declare_var(
1509                 binding, type=entry.type, is_cdef=True, pos=entry.pos)
1510         node.needs_closure = True
1511         # Do it here because other classes are already checked
1512         target_module_scope.check_c_class(func_scope.scope_class)
1513
1514     def visit_LambdaNode(self, node):
1515         was_in_lambda = self.in_lambda
1516         self.in_lambda = True
1517         self.create_class_from_scope(node.def_node, self.module_scope, node)
1518         self.visitchildren(node)
1519         self.in_lambda = was_in_lambda
1520         return node
1521
1522     def visit_FuncDefNode(self, node):
1523         if self.in_lambda:
1524             self.visitchildren(node)
1525             return node
1526         if node.needs_closure or self.path:
1527             self.create_class_from_scope(node, self.module_scope)
1528             self.path.append(node)
1529             self.visitchildren(node)
1530             self.path.pop()
1531         return node
1532
1533
1534 class GilCheck(VisitorTransform):
1535     """
1536     Call `node.gil_check(env)` on each node to make sure we hold the
1537     GIL when we need it.  Raise an error when on Python operations
1538     inside a `nogil` environment.
1539     """
1540     def __call__(self, root):
1541         self.env_stack = [root.scope]
1542         self.nogil = False
1543         return super(GilCheck, self).__call__(root)
1544
1545     def visit_FuncDefNode(self, node):
1546         self.env_stack.append(node.local_scope)
1547         was_nogil = self.nogil
1548         self.nogil = node.local_scope.nogil
1549         if self.nogil and node.nogil_check:
1550             node.nogil_check(node.local_scope)
1551         self.visitchildren(node)
1552         self.env_stack.pop()
1553         self.nogil = was_nogil
1554         return node
1555
1556     def visit_GILStatNode(self, node):
1557         env = self.env_stack[-1]
1558         if self.nogil and node.nogil_check: node.nogil_check()
1559         was_nogil = self.nogil
1560         self.nogil = (node.state == 'nogil')
1561         self.visitchildren(node)
1562         self.nogil = was_nogil
1563         return node
1564
1565     def visit_Node(self, node):
1566         if self.env_stack and self.nogil and node.nogil_check:
1567             node.nogil_check(self.env_stack[-1])
1568         self.visitchildren(node)
1569         return node
1570
1571
1572 class TransformBuiltinMethods(EnvTransform):
1573
1574     def visit_SingleAssignmentNode(self, node):
1575         if node.declaration_only:
1576             return None
1577         else:
1578             self.visitchildren(node)
1579             return node
1580
1581     def visit_AttributeNode(self, node):
1582         self.visitchildren(node)
1583         return self.visit_cython_attribute(node)
1584
1585     def visit_NameNode(self, node):
1586         return self.visit_cython_attribute(node)
1587
1588     def visit_cython_attribute(self, node):
1589         attribute = node.as_cython_attribute()
1590         if attribute:
1591             if attribute == u'compiled':
1592                 node = ExprNodes.BoolNode(node.pos, value=True)
1593             elif attribute == u'NULL':
1594                 node = ExprNodes.NullNode(node.pos)
1595             elif attribute in (u'set', u'frozenset'):
1596                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
1597                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
1598             elif not PyrexTypes.parse_basic_type(attribute):
1599                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
1600         return node
1601
1602     def visit_SimpleCallNode(self, node):
1603
1604         # locals builtin
1605         if isinstance(node.function, ExprNodes.NameNode):
1606             if node.function.name == 'locals':
1607                 lenv = self.current_env()
1608                 entry = lenv.lookup_here('locals')
1609                 if entry:
1610                     # not the builtin 'locals'
1611                     return node
1612                 if len(node.args) > 0:
1613                     error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
1614                     return node
1615                 pos = node.pos
1616                 items = [ ExprNodes.DictItemNode(pos,
1617                                                  key=ExprNodes.StringNode(pos, value=var),
1618                                                  value=ExprNodes.NameNode(pos, name=var))
1619                           for var in lenv.entries ]
1620                 return ExprNodes.DictNode(pos, key_value_pairs=items)
1621
1622         # cython.foo
1623         function = node.function.as_cython_attribute()
1624         if function:
1625             if function in InterpretCompilerDirectives.unop_method_nodes:
1626                 if len(node.args) != 1:
1627                     error(node.function.pos, u"%s() takes exactly one argument" % function)
1628                 else:
1629                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
1630             elif function in InterpretCompilerDirectives.binop_method_nodes:
1631                 if len(node.args) != 2:
1632                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
1633                 else:
1634                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
1635             elif function == u'cast':
1636                 if len(node.args) != 2:
1637                     error(node.function.pos, u"cast() takes exactly two arguments")
1638                 else:
1639                     type = node.args[0].analyse_as_type(self.current_env())
1640                     if type:
1641                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
1642                     else:
1643                         error(node.args[0].pos, "Not a type")
1644             elif function == u'sizeof':
1645                 if len(node.args) != 1:
1646                     error(node.function.pos, u"sizeof() takes exactly one argument")
1647                 else:
1648                     type = node.args[0].analyse_as_type(self.current_env())
1649                     if type:
1650                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
1651                     else:
1652                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
1653             elif function == 'cmod':
1654                 if len(node.args) != 2:
1655                     error(node.function.pos, u"cmod() takes exactly two arguments")
1656                 else:
1657                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
1658                     node.cdivision = True
1659             elif function == 'cdiv':
1660                 if len(node.args) != 2:
1661                     error(node.function.pos, u"cdiv() takes exactly two arguments")
1662                 else:
1663                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
1664                     node.cdivision = True
1665             elif function == u'set':
1666                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
1667             else:
1668                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
1669
1670         self.visitchildren(node)
1671         return node
1672
1673
1674 class DebugTransform(CythonTransform):
1675     """
1676     Create debug information and all functions' visibility to extern in order
1677     to enable debugging.
1678     """
1679
1680     def __init__(self, context, options, result):
1681         super(DebugTransform, self).__init__(context)
1682         self.visited = cython.set()
1683         # our treebuilder and debug output writer
1684         # (see Cython.Debugger.debug_output.CythonDebugWriter)
1685         self.tb = self.context.gdb_debug_outputwriter
1686         #self.c_output_file = options.output_file
1687         self.c_output_file = result.c_file
1688
1689         # Closure support, basically treat nested functions as if the AST were
1690         # never nested
1691         self.nested_funcdefs = []
1692
1693         # tells visit_NameNode whether it should register step-into functions
1694         self.register_stepinto = False
1695
1696     def visit_ModuleNode(self, node):
1697         self.tb.module_name = node.full_module_name
1698         attrs = dict(
1699             module_name=node.full_module_name,
1700             filename=node.pos[0].filename,
1701             c_filename=self.c_output_file)
1702
1703         self.tb.start('Module', attrs)
1704
1705         # serialize functions
1706         self.tb.start('Functions')
1707         # First, serialize functions normally...
1708         self.visitchildren(node)
1709
1710         # ... then, serialize nested functions
1711         for nested_funcdef in self.nested_funcdefs:
1712             self.visit_FuncDefNode(nested_funcdef)
1713
1714         self.register_stepinto = True
1715         self.serialize_modulenode_as_function(node)
1716         self.register_stepinto = False
1717         self.tb.end('Functions')
1718
1719         # 2.3 compatibility. Serialize global variables
1720         self.tb.start('Globals')
1721         entries = {}
1722
1723         for k, v in node.scope.entries.iteritems():
1724             if (v.qualified_name not in self.visited and not
1725                 v.name.startswith('__pyx_') and not
1726                 v.type.is_cfunction and not
1727                 v.type.is_extension_type):
1728                 entries[k]= v
1729
1730         self.serialize_local_variables(entries)
1731         self.tb.end('Globals')
1732         # self.tb.end('Module') # end Module after the line number mapping in
1733         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
1734         return node
1735
1736     def visit_FuncDefNode(self, node):
1737         self.visited.add(node.local_scope.qualified_name)
1738
1739         if getattr(node, 'is_wrapper', False):
1740             return node
1741
1742         if self.register_stepinto:
1743             self.nested_funcdefs.append(node)
1744             return node
1745
1746         if node.py_func is None:
1747             pf_cname = ''
1748         else:
1749             pf_cname = node.py_func.entry.func_cname
1750
1751         attrs = dict(
1752             name=node.entry.name,
1753             cname=node.entry.func_cname,
1754             pf_cname=pf_cname,
1755             qualified_name=node.local_scope.qualified_name,
1756             lineno=str(node.pos[1]))
1757
1758         self.tb.start('Function', attrs=attrs)
1759
1760         self.tb.start('Locals')
1761         self.serialize_local_variables(node.local_scope.entries)
1762         self.tb.end('Locals')
1763
1764         self.tb.start('Arguments')
1765         for arg in node.local_scope.arg_entries:
1766             self.tb.start(arg.name)
1767             self.tb.end(arg.name)
1768         self.tb.end('Arguments')
1769
1770         self.tb.start('StepIntoFunctions')
1771         self.register_stepinto = True
1772         self.visitchildren(node)
1773         self.register_stepinto = False
1774         self.tb.end('StepIntoFunctions')
1775         self.tb.end('Function')
1776
1777         return node
1778
1779     def visit_NameNode(self, node):
1780         if (self.register_stepinto and
1781             node.type.is_cfunction and
1782             getattr(node, 'is_called', False) and
1783             node.entry.func_cname is not None):
1784             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
1785             # declared functions are not 'in_cinclude'.
1786             # This means we will list called 'cdef' functions as
1787             # "step into functions", but this is not an issue as they will be
1788             # recognized as Cython functions anyway.
1789             attrs = dict(name=node.entry.func_cname)
1790             self.tb.start('StepIntoFunction', attrs=attrs)
1791             self.tb.end('StepIntoFunction')
1792
1793         self.visitchildren(node)
1794         return node
1795
1796     def serialize_modulenode_as_function(self, node):
1797         """
1798         Serialize the module-level code as a function so the debugger will know
1799         it's a "relevant frame" and it will know where to set the breakpoint
1800         for 'break modulename'.
1801         """
1802         name = node.full_module_name.rpartition('.')[-1]
1803
1804         cname_py2 = 'init' + name
1805         cname_py3 = 'PyInit_' + name
1806
1807         py2_attrs = dict(
1808             name=name,
1809             cname=cname_py2,
1810             pf_cname='',
1811             # Ignore the qualified_name, breakpoints should be set using
1812             # `cy break modulename:lineno` for module-level breakpoints.
1813             qualified_name='',
1814             lineno='1',
1815             is_initmodule_function="True",
1816         )
1817
1818         py3_attrs = dict(py2_attrs, cname=cname_py3)
1819
1820         self._serialize_modulenode_as_function(node, py2_attrs)
1821         self._serialize_modulenode_as_function(node, py3_attrs)
1822
1823     def _serialize_modulenode_as_function(self, node, attrs):
1824         self.tb.start('Function', attrs=attrs)
1825
1826         self.tb.start('Locals')
1827         self.serialize_local_variables(node.scope.entries)
1828         self.tb.end('Locals')
1829
1830         self.tb.start('Arguments')
1831         self.tb.end('Arguments')
1832
1833         self.tb.start('StepIntoFunctions')
1834         self.register_stepinto = True
1835         self.visitchildren(node)
1836         self.register_stepinto = False
1837         self.tb.end('StepIntoFunctions')
1838
1839         self.tb.end('Function')
1840
1841     def serialize_local_variables(self, entries):
1842         for entry in entries.values():
1843             if entry.type.is_pyobject:
1844                 vartype = 'PythonObject'
1845             else:
1846                 vartype = 'CObject'
1847
1848             if entry.from_closure:
1849                 # We're dealing with a closure where a variable from an outer
1850                 # scope is accessed, get it from the scope object.
1851                 cname = '%s->%s' % (Naming.cur_scope_cname,
1852                                     entry.outer_entry.cname)
1853
1854                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
1855                                       entry.scope.name,
1856                                       entry.name)
1857             elif entry.in_closure:
1858                 cname = '%s->%s' % (Naming.cur_scope_cname,
1859                                     entry.cname)
1860                 qname = entry.qualified_name
1861             else:
1862                 cname = entry.cname
1863                 qname = entry.qualified_name
1864
1865             if not entry.pos:
1866                 # this happens for variables that are not in the user's code,
1867                 # e.g. for the global __builtins__, __doc__, etc. We can just
1868                 # set the lineno to 0 for those.
1869                 lineno = '0'
1870             else:
1871                 lineno = str(entry.pos[1])
1872
1873             attrs = dict(
1874                 name=entry.name,
1875                 cname=cname,
1876                 qualified_name=qname,
1877                 type=vartype,
1878                 lineno=lineno)
1879
1880             self.tb.start('LocalVar', attrs)
1881             self.tb.end('LocalVar')
1882