Partial merge of trunk progress. Some tests still fail.
[cython.git] / Cython / Compiler / ParseTreeTransforms.py
index b57d374f2eaf10e1b52f22aa53d6a1b1ca717f51..a660db1337a90edea8309db24639188cebec3831 100644 (file)
@@ -11,11 +11,12 @@ import Naming
 import ExprNodes
 import Nodes
 import Options
+import Builtin
 
 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
 from Cython.Compiler.ModuleNode import ModuleNode
-from Cython.Compiler.UtilNodes import LetNode, LetRefNode
+from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
@@ -64,7 +65,6 @@ class SkipDeclarations(object):
     def visit_CStructOrUnionDefNode(self, node):
         return node
 
-
 class NormalizeTree(CythonTransform):
     """
     This transform fixes up a few things after parsing
@@ -183,6 +183,7 @@ class PostParse(ScopeTrackingTransform):
 
     def visit_ModuleNode(self, node):
         self.lambda_counter = 1
+        self.genexpr_counter = 1
         return super(PostParse, self).visit_ModuleNode(node)
 
     def visit_LambdaNode(self, node):
@@ -190,14 +191,33 @@ class PostParse(ScopeTrackingTransform):
         lambda_id = self.lambda_counter
         self.lambda_counter += 1
         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
-
-        body = Nodes.ReturnStatNode(
-            node.result_expr.pos, value = node.result_expr)
+        collector = YieldNodeCollector()
+        collector.visitchildren(node.result_expr)
+        if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
+            body = Nodes.ExprStatNode(
+                node.result_expr.pos, expr=node.result_expr)
+        else:
+            body = Nodes.ReturnStatNode(
+                node.result_expr.pos, value=node.result_expr)
         node.def_node = Nodes.DefNode(
             node.pos, name=node.name, lambda_name=node.lambda_name,
             args=node.args, star_arg=node.star_arg,
             starstar_arg=node.starstar_arg,
-            body=body)
+            body=body, doc=None)
+        self.visitchildren(node)
+        return node
+
+    def visit_GeneratorExpressionNode(self, node):
+        # unpack a generator expression into the corresponding DefNode
+        genexpr_id = self.genexpr_counter
+        self.genexpr_counter += 1
+        node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
+
+        node.def_node = Nodes.DefNode(node.pos, name=node.name,
+                                      doc=None,
+                                      args=[], star_arg=None,
+                                      starstar_arg=None,
+                                      body=node.loop)
         self.visitchildren(node)
         return node
 
@@ -300,6 +320,20 @@ class PostParse(ScopeTrackingTransform):
 
         return assign_node
 
+    def _flatten_sequence(self, seq, result):
+        for arg in seq.args:
+            if arg.is_sequence_constructor:
+                self._flatten_sequence(arg, result)
+            else:
+                result.append(arg)
+        return result
+
+    def visit_DelStatNode(self, node):
+        self.visitchildren(node)
+        node.args = self._flatten_sequence(node, [])
+        return node
+
+
 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
     """Replace rhs items by LetRefNodes if they appear more than once.
     Creates a sequence of LetRefNodes that set up the required temps
@@ -462,7 +496,7 @@ def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args)
 
     # right side of the starred target
     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
-                                            lhs_args[-lhs_remaining:])):
+                                            lhs_args[starred + 1:])):
         targets.append(expr)
 
     # the starred target itself, must be assigned a (potentially empty) list
@@ -518,7 +552,7 @@ class PxdPostParse(CythonTransform, SkipDeclarations):
         if isinstance(node, Nodes.CFuncDefNode):
             if u'inline' in node.modifiers and self.scope_type == 'pxd':
                 node.inline_in_pxd = True
-                if node.visibility != 'private':
+                if node.c_visibility != 'private':
                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
                 elif node.api:
                     err = self.ERR_NOGO_WITH_INLINE % 'api'
@@ -585,7 +619,7 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         super(InterpretCompilerDirectives, self).__init__(context)
         self.compilation_directive_defaults = {}
         for key, value in compilation_directive_defaults.items():
-            self.compilation_directive_defaults[unicode(key)] = value
+            self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
         self.cython_module_names = cython.set()
         self.directive_names = {}
 
@@ -605,8 +639,8 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
                 self.wrong_scope_error(node.pos, key, 'module')
                 del node.directive_comments[key]
 
-        directives = copy.copy(Options.directive_defaults)
-        directives.update(self.compilation_directive_defaults)
+        directives = copy.deepcopy(Options.directive_defaults)
+        directives.update(copy.deepcopy(self.compilation_directive_defaults))
         directives.update(node.directive_comments)
         self.directives = directives
         node.directives = directives
@@ -864,81 +898,55 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         return self.visit_Node(node)
 
 class WithTransform(CythonTransform, SkipDeclarations):
-
-    # EXCINFO is manually set to a variable that contains
-    # the exc_info() tuple that can be generated by the enclosing except
-    # statement.
-    template_without_target = TreeFragment(u"""
-        MGR = EXPR
-        EXIT = MGR.__exit__
-        MGR.__enter__()
-        EXC = True
-        try:
-            try:
-                EXCINFO = None
-                BODY
-            except:
-                EXC = False
-                if not EXIT(*EXCINFO):
-                    raise
-        finally:
-            if EXC:
-                EXIT(None, None, None)
-    """, temps=[u'MGR', u'EXC', u"EXIT"],
-    pipeline=[NormalizeTree(None)])
-
-    template_with_target = TreeFragment(u"""
-        MGR = EXPR
-        EXIT = MGR.__exit__
-        VALUE = MGR.__enter__()
-        EXC = True
-        try:
-            try:
-                EXCINFO = None
-                TARGET = VALUE
-                BODY
-            except:
-                EXC = False
-                if not EXIT(*EXCINFO):
-                    raise
-        finally:
-            if EXC:
-                EXIT(None, None, None)
-            MGR = EXIT = VALUE = EXC = None
-
-    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
-    pipeline=[NormalizeTree(None)])
-
     def visit_WithStatNode(self, node):
-        # TODO: Cleanup badly needed
-        TemplateTransform.temp_name_counter += 1
-        handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
-
-        self.visitchildren(node, ['body'])
-        excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
-        if node.target is not None:
-            result = self.template_with_target.substitute({
-                u'EXPR' : node.manager,
-                u'BODY' : node.body,
-                u'TARGET' : node.target,
-                u'EXCINFO' : excinfo_temp
-                }, pos=node.pos)
-        else:
-            result = self.template_without_target.substitute({
-                u'EXPR' : node.manager,
-                u'BODY' : node.body,
-                u'EXCINFO' : excinfo_temp
-                }, pos=node.pos)
-
-        # Set except excinfo target to EXCINFO
-        try_except = result.stats[-1].body.stats[-1]
-        try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
-#            excinfo_temp.ref(node.pos))
-
-#        result.stats[-1].body.stats[-1] = TempsBlockNode(
-#            node.pos, temps=[excinfo_temp], body=try_except)
-
-        return result
+        self.visitchildren(node, 'body')
+        pos = node.pos
+        body, target, manager = node.body, node.target, node.manager
+        node.target_temp = ExprNodes.TempNode(pos, type=PyrexTypes.py_object_type)
+        if target is not None:
+            node.has_target = True
+            body = Nodes.StatListNode(
+                pos, stats = [
+                    Nodes.WithTargetAssignmentStatNode(
+                        pos, lhs = target, rhs = node.target_temp),
+                    body
+                    ])
+            node.target = None
+
+        excinfo_target = ResultRefNode(
+            pos=pos, type=Builtin.tuple_type, may_hold_none=False)
+        except_clause = Nodes.ExceptClauseNode(
+            pos, body = Nodes.IfStatNode(
+                pos, if_clauses = [
+                    Nodes.IfClauseNode(
+                        pos, condition = ExprNodes.NotNode(
+                            pos, operand = ExprNodes.WithExitCallNode(
+                                pos, with_stat = node,
+                                args = excinfo_target)),
+                        body = Nodes.ReraiseStatNode(pos),
+                        ),
+                    ],
+                else_clause = None),
+            pattern = None,
+            target = None,
+            excinfo_target = excinfo_target,
+            )
+
+        node.body = Nodes.TryFinallyStatNode(
+            pos, body = Nodes.TryExceptStatNode(
+                pos, body = body,
+                except_clauses = [except_clause],
+                else_clause = None,
+                ),
+            finally_clause = Nodes.ExprStatNode(
+                pos, expr = ExprNodes.WithExitCallNode(
+                    pos, with_stat = node,
+                    args = ExprNodes.TupleNode(
+                        pos, args = [ExprNodes.NoneNode(pos) for _ in range(3)]
+                        ))),
+            handle_error_case = False,
+            )
+        return node
 
     def visit_ExprNode(self, node):
         # With statements are never inside expressions.
@@ -1018,16 +1026,27 @@ property NAME:
         return ATTR
     """, level='c_class')
 
-    repr_tree = TreeFragment(u"""
-def NAME(self):
-    return FORMAT % ATTRS
-    """, level='c_class')
+    struct_or_union_wrapper = TreeFragment(u"""
+cdef class NAME:
+    cdef TYPE value
+    def __init__(self, MEMBER=None):
+        cdef int count
+        count = 0
+        INIT_ASSIGNMENTS
+        if IS_UNION and count > 1:
+            raise ValueError, "At most one union member should be specified."
+    def __str__(self):
+        return STR_FORMAT % MEMBER_TUPLE
+    def __repr__(self):
+        return REPR_FORMAT % MEMBER_TUPLE
+    """)
+
+    init_assignment = TreeFragment(u"""
+if VALUE is not None:
+    ATTR = VALUE
+    count += 1
+    """)
 
-    init_assign = TreeFragment(u"""
-if VAR is not None:
-    ATTR = VAR
-    """, level='c_class')
-    
     def __call__(self, root):
         self.env_stack = [root.scope]
         # needed to determine if a cdef var is declared after it's used.
@@ -1079,7 +1098,7 @@ if VAR is not None:
             if not lenv.lookup_here(var):   # don't redeclare args
                 type = type_node.analyse_as_type(lenv)
                 if type:
-                    lenv.declare_var(var, type, type_node.pos)
+                    lenv.declare_var(name = var, type = type, pos = type_node.pos)
                 else:
                     error(type_node.pos, "Not a type")
         node.body.analyse_declarations(lenv)
@@ -1111,22 +1130,15 @@ if VAR is not None:
         return node
 
     def visit_CStructOrUnionDefNode(self, node):
-        # Create a shadow node if needed.
+        # Create a wrapper node if needed.
         # We want to use the struct type information (so it can't happen
         # before this phase) but also create new objects to be declared
         # (so it can't happen later).
-        # Note that we don't need to return the original node, as it is
+        # Note that we don't return the original node, as it is
         # never used after this phase.
         if True: # private (default)
             return None
-        # cdef struct_type value
-        class_body = [Nodes.CVarDefNode(
-            node.pos,
-            base_type = Nodes.CSimpleBaseTypeNode(node.pos, name=node.name),
-            declarators = [Nodes.CNameDeclaratorNode(node.pos, name='value', cname='value')],
-            visibility = 'private',
-            in_pxd = False)]
-        # setters/getters
+
         self_value = ExprNodes.AttributeNode(
             pos = node.pos,
             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
@@ -1137,6 +1149,45 @@ if VAR is not None:
             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
                                                       obj = self_value,
                                                       attribute = entry.name))
+        # __init__ assignments
+        init_assignments = []
+        for entry, attr in zip(var_entries, attributes):
+            # TODO: branch on visibility
+            init_assignments.append(self.init_assignment.substitute({
+                    u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
+                    u"ATTR": attr,
+                }, pos = entry.pos))
+
+        # create the class
+        str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
+        wrapper_class = self.struct_or_union_wrapper.substitute({
+            u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
+            u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
+            u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
+            u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
+            u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
+        }, pos = node.pos).stats[0]
+        wrapper_class.class_name = node.name
+        wrapper_class.shadow = True
+        class_body = wrapper_class.body.stats
+
+        # fix value type
+        assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
+        class_body[0].base_type.name = node.name
+
+        # fix __init__ arguments
+        init_method = class_body[1]
+        assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
+        arg_template = init_method.args[1]
+        if not node.entry.type.is_struct:
+            arg_template.kw_only = True
+        del init_method.args[1]
+        for entry, attr in zip(var_entries, attributes):
+            arg = copy.deepcopy(arg_template)
+            arg.declarator.name = entry.name
+            init_method.args.append(arg)
+
+        # setters/getters
         for entry, attr in zip(var_entries, attributes):
             # TODO: branch on visibility
             if entry.type.is_pyobject:
@@ -1147,78 +1198,10 @@ if VAR is not None:
                     u"ATTR": attr,
                 }, pos = entry.pos).stats[0]
             property.name = entry.name
-            class_body.append(property)
-        
-        # __init__
-        self_base_type = Nodes.CSimpleBaseTypeNode(node.pos,
-            name = None, module_path = [],
-            is_basic_c_type = 0, signed = 0,
-            complex = 0, longness = 0,
-            is_self_arg = 1, templates = None)
-        init_args = [Nodes.CArgDeclNode(
-            node.pos,
-            base_type = self_base_type,
-            declarator = Nodes.CNameDeclaratorNode(node.pos, name = u"self", cname = None),
-            default = None)]
-        empty_base_type = Nodes.CSimpleBaseTypeNode(node.pos,
-            name = None, module_path = [],
-            is_basic_c_type = 0, signed = 0,
-            complex = 0, longness = 0,
-            is_self_arg = 0, templates = None)
-        init_body = []
-        for entry, attr in zip(var_entries, attributes):
-            # TODO: branch on visibility
-            init_args.append(Nodes.CArgDeclNode(
-                entry.pos,
-                base_type = empty_base_type,
-                declarator = Nodes.CNameDeclaratorNode(entry.pos, name = entry.name, cname = None),
-                default = ExprNodes.NoneNode(entry.pos)))
-            init_body.append(self.init_assign.substitute({
-                    u"VAR": ExprNodes.NameNode(entry.pos, name = entry.name),
-                    u"ATTR": attr,
-                }, pos = entry.pos))
-        init_method = Nodes.DefNode(
-            pos = node.pos,
-            args = init_args,
-            name = u"__init__",
-            decorators = [],
-            body = Nodes.StatListNode(node.pos, stats=init_body))
-        class_body.append(init_method)
-
-        # __str__
-        attr_tuple = ExprNodes.TupleNode(node.pos, args=attributes)
-        format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
-        repr_method = self.repr_tree.substitute({
-                u"FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(format)),
-                u"ATTRS": attr_tuple,
-            }, pos = node.pos).stats[0]
-        repr_method.name = "__str__"
-        class_body.append(repr_method)
-
-        # __repr__
-        format = u"%s(%s)" % (node.entry.type.name, ("%r, " * len(attributes))[:-2])
-        repr_method = self.repr_tree.substitute({
-                u"FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(format)),
-                u"ATTRS": attr_tuple,
-            }, pos = node.pos).stats[0]
-        repr_method.name = "__repr__"
-        class_body.append(repr_method)
-
-        # Now create the class.
-        shadow = Nodes.CClassDefNode(node.pos,
-            visibility = 'public',
-            module_name = None,
-            class_name = node.name,
-            base_class_module = None,
-            base_class_name = None,
-            decorators = None,
-            body = Nodes.StatListNode(node.pos, stats=class_body),
-            in_pxd = False,
-            doc = None,
-            shadow = True)
-
-        shadow.analyse_declarations(self.env_stack[-1])
-        return self.visit_CClassDefNode(shadow)
+            wrapper_class.body.stats.append(property)
+
+        wrapper_class.analyse_declarations(self.env_stack[-1])
+        return self.visit_CClassDefNode(wrapper_class)
 
     # Some nodes are no longer needed after declaration
     # analysis and can be dropped. The analysis was performed
@@ -1244,7 +1227,8 @@ if VAR is not None:
     def visit_CNameDeclaratorNode(self, node):
         if node.name in self.seen_vars_stack[-1]:
             entry = self.env_stack[-1].lookup(node.name)
-            if entry is None or entry.visibility != 'extern':
+            if (entry is None or entry.c_visibility != 'extern'
+                and not entry.scope.is_c_class_scope):
                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
         self.visitchildren(node)
         return node
@@ -1262,6 +1246,10 @@ if VAR is not None:
                 template = self.basic_property
         elif entry.visibility == 'readonly':
             template = self.basic_property_ro
+        else:
+            error(entry.pos,
+                  "python methods may not have '%s' Python visibility" %
+                  entry.visibility)
         property = template.substitute({
                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
@@ -1389,7 +1377,8 @@ class AlignFunctionDefinitions(CythonTransform):
                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
             else:
                 error(node.pos, "'%s' redeclared" % node.name)
-                error(pxd_def.pos, "previous declaration here")
+                if pxd_def.pos:
+                    error(pxd_def.pos, "previous declaration here")
                 return None
         else:
             return node
@@ -1407,19 +1396,57 @@ class AlignFunctionDefinitions(CythonTransform):
 
     def visit_DefNode(self, node):
         pxd_def = self.scope.lookup(node.name)
-        if pxd_def:
+        if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
             if not pxd_def.is_cfunction:
                 error(node.pos, "'%s' redeclared" % node.name)
-                error(pxd_def.pos, "previous declaration here")
+                if pxd_def.pos:
+                    error(pxd_def.pos, "previous declaration here")
                 return None
             node = node.as_cfunction(pxd_def)
-        elif self.scope.is_module_scope and self.directives['auto_cpdef']:
+        elif (self.scope.is_module_scope and self.directives['auto_cpdef']
+              and node.is_cdef_func_compatible()):
             node = node.as_cfunction(scope=self.scope)
-        # Enable this when internal def functions are allowed.
+        # Enable this when nested cdef functions are allowed.
         # self.visitchildren(node)
         return node
 
 
+class YieldNodeCollector(TreeVisitor):
+
+    def __init__(self):
+        super(YieldNodeCollector, self).__init__()
+        self.yields = []
+        self.returns = []
+        self.has_return_value = False
+
+    def visit_Node(self, node):
+        return self.visitchildren(node)
+
+    def visit_YieldExprNode(self, node):
+        if self.has_return_value:
+            error(node.pos, "'yield' outside function")
+        self.yields.append(node)
+        self.visitchildren(node)
+
+    def visit_ReturnStatNode(self, node):
+        if node.value:
+            self.has_return_value = True
+            if self.yields:
+                error(node.pos, "'return' with argument inside generator")
+        self.returns.append(node)
+
+    def visit_ClassDefNode(self, node):
+        pass
+
+    def visit_FuncDefNode(self, node):
+        pass
+
+    def visit_LambdaNode(self, node):
+        pass
+
+    def visit_GeneratorExpressionNode(self, node):
+        pass
+
 class MarkClosureVisitor(CythonTransform):
 
     def visit_ModuleNode(self, node):
@@ -1432,6 +1459,27 @@ class MarkClosureVisitor(CythonTransform):
         self.visitchildren(node)
         node.needs_closure = self.needs_closure
         self.needs_closure = True
+
+        collector = YieldNodeCollector()
+        collector.visitchildren(node)
+
+        if collector.yields:
+            for i, yield_expr in enumerate(collector.yields):
+                yield_expr.label_num = i + 1
+
+            gbody = Nodes.GeneratorBodyDefNode(pos=node.pos,
+                                               name=node.name,
+                                               body=node.body)
+            generator = Nodes.GeneratorDefNode(pos=node.pos,
+                                               name=node.name,
+                                               args=node.args,
+                                               star_arg=node.star_arg,
+                                               starstar_arg=node.starstar_arg,
+                                               doc=node.doc,
+                                               decorators=node.decorators,
+                                               gbody=gbody,
+                                               lambda_name=node.lambda_name)
+            return generator
         return node
 
     def visit_CFuncDefNode(self, node):
@@ -1452,7 +1500,6 @@ class MarkClosureVisitor(CythonTransform):
         self.needs_closure = True
         return node
 
-
 class CreateClosureClasses(CythonTransform):
     # Output closure classes in module scope for all functions
     # that really need it.
@@ -1461,24 +1508,88 @@ class CreateClosureClasses(CythonTransform):
         super(CreateClosureClasses, self).__init__(context)
         self.path = []
         self.in_lambda = False
+        self.generator_class = None
 
     def visit_ModuleNode(self, node):
         self.module_scope = node.scope
         self.visitchildren(node)
         return node
 
-    def get_scope_use(self, node):
+    def create_generator_class(self, target_module_scope, pos):
+        if self.generator_class:
+            return self.generator_class
+        # XXX: make generator class creation cleaner
+        entry = target_module_scope.declare_c_class(name='__pyx_Generator',
+                    objstruct_cname='__pyx_Generator_object',
+                    typeobj_cname='__pyx_Generator_type',
+                    pos=pos, defining=True, implementing=True)
+        klass = entry.type.scope
+        klass.is_internal = True
+        klass.directives = {'final': True}
+
+        body_type = PyrexTypes.create_typedef_type('generator_body',
+                                                   PyrexTypes.c_void_ptr_type,
+                                                   '__pyx_generator_body_t')
+        klass.declare_var(pos=pos, name='body', cname='body',
+                          visibility='private', type=body_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='is_running', cname='is_running', type=PyrexTypes.c_int_type,
+                          is_cdef=True)
+        klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
+                          is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+
+        import TypeSlots
+        e = klass.declare_pyfunction(name='send', pos=pos)
+        e.func_cname = '__Pyx_Generator_Send'
+        e.signature = TypeSlots.binaryfunc
+
+        e = klass.declare_pyfunction(name='close', pos=pos)
+        e.func_cname = '__Pyx_Generator_Close'
+        e.signature = TypeSlots.unaryfunc
+
+        e = klass.declare_pyfunction(name='throw', pos=pos)
+        e.func_cname = '__Pyx_Generator_Throw'
+        e.signature = TypeSlots.pyfunction_signature
+
+        e = klass.declare_var(
+            name='__iter__', type=PyrexTypes.py_object_type,
+            c_visibility='public', pos=pos)
+        e.func_cname = 'PyObject_SelfIter'
+
+        e = klass.declare_var(
+            name='__next__', type=PyrexTypes.py_object_type,
+            c_visibility='public', pos=pos)
+        e.func_cname = '__Pyx_Generator_Next'
+
+        self.generator_class = entry.type
+        return self.generator_class
+
+    def find_entries_used_in_closures(self, node):
         from_closure = []
         in_closure = []
         for name, entry in node.local_scope.entries.items():
             if entry.from_closure:
                 from_closure.append((name, entry))
-            elif entry.in_closure and not entry.from_closure:
+            elif entry.in_closure:
                 in_closure.append((name, entry))
         return from_closure, in_closure
 
     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
-        from_closure, in_closure = self.get_scope_use(node)
+        # skip generator body
+        if node.is_generator_body:
+            return
+        # move local variables into closure
+        if node.is_generator:
+            for entry in node.local_scope.entries.values():
+                if not entry.from_closure:
+                    entry.in_closure = True
+
+        from_closure, in_closure = self.find_entries_used_in_closures(node)
         in_closure.sort()
 
         # Now from the begining
@@ -1497,8 +1608,11 @@ class CreateClosureClasses(CythonTransform):
                 inner_node = node.assmt.rhs
             inner_node.needs_self_code = False
             node.needs_outer_scope = False
-        # Simple cases
-        if not in_closure and not from_closure:
+
+        base_type = None
+        if node.is_generator:
+            base_type = self.create_generator_class(target_module_scope, node.pos)
+        elif not in_closure and not from_closure:
             return
         elif not in_closure:
             func_scope.is_passthrough = True
@@ -1508,8 +1622,10 @@ class CreateClosureClasses(CythonTransform):
 
         as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
 
-        entry = target_module_scope.declare_c_class(name = as_name,
-            pos = node.pos, defining = True, implementing = True)
+        entry = target_module_scope.declare_c_class(
+            name=as_name, pos=node.pos, defining=True,
+            implementing=True, base_type=base_type)
+
         func_scope.scope_class = entry
         class_scope = entry.type.scope
         class_scope.is_internal = True
@@ -1524,11 +1640,13 @@ class CreateClosureClasses(CythonTransform):
                                     is_cdef=True)
             node.needs_outer_scope = True
         for name, entry in in_closure:
-            class_scope.declare_var(pos=entry.pos,
+            closure_entry = class_scope.declare_var(pos=entry.pos,
                                     name=entry.name,
                                     cname=entry.cname,
                                     type=entry.type,
                                     is_cdef=True)
+            if entry.is_declared_generic:
+                closure_entry.is_declared_generic = 1
         node.needs_closure = True
         # Do it here because other classes are already checked
         target_module_scope.check_c_class(func_scope.scope_class)
@@ -1621,25 +1739,40 @@ class TransformBuiltinMethods(EnvTransform):
                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
         return node
 
-    def visit_SimpleCallNode(self, node):
+    def _inject_locals(self, node, func_name):
+        # locals()/dir() builtins
+        lenv = self.current_env()
+        entry = lenv.lookup_here(func_name)
+        if entry:
+            # not the builtin
+            return node
+        pos = node.pos
+        if func_name == 'locals':
+            if len(node.args) > 0:
+                error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
+                      % len(node.args))
+                return node
+            items = [ ExprNodes.DictItemNode(pos,
+                                             key=ExprNodes.StringNode(pos, value=var),
+                                             value=ExprNodes.NameNode(pos, name=var))
+                      for var in lenv.entries ]
+            return ExprNodes.DictNode(pos, key_value_pairs=items)
+        else:
+            if len(node.args) > 1:
+                error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
+                      % len(node.args))
+                return node
+            elif len(node.args) == 1:
+                # optimised in Builtin.py
+                return node
+            items = [ ExprNodes.StringNode(pos, value=var) for var in lenv.entries ]
+            return ExprNodes.ListNode(pos, args=items)
 
-        # locals builtin
+    def visit_SimpleCallNode(self, node):
         if isinstance(node.function, ExprNodes.NameNode):
-            if node.function.name == 'locals':
-                lenv = self.current_env()
-                entry = lenv.lookup_here('locals')
-                if entry:
-                    # not the builtin 'locals'
-                    return node
-                if len(node.args) > 0:
-                    error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d" % len(node.args))
-                    return node
-                pos = node.pos
-                items = [ ExprNodes.DictItemNode(pos,
-                                                 key=ExprNodes.StringNode(pos, value=var),
-                                                 value=ExprNodes.NameNode(pos, name=var))
-                          for var in lenv.entries ]
-                return ExprNodes.DictNode(pos, key_value_pairs=items)
+            func_name = node.function.name
+            if func_name in ('dir', 'locals'):
+                return self._inject_locals(node, func_name)
 
         # cython.foo
         function = node.function.as_cython_attribute()