From: Vitja Makarov Date: Sun, 9 Jan 2011 08:04:22 +0000 (+0300) Subject: Add support for generator expressions, currently breaking inlining. X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=4bc0e6b1126b98d9306a7c9b9f9d07f66191511a;p=cython.git Add support for generator expressions, currently breaking inlining. --- diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index a092ae93..cb9211db 100755 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -4343,15 +4343,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode): self.value_expr.annotate(code) -class GeneratorExpressionNode(ScopedExprNode): - # A generator expression, e.g. (i for i in range(10)) - # - # Result is a generator. +class InlinedGeneratorExpressionNode(ScopedExprNode): + # An inlined generator expression for which the result is + # calculated inside of the loop. This will only be created by + # transforms when replacing builtin calls on generator + # expressions. # - # loop ForStatNode the for-loop, containing a YieldExprNode + # loop ForStatNode the for-loop, not containing any YieldExprNodes + # result_node ResultRefNode the reference to the result value temp + # orig_func String the name of the builtin function this node replaces child_attrs = ["loop"] - + loop_analysed = False type = py_object_type def analyse_scoped_declarations(self, env): @@ -4362,30 +4365,12 @@ class GeneratorExpressionNode(ScopedExprNode): self.loop.analyse_expressions(env) self.is_temp = True - def analyse_scoped_expressions(self, env): - if self.has_local_scope: - self.loop.analyse_expressions(env) - def may_be_none(self): return False def annotate(self, code): self.loop.annotate(code) - -class InlinedGeneratorExpressionNode(GeneratorExpressionNode): - # An inlined generator expression for which the result is - # calculated inside of the loop. This will only be created by - # transforms when replacing builtin calls on generator - # expressions. - # - # loop ForStatNode the for-loop, not containing any YieldExprNodes - # result_node ResultRefNode the reference to the result value temp - # orig_func String the name of the builtin function this node replaces - - child_attrs = ["loop"] - loop_analysed = False - def infer_type(self, env): return self.result_node.infer_type(env) @@ -4398,7 +4383,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode): def analyse_scoped_expressions(self, env): self.loop_analysed = True - GeneratorExpressionNode.analyse_scoped_expressions(self, env) + if self.has_local_scope: + self.loop.analyse_expressions(env) def coerce_to(self, dst_type, env): if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed: @@ -4409,7 +4395,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode): # assignments. self.result_node.type = self.type = dst_type return self - return GeneratorExpressionNode.coerce_to(self, dst_type, env) + return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env) def generate_result_code(self, code): self.result_node.result_code = self.result() @@ -4957,6 +4943,35 @@ class LambdaNode(InnerFunctionNode): env.add_lambda_def(self.def_node) +class GeneratorExpressionNode(LambdaNode): + # A generator expression, e.g. (i for i in range(10)) + # + # Result is a generator. + # + # loop ForStatNode the for-loop, containing a YieldExprNode + # def_node DefNode the underlying generator 'def' node + + child_attrs = ["loop", "def_node"] + name = StringEncoding.EncodedString('') + binding = False + + def analyse_declarations(self, env): + # XXX: dirty hack to disable assignment synthesis + self.def_node.needs_assignment_synthesis = lambda *args, **kwargs: False + self.def_node.analyse_declarations(env) + #super(GeneratorExpressionNode, self).analyse_declarations(env) + env.add_lambda_def(self.def_node) + + def generate_result_code(self, code): + code.putln( + '%s = %s(%s, NULL); %s' % ( + self.result(), + self.def_node.entry.func_cname, + self.self_result_code(), + code.error_goto_if_null(self.result(), self.pos))) + code.put_gotref(self.py_result()) + + class YieldExprNode(ExprNode): # Yield expression node # diff --git a/Cython/Compiler/Optimize.py b/Cython/Compiler/Optimize.py index 77fdaac1..d2e329c7 100644 --- a/Cython/Compiler/Optimize.py +++ b/Cython/Compiler/Optimize.py @@ -1167,11 +1167,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): self.yield_nodes = [] visit_Node = Visitor.TreeVisitor.visitchildren - def visit_YieldExprNode(self, node): + # XXX: disable inlining while it's not back supported + def __visit_YieldExprNode(self, node): self.yield_nodes.append(node) self.visitchildren(node) - def visit_ExprStatNode(self, node): + def __visit_ExprStatNode(self, node): self.visitchildren(node) if node.expr in self.yield_nodes: self.yield_stat_nodes[node.expr] = node diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index fe3c15e3..5c9e2afa 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -183,6 +183,7 @@ class PostParse(ScopeTrackingTransform): def visit_ModuleNode(self, node): self.lambda_counter = 1 + self.genexpr_counter = 1 return super(PostParse, self).visit_ModuleNode(node) def visit_LambdaNode(self, node): @@ -201,6 +202,20 @@ class PostParse(ScopeTrackingTransform): self.visitchildren(node) return node + def visit_GeneratorExpressionNode(self, node): + # unpack a generator expression into the corresponding DefNode + genexpr_id = self.genexpr_counter + self.genexpr_counter += 1 + node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id) + + node.def_node = Nodes.DefNode(node.pos, name=node.genexpr_name, + doc=None, + args=[], star_arg=None, + starstar_arg=None, + body=node.loop) + self.visitchildren(node) + return node + # cdef variables def handle_bufferdefaults(self, decl): if not isinstance(decl.default, ExprNodes.DictNode): diff --git a/tests/bugs.txt b/tests/bugs.txt index c97798ef..592758ff 100644 --- a/tests/bugs.txt +++ b/tests/bugs.txt @@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408 compile.cpp_operators cpp_templated_ctypedef cpp_structs -genexpr_T491 with_statement_module_level_T536 function_as_method_T494 closure_inside_cdef_T554