gcc branch prediction

For some reason, it actually makes it slower in some of the most common cases (now commented out). Why? It does help in others.

diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py
index 58ae56241bc58e5a93f33c776aa64248e0ce6bac..dec44344636c9455a16cff3f2037ffb6d310ff44 100644 (file)
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
@@ -3,6 +3,7 @@
 #
 
 import Naming
+import Options
 from Pyrex.Utils import open_new_file
 from PyrexTypes import py_object_type, typecast
 
@@ -294,6 +295,10 @@ class CCodeWriter:
                 doc_code,
                 term))
     
+    def put_error_if_neg(self, pos, value):
+#        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
+        return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
+
     def error_goto(self, pos):
         lbl = self.error_label
         self.use_label(lbl)
@@ -304,6 +309,21 @@ class CCodeWriter:
             Naming.lineno_cname,
             pos[1],
             lbl)
+            
+    def error_goto_if(self, cond, pos):
+        if Options.gcc_branch_hints or 0: # TODO this path is almost _never_ taken, yet this macro makes is slower!
+            return "if (unlikely(%s)) %s" % (cond, self.error_goto(pos))
+        else:
+            return "if (%s) %s" % (cond, self.error_goto(pos))
+            
+    def error_goto_if_null(self, cname, pos):
+        return self.error_goto_if("!%s" % cname, pos)
+    
+    def error_goto_if_neg(self, cname, pos):
+        return self.error_goto_if("%s < 0" % cname, pos)
+    
+    def error_goto_if_PyErr(self, pos):
+        return self.error_goto_if("PyErr_Occurred()", pos)
     
     def lookup_filename(self, filename):
         try:

diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py
index 88d264a88a884613d8262d4776de1250a5b11d5b..96df5edbabba4d32f8ddbcc4e0ab3ac7d516a32c 100644 (file)
--- a/Cython/Compiler/ExprNodes.py
+++ b/Cython/Compiler/ExprNodes.py
@@ -646,11 +646,10 @@ class LongNode(AtomicExprNode):
     
     def generate_evaluation_code(self, code):
         code.putln(
-            '%s = PyLong_FromString("%s", 0, 0); if (!%s) %s' % (
+            '%s = PyLong_FromString("%s", 0, 0); %s' % (
                 self.result_code,
                 self.value,
-                self.result_code,
-                code.error_goto(self.pos)))    
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class ImagNode(AtomicExprNode):
@@ -664,11 +663,10 @@ class ImagNode(AtomicExprNode):
     
     def generate_evaluation_code(self, code):
         code.putln(
-            "%s = PyComplex_FromDoubles(0.0, %s); if (!%s) %s" % (
+            "%s = PyComplex_FromDoubles(0.0, %s); %s" % (
                 self.result_code,
                 self.value,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class NameNode(AtomicExprNode):
@@ -797,20 +795,18 @@ class NameNode(AtomicExprNode):
             if Options.intern_names:
                 #assert entry.interned_cname is not None
                 code.putln(
-                    '%s = __Pyx_GetName(%s, %s); if (!%s) %s' % (
+                    '%s = __Pyx_GetName(%s, %s); %s' % (
                     self.result_code,
                     namespace, 
                     entry.interned_cname,
-                    self.result_code, 
-                    code.error_goto(self.pos)))                
+                    code.error_goto_if_null(self.result_code, self.pos)))              
             else:
                 code.putln(
-                    '%s = __Pyx_GetName(%s, "%s"); if (!%s) %s' % (
+                    '%s = __Pyx_GetName(%s, "%s"); %s' % (
                     self.result_code,
                     namespace, 
                     self.entry.name,
-                    self.result_code, 
-                    code.error_goto(self.pos)))                
+                    code.error_goto_if_null(self.result_code, self.pos)))              
 
     def generate_assignment_code(self, rhs, code):
         entry = self.entry
@@ -819,19 +815,17 @@ class NameNode(AtomicExprNode):
         if entry.is_pyglobal:
             namespace = self.entry.namespace_cname
             if Options.intern_names:
-                code.putln(
-                    'if (PyObject_SetAttr(%s, %s, %s) < 0) %s' % (
+                code.put_error_if_neg(self.pos, 
+                    'PyObject_SetAttr(%s, %s, %s)' % (
                         namespace, 
                         entry.interned_cname,
-                        rhs.py_result(), 
-                        code.error_goto(self.pos)))
+                        rhs.py_result()))
             else:
-                code.putln(
-                    'if (PyObject_SetAttrString(%s, "%s", %s) < 0) %s' % (
+                code.put_error_if_neg(self.pos,
+                    'PyObject_SetAttrString(%s, "%s", %s)' % (
                         namespace, 
                         entry.name,
-                        rhs.py_result(), 
-                        code.error_goto(self.pos)))
+                        rhs.py_result()))
             if debug_disposal_code:
                 print "NameNode.generate_assignment_code:"
                 print "...generating disposal code for", rhs
@@ -856,11 +850,10 @@ class NameNode(AtomicExprNode):
         if not self.entry.is_pyglobal:
             error(self.pos, "Deletion of local or C global name not supported")
             return
-        code.putln(
-            'if (PyObject_DelAttrString(%s, "%s") < 0) %s' % (
+        code.put_error_if_neg(self.pos, 
+            'PyObject_DelAttrString(%s, "%s")' % (
                 Naming.module_cname,
-                self.entry.name,
-                code.error_goto(self.pos)))
+                self.entry.name))
             
 
 class BackquoteNode(ExprNode):
@@ -878,11 +871,10 @@ class BackquoteNode(ExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = PyObject_Repr(%s); if (!%s) %s" % (
+            "%s = PyObject_Repr(%s); %s" % (
                 self.result_code,
                 self.arg.py_result(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class ImportNode(ExprNode):
@@ -910,12 +902,11 @@ class ImportNode(ExprNode):
         else:
             name_list_code = "0"
         code.putln(
-            "%s = __Pyx_Import(%s, %s); if (!%s) %s" % (
+            "%s = __Pyx_Import(%s, %s); %s" % (
                 self.result_code,
                 self.module_name.py_result(),
                 name_list_code,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class IteratorNode(ExprNode):
@@ -947,11 +938,10 @@ class IteratorNode(ExprNode):
                 self.result_code,
                 self.sequence.py_result(),
                 self.result_code))
-        code.putln("else { %s = PyObject_GetIter(%s); if (!%s) %s }" % (
+        code.putln("else { %s = PyObject_GetIter(%s); %s }" % (
                 self.result_code,
                 self.sequence.py_result(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class NextNode(AtomicExprNode):
@@ -986,9 +976,7 @@ class NextNode(AtomicExprNode):
         code.putln(
             "if (!%s) {" %
                 self.result_code)
-        code.putln(
-                "if (PyErr_Occurred()) %s" %
-                    code.error_goto(self.pos))
+        code.error_goto_if_PyErr(self.pos)
         code.putln("break;")
         code.putln("}")
         code.putln("}")
@@ -1007,10 +995,9 @@ class ExcValueNode(AtomicExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = __Pyx_GetExcValue(); if (!%s) %s" % (
+            "%s = __Pyx_GetExcValue(); %s" % (
                 self.result_code,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class TempNode(AtomicExprNode):
@@ -1137,12 +1124,11 @@ class IndexNode(ExprNode):
     def generate_generic_code_result(self, code):
         self.py_index.generate_result_code(code)
         code.putln(
-            "%s = PyObject_GetItem(%s, %s); if (!%s) %s" % (
+            "%s = PyObject_GetItem(%s, %s); %s" % (
                 self.result_code,
                 self.base.py_result(),
                 self.py_index.py_result(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
         if self.is_temp:
             self.py_index.generate_disposal_code(code)
                                 
@@ -1177,23 +1163,21 @@ class IndexNode(ExprNode):
     
     def generate_generic_assignment_code(self, rhs, code):
         self.py_index.generate_result_code(code)
-        code.putln(
-            "if (PyObject_SetItem(%s, %s, %s) < 0) %s" % (
+        code.put_error_if_neg(self.pos, 
+            "PyObject_SetItem(%s, %s, %s)" % (
                 self.base.py_result(),
                 self.py_index.py_result(),
-                rhs.py_result(),
-                code.error_goto(self.pos)))
+                rhs.py_result()))
         if self.is_temp:
             self.py_index.generate_disposal_code(code)
     
     def generate_deletion_code(self, code):
         self.generate_subexpr_evaluation_code(code)
         self.py_index.generate_evaluation_code(code)
-        code.putln(
-            "if (PyObject_DelItem(%s, %s) < 0) %s" % (
+        code.put_error_if_neg(self.pos, 
+            "PyObject_DelItem(%s, %s)" % (
                 self.base.py_result(),
-                self.py_index.py_result(),
-                code.error_goto(self.pos)))
+                self.py_index.py_result()))
         self.generate_subexpr_disposal_code(code)
         self.py_index.generate_disposal_code(code)
         
@@ -1227,34 +1211,31 @@ class SliceIndexNode(ExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = PySequence_GetSlice(%s, %s, %s); if (!%s) %s" % (
+            "%s = PySequence_GetSlice(%s, %s, %s); %s" % (
                 self.result_code,
                 self.base.py_result(),
                 self.start_code(),
                 self.stop_code(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
     
     def generate_assignment_code(self, rhs, code):
         self.generate_subexpr_evaluation_code(code)
-        code.putln(
-            "if (PySequence_SetSlice(%s, %s, %s, %s) < 0) %s" % (
+        code.put_error_if_neg(self.pos, 
+            "PySequence_SetSlice(%s, %s, %s, %s)" % (
                 self.base.py_result(),
                 self.start_code(),
                 self.stop_code(),
-                rhs.result_code,
-                code.error_goto(self.pos)))
+                rhs.result_code))
         self.generate_subexpr_disposal_code(code)
         rhs.generate_disposal_code(code)
 
     def generate_deletion_code(self, code):
         self.generate_subexpr_evaluation_code(code)
-        code.putln(
-            "if (PySequence_DelSlice(%s, %s, %s) < 0) %s" % (
+        code.put_error_if_neg(self.pos,
+            "PySequence_DelSlice(%s, %s, %s)" % (
                 self.base.py_result(),
                 self.start_code(),
-                self.stop_code(),
-                code.error_goto(self.pos)))
+                self.stop_code()))
         self.generate_subexpr_disposal_code(code)
     
     def start_code(self):
@@ -1295,13 +1276,12 @@ class SliceNode(ExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = PySlice_New(%s, %s, %s); if (!%s) %s" % (
+            "%s = PySlice_New(%s, %s, %s); %s" % (
                 self.result_code,
                 self.start.py_result(), 
                 self.stop.py_result(), 
                 self.step.py_result(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 class SimpleCallNode(ExprNode):
     #  Function call without keyword, * or ** args.
@@ -1426,12 +1406,11 @@ class SimpleCallNode(ExprNode):
             else:
                 arg_code = "0"
             code.putln(
-                "%s = PyObject_CallObject(%s, %s); if (!%s) %s" % (
+                "%s = PyObject_CallObject(%s, %s); %s" % (
                     self.result_code,
                     self.function.py_result(),
                     arg_code,
-                    self.result_code,
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_null(self.result_code, self.pos)))
         elif func_type.is_cfunction:
             exc_checks = []
             if self.type.is_pyobject:
@@ -1455,11 +1434,10 @@ class SimpleCallNode(ExprNode):
                 else:
                     lhs = ""
                 code.putln(
-                    "%s%s; if (%s) %s" % (
+                    "%s%s; %s" % (
                         lhs,
                         rhs,
-                        " && ".join(exc_checks),
-                        code.error_goto(self.pos)))    
+                        code.error_goto_if(" && ".join(exc_checks), self.pos)))    
 
 class GeneralCallNode(ExprNode):
     #  General Python function call, including keyword,
@@ -1490,11 +1468,10 @@ class GeneralCallNode(ExprNode):
         
     def generate_result_code(self, code):
         if self.keyword_args and self.starstar_arg:
-            code.putln(
-                "if (PyDict_Update(%s, %s) < 0) %s" % (
+            code.put_error_if_neg(self.pos, 
+                "PyDict_Update(%s, %s)" % (
                     self.keyword_args.py_result(), 
-                    self.starstar_arg.py_result(),
-                    code.error_goto(self.pos)))
+                    self.starstar_arg.py_result()))
             keyword_code = self.keyword_args.py_result()
         elif self.keyword_args:
             keyword_code = self.keyword_args.py_result()
@@ -1512,11 +1489,10 @@ class GeneralCallNode(ExprNode):
                 self.positional_args.py_result(),
                 keyword_code)
         code.putln(
-            "%s = %s; if (!%s) %s" % (
+            "%s = %s; %s" % (
                 self.result_code,
                 call_code,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class AsTupleNode(ExprNode):
@@ -1535,11 +1511,10 @@ class AsTupleNode(ExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = PySequence_Tuple(%s); if (!%s) %s" % (
+            "%s = PySequence_Tuple(%s); %s" % (
                 self.result_code,
                 self.arg.py_result(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
     
 
 class AttributeNode(ExprNode):
@@ -1745,38 +1720,34 @@ class AttributeNode(ExprNode):
         if self.is_py_attr:
             if Options.intern_names:
                 code.putln(
-                    '%s = PyObject_GetAttr(%s, %s); if (!%s) %s' % (
+                    '%s = PyObject_GetAttr(%s, %s); %s' % (
                         self.result_code,
                         self.obj.py_result(),
                         self.interned_attr_cname,
-                        self.result_code,
-                        code.error_goto(self.pos)))
+                        code.error_goto_if_null(self.result_code, self.pos)))
             else:
                 code.putln(
-                    '%s = PyObject_GetAttrString(%s, "%s"); if (!%s) %s' % (
+                    '%s = PyObject_GetAttrString(%s, "%s"); %s' % (
                         self.result_code,
                         self.objpy_result(),
                         self.attribute,
-                        self.result_code,
-                        code.error_goto(self.pos)))
+                        code.error_goto_if_null(self.result_code, self.pos)))
     
     def generate_assignment_code(self, rhs, code):
         self.obj.generate_evaluation_code(code)
         if self.is_py_attr:
             if Options.intern_names:
-                code.putln(
-                    'if (PyObject_SetAttr(%s, %s, %s) < 0) %s' % (
+                code.put_error_if_neg(self.pos, 
+                    'PyObject_SetAttr(%s, %s, %s)' % (
                         self.obj.py_result(),
                         self.interned_attr_cname,
-                        rhs.py_result(),
-                        code.error_goto(self.pos)))
+                        rhs.py_result()))
             else:
-                code.putln(
-                    'if (PyObject_SetAttrString(%s, "%s", %s) < 0) %s' % (
+                code.put_error_if_neg(self.pos,
+                    'PyObject_SetAttrString(%s, "%s", %s)' % (
                         self.obj.py_result(),
                         self.attribute,
-                        rhs.py_result(),
-                        code.error_goto(self.pos)))
+                        rhs.py_result()))
             rhs.generate_disposal_code(code)
         else:
             select_code = self.result_code
@@ -1795,17 +1766,15 @@ class AttributeNode(ExprNode):
         self.obj.generate_evaluation_code(code)
         if self.is_py_attr:
             if Options.intern_names:
-                code.putln(
-                    'if (PyObject_DelAttr(%s, %s) < 0) %s' % (
+                code.put_error_if_neg(self.pos,
+                    'PyObject_DelAttr(%s, %s)' % (
                         self.obj.py_result(),
-                        self.interned_attr_cname,
-                        code.error_goto(self.pos)))
+                        self.interned_attr_cname))
             else:
-                code.putln(
-                    'if (PyObject_DelAttrString(%s, "%s") < 0) %s' % (
+                code.put_error_if_neg(self.pos, 
+                    'PyObject_DelAttrString(%s, "%s")' % (
                         self.obj.py_result(),
-                        self.attribute,
-                        code.error_goto(self.pos)))
+                        self.attribute))
         else:
             error(self.pos, "Cannot delete C attribute of extension type")
         self.obj.generate_disposal_code(code)
@@ -1899,29 +1868,26 @@ class SequenceNode(ExprNode):
         code.putln("else {")
 
         code.putln(
-            "%s = PyObject_GetIter(%s); if (!%s) %s" % (
+            "%s = PyObject_GetIter(%s); %s" % (
                 self.iterator.result_code,
                 rhs.py_result(),
-                self.iterator.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.iterator.result_code, self.pos)))
         rhs.generate_disposal_code(code)
         for i in range(len(self.args)):
             item = self.unpacked_items[i]
             unpack_code = "__Pyx_UnpackItem(%s)" % (
                 self.iterator.py_result())
             code.putln(
-                "%s = %s; if (!%s) %s" % (
+                "%s = %s; %s" % (
                     item.result_code,
                     typecast(item.ctype(), py_object_type, unpack_code),
-                    item.result_code,
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_null(item.result_code, self.pos)))
             value_node = self.coerced_unpacked_items[i]
             value_node.generate_evaluation_code(code)
             self.args[i].generate_assignment_code(value_node, code)
-        code.putln(
-            "if (__Pyx_EndUnpack(%s) < 0) %s" % (
-                self.iterator.py_result(),
-                code.error_goto(self.pos)))
+        code.put_error_if_neg(self.pos, 
+            "__Pyx_EndUnpack(%s)" % (
+                self.iterator.py_result()))
         if debug_disposal_code:
             print "UnpackNode.generate_assignment_code:"
             print "...generating disposal code for", iterator
@@ -1935,11 +1901,10 @@ class TupleNode(SequenceNode):
 
     def generate_operation_code(self, code):
         code.putln(
-            "%s = PyTuple_New(%s); if (!%s) %s" % (
+            "%s = PyTuple_New(%s); %s" % (
                 self.result_code,
                 len(self.args),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
         for i in range(len(self.args)):
             arg = self.args[i]
             if not arg.result_in_temp():
@@ -1962,11 +1927,10 @@ class ListNode(SequenceNode):
     #  List constructor.
     
     def generate_operation_code(self, code):
-        code.putln("%s = PyList_New(%s); if (!%s) %s" %
+        code.putln("%s = PyList_New(%s); %s" %
             (self.result_code,
             len(self.args),
-            self.result_code,
-            code.error_goto(self.pos)))
+            code.error_goto_if_null(self.result_code, self.pos)))
         for i in range(len(self.args)):
             arg = self.args[i]
             #if not arg.is_temp:
@@ -2002,11 +1966,10 @@ class ListComprehensionNode(SequenceNode):
         self.loop.analyse_expressions(env)
         
     def generate_operation_code(self, code):
-        code.putln("%s = PyList_New(%s); if (!%s) %s" %
+        code.putln("%s = PyList_New(%s); %s" %
             (self.result_code,
             0,
-            self.result_code,
-            code.error_goto(self.pos)))
+            code.error_goto_if_null(self.result_code, self.pos)))
         self.loop.generate_execution_code(code)
 
 
@@ -2022,12 +1985,11 @@ class ListComprehensionAppendNode(ExprNode):
         self.is_temp = 1
     
     def generate_result_code(self, code):
-        code.putln("%s = PyList_Append(%s, %s); if (%s) %s" %
+        code.putln("%s = PyList_Append(%s, %s); %s" %
             (self.result_code,
             self.target.result_code,
             self.expr.result_code,
-            self.result_code, 
-            code.error_goto(self.pos)))
+            code.error_goto_if(self.result_code, self.pos)))
 
 
 class DictNode(ExprNode):
@@ -2061,19 +2023,17 @@ class DictNode(ExprNode):
         #  Custom method used here because key-value
         #  pairs are evaluated and used one at a time.
         code.putln(
-            "%s = PyDict_New(); if (!%s) %s" % (
+            "%s = PyDict_New(); %s" % (
                 self.result_code,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
         for key, value in self.key_value_pairs:
             key.generate_evaluation_code(code)
             value.generate_evaluation_code(code)
-            code.putln(
-                "if (PyDict_SetItem(%s, %s, %s) < 0) %s" % (
+            code.put_error_if_neg(self.pos, 
+                "PyDict_SetItem(%s, %s, %s)" % (
                     self.result_code,
                     key.py_result(),
-                    value.py_result(),
-                    code.error_goto(self.pos)))
+                    value.py_result()))
             key.generate_disposal_code(code)
             value.generate_disposal_code(code)
     
@@ -2105,20 +2065,18 @@ class ClassNode(ExprNode):
     
     def generate_result_code(self, code):
         if self.doc:
-            code.putln(
-                'if (PyDict_SetItemString(%s, "__doc__", %s) < 0) %s' % (
+            code.put_error_if_neg(self.pos, 
+                'PyDict_SetItemString(%s, "__doc__", %s)' % (
                     self.dict.py_result(),
-                    self.doc.py_result(),
-                    code.error_goto(self.pos)))
+                    self.doc.py_result()))
         code.putln(
-            '%s = __Pyx_CreateClass(%s, %s, %s, "%s"); if (!%s) %s' % (
+            '%s = __Pyx_CreateClass(%s, %s, %s, "%s"); %s' % (
                 self.result_code,
                 self.bases.py_result(),
                 self.dict.py_result(),
                 self.name.py_result(),
                 self.module_name,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class UnboundMethodNode(ExprNode):
@@ -2138,12 +2096,11 @@ class UnboundMethodNode(ExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = PyMethod_New(%s, 0, %s); if (!%s) %s" % (
+            "%s = PyMethod_New(%s, 0, %s); %s" % (
                 self.result_code,
                 self.function.py_result(),
                 self.class_cname,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class PyCFunctionNode(AtomicExprNode):
@@ -2159,11 +2116,10 @@ class PyCFunctionNode(AtomicExprNode):
     
     def generate_result_code(self, code):
         code.putln(
-            "%s = PyCFunction_New(&%s, 0); if (!%s) %s" % (
+            "%s = PyCFunction_New(&%s, 0); %s" % (
                 self.result_code,
                 self.pymethdef_cname,
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
 
 #-------------------------------------------------------------------
 #
@@ -2213,12 +2169,11 @@ class UnopNode(ExprNode):
     def generate_py_operation_code(self, code):
         function = self.py_operation_function()
         code.putln(
-            "%s = %s(%s); if (!%s) %s" % (
+            "%s = %s(%s); %s" % (
                 self.result_code, 
                 function, 
                 self.operand.py_result(),
-                self.result_code,
-                code.error_goto(self.pos)))
+                code.error_goto_if_null(self.result_code, self.pos)))
         
     def type_error(self):
         if not self.operand.type.is_error:
@@ -2497,14 +2452,13 @@ class BinopNode(ExprNode):
             else:
                 extra_args = ""
             code.putln(
-                "%s = %s(%s, %s%s); if (!%s) %s" % (
+                "%s = %s(%s, %s%s); %s" % (
                     self.result_code, 
                     function, 
                     self.operand1.py_result(),
                     self.operand2.py_result(),
                     extra_args,
-                    self.result_code,
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_null(self.result_code, self.pos)))
         else:
             if self.is_temp:
                 self.generate_c_operation_code(code)
@@ -2744,11 +2698,10 @@ class BoolBinopNode(ExprNode):
         if self.type.is_pyobject:
             test_result = self.temp_bool.result_code
             code.putln(
-                "%s = PyObject_IsTrue(%s); if (%s < 0) %s" % (
+                "%s = PyObject_IsTrue(%s); %s" % (
                     test_result,
                     self.operand1.py_result(),
-                    test_result,
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_neg(test_result, self.pos)))
         else:
             test_result = self.operand1.result_code
         return test_result
@@ -2873,24 +2826,22 @@ class CmpNode:
             operand1, op , operand2):
         if op == 'in' or op == 'not_in':
             code.putln(
-                "%s = PySequence_Contains(%s, %s); if (%s < 0) %s" % (
+                "%s = PySequence_Contains(%s, %s); %s" % (
                     result_code, 
                     operand2.py_result(), 
                     operand1.py_result(), 
-                    result_code,
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_neg(result_code, self.pos)))
             if op == 'not_in':
                 code.putln(
                     "%s = !%s;" % (
                         result_code, result_code))
         elif (operand1.type.is_pyobject
             and op not in ('is', 'is_not')):
-                code.putln(
-                    "if (PyObject_Cmp(%s, %s, &%s) < 0) %s" % (
+                code.put_error_if_neg(self.pos, 
+                    "PyObject_Cmp(%s, %s, &%s)" % (
                         operand1.py_result(), 
                         operand2.py_result(), 
-                        result_code,
-                        code.error_goto(self.pos)))
+                        result_code))
                 code.putln(
                     "%s = %s %s 0;" % (
                         result_code, result_code, op))
@@ -3178,12 +3129,11 @@ class CoerceToPyTypeNode(CoercionNode):
     
     def generate_result_code(self, code):
         function = self.arg.type.to_py_function
-        code.putln('%s = %s(%s); if (!%s) %s' % (
+        code.putln('%s = %s(%s); %s' % (
             self.result_code, 
             function, 
             self.arg.result_code, 
-            self.result_code, 
-            code.error_goto(self.pos)))
+            code.error_goto_if_null(self.result_code, self.pos)))
 
 
 class CoerceFromPyTypeNode(CoercionNode):
@@ -3207,10 +3157,10 @@ class CoerceFromPyTypeNode(CoercionNode):
         rhs = "%s(%s)" % (function, operand)
         if self.type.is_enum:
             rhs = typecast(self.type, c_long_type, rhs)
-        code.putln('%s = %s; if (PyErr_Occurred()) %s' % (
+        code.putln('%s = %s; %s' % (
             self.result_code, 
             rhs, 
-            code.error_goto(self.pos)))
+            code.error_goto_if_PyErr(self.pos)))
 
 
 class CoerceToBooleanNode(CoercionNode):
@@ -3234,11 +3184,10 @@ class CoerceToBooleanNode(CoercionNode):
     def generate_result_code(self, code):
         if self.arg.type.is_pyobject:
             code.putln(
-                "%s = PyObject_IsTrue(%s); if (%s < 0) %s" % (
+                "%s = PyObject_IsTrue(%s); %s" % (
                     self.result_code, 
                     self.arg.py_result(), 
-                    self.result_code,
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_neg(self.result_code, self.pos)))
 
 
 class CoerceToTempNode(CoercionNode):

diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
index 147c6ea70b45ee0188ba5574032981b31d750c21..649420024f45114089f2a375b3838beb14aa0e50 100644 (file)
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
@@ -155,6 +155,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         #      code.putln('#include "%s"' % filename)
         code.putln('')
         code.put(Nodes.utility_function_predeclarations)
+        code.put(Nodes.branch_prediction_macros)
         #if Options.intern_names:
         #      code.putln(Nodes.get_name_interned_predeclaration)
         #else:
@@ -1207,13 +1208,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             objstruct = type.objstruct_cname
         else:
             objstruct = "struct %s" % type.objstruct_cname
-        code.putln('%s = __Pyx_ImportType("%s", "%s", sizeof(%s)); if (!%s) %s' % (
+        code.putln('%s = __Pyx_ImportType("%s", "%s", sizeof(%s)); %s' % (
             type.typeptr_cname,
             type.module_name, 
             type.name,
             objstruct,
-            type.typeptr_cname,
-            code.error_goto(pos)))
+            code.error_goto_if_null(type.typeptr_cname, pos)))
         self.use_type_import_utility_code(env)
         if type.vtabptr_cname:
             code.putln(

diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py
index d2c1509319267f1d0fa621d9f8be168633aa6fda..fbc3e35ef6e46b2bd69498fe1edf7b01cd789242 100644 (file)
--- a/Cython/Compiler/Nodes.py
+++ b/Cython/Compiler/Nodes.py
@@ -744,7 +744,7 @@ class CFuncDefNode(FuncDefNode):
             typeptr_cname = arg.type.typeptr_cname
             arg_code = "((PyObject *)%s)" % arg.cname
             code.putln(
-                'if (!__Pyx_ArgTypeTest(%s, %s, %d, "%s")) %s' % (
+                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s"))) %s' % (
                     arg_code, 
                     typeptr_cname,
                     not arg.not_none,
@@ -1029,7 +1029,7 @@ class DefNode(FuncDefNode):
                     Naming.kwdlist_cname] + arg_addrs
             pt_argstring = string.join(pt_arglist, ", ")
             code.put(
-                'if (!PyArg_ParseTupleAndKeywords(%s)) ' %
+                'if (unlikely(!PyArg_ParseTupleAndKeywords(%s))) ' %
                     pt_argstring)
             error_return_code = "return %s;" % self.error_value()
             if has_starargs:
@@ -1102,11 +1102,11 @@ class DefNode(FuncDefNode):
         new_type = arg.type
         func = new_type.from_py_function
         if func:
-            code.putln("%s = %s(%s); if (PyErr_Occurred()) %s" % (
+            code.putln("%s = %s(%s); %s" % (
                 arg.entry.cname,
                 func,
                 arg.hdr_cname,
-                code.error_goto(arg.pos)))
+                code.error_goto_if_PyErr(arg.pos)))
         else:
             error(arg.pos, 
                 "Cannot convert Python object argument to type '%s'" 
@@ -1116,12 +1116,11 @@ class DefNode(FuncDefNode):
         old_type = arg.hdr_type
         func = old_type.to_py_function
         if func:
-            code.putln("%s = %s(%s); if (!%s) %s" % (
+            code.putln("%s = %s(%s); %s" % (
                 arg.entry.cname,
                 func,
                 arg.hdr_cname,
-                arg.entry.cname,
-                code.error_goto(arg.pos)))
+                code.error_goto_if_null(arg.entry.cname, arg.pos)))
         else:
             error(arg.pos,
                 "Cannot convert argument of type '%s' to Python object"
@@ -1141,7 +1140,7 @@ class DefNode(FuncDefNode):
             typeptr_cname = arg.type.typeptr_cname
             arg_code = "((PyObject *)%s)" % arg.entry.cname
             code.putln(
-                'if (!__Pyx_ArgTypeTest(%s, %s, %d, "%s")) %s' % (
+                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s"))) %s' % (
                     arg_code, 
                     typeptr_cname,
                     not arg.not_none,
@@ -1623,13 +1622,12 @@ class InPlaceAssignmentNode(AssignmentNode):
         self.dup.generate_result_code(code)
         if self.lhs.type.is_pyobject:
             code.putln(
-                "%s = %s(%s, %s); if (!%s) %s" % (
+                "%s = %s(%s, %s); %s" % (
                     self.result.result_code, 
                     self.py_operation_function(), 
                     self.dup.py_result(),
                     self.rhs.py_result(),
-                    self.result.py_result(),
-                    code.error_goto(self.pos)))
+                    code.error_goto_if_null(self.result.py_result(), self.pos)))
             self.rhs.generate_disposal_code(code)
             self.dup.generate_disposal_code(code)
             self.lhs.generate_assignment_code(self.result, code)
@@ -1910,7 +1908,7 @@ class AssertStatNode(StatNode):
         if self.value:
             self.value.generate_evaluation_code(code)
         code.putln(
-            "if (!%s) {" %
+            "if (unlikely(!%s)) {" %
                 self.cond.result_code)
         if self.value:
             code.putln(
@@ -2563,22 +2561,20 @@ class FromImportStatNode(StatNode):
         if Options.intern_names:
             for cname, target in self.interned_items:
                 code.putln(
-                    '%s = PyObject_GetAttr(%s, %s); if (!%s) %s' % (
+                    '%s = PyObject_GetAttr(%s, %s); %s' % (
                         self.item.result_code, 
                         self.module.py_result(),
                         cname,
-                        self.item.result_code,
-                        code.error_goto(self.pos)))
+                        code.error_goto_if_null(self.item.result_code, self.pos)))
                 target.generate_assignment_code(self.item, code)
         else:
             for name, target in self.items:
                 code.putln(
-                    '%s = PyObject_GetAttrString(%s, "%s"); if (!%s) %s' % (
+                    '%s = PyObject_GetAttrString(%s, "%s"); %s' % (
                         self.item.result_code, 
                         self.module.py_result(),
                         name,
-                        self.item.result_code,
-                        code.error_goto(self.pos)))
+                        code.error_goto_if_null(self.item.result_code, self.pos)))
                 target.generate_assignment_code(self.item, code)
         self.module.generate_disposal_code(code)
 
@@ -2600,7 +2596,20 @@ static inline int __Pyx_PyObject_IsTrue(PyObject* x) {
    else return PyObject_IsTrue(x);
 }
 
-"""
+""" 
+
+if Options.gcc_branch_hints:
+    branch_prediction_macros = \
+    """
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)
+    """
+else:
+    branch_prediction_macros = \
+    """
+#define likely(x)   (x)
+#define unlikely(x) (x)
+    """
 
 #get_name_predeclaration = \
 #"static PyObject *__Pyx_GetName(PyObject *dict, char *name); /*proto*/"

diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py
index 1ce74656c367549a8db8144a9d16ecd3d4f00661..0456cf7723b9ad72eddb637da197f28d82b472dc 100644 (file)
--- a/Cython/Compiler/Options.py
+++ b/Cython/Compiler/Options.py
@@ -6,3 +6,4 @@ intern_names = 1    #  Intern global variable and attribute names
 cache_builtins = 1  #  Perform lookups on builtin names only once
 
 embed_pos_in_docstring = 0
+gcc_branch_hints = 1