From 8e18e1c05dd8f92ae3752d70569d4f906aa9f516 Mon Sep 17 00:00:00 2001
From: Robert Bradshaw <robertwb@math.washington.edu>
Date: Mon, 13 Sep 2010 21:09:25 -0700
Subject: [PATCH] Parallel cythonization.

---
 Cython/Compiler/Dependencies.py | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/Cython/Compiler/Dependencies.py b/Cython/Compiler/Dependencies.py
index 979ef210..a784a059 100644
--- a/Cython/Compiler/Dependencies.py
+++ b/Cython/Compiler/Dependencies.py
@@ -340,9 +340,7 @@ def create_extension_list(filepatterns, ctx=None):
             module_list.append(Extension(name=name, sources=[file], **deps.distutils_info(file).values))
     return module_list
 
-cython_py = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../cython.py'))
-
-def cythonize(module_list, ctx=None):
+def cythonize(module_list, ctx=None, nthreads=0):
     deps = create_dependency_tree(ctx)
     to_compile = []
     for m in module_list:
@@ -358,6 +356,8 @@ def cythonize(module_list, ctx=None):
                     c_timestamp = os.path.getmtime(c_file)
                 else:
                     c_timestamp = -1
+                # Priority goes first to modified files, second to direct
+                # dependents, and finally to indirect dependents.
                 if c_timestamp < deps.timestamp(source):
                     dep_timestamp, dep = deps.timestamp(source), source
                     priority = 0
@@ -373,8 +373,27 @@ def cythonize(module_list, ctx=None):
         m.sources = new_sources
     to_compile.sort()
     # TODO: invoke directly
-    for priority, pyx_file, c_file in to_compile:
-        cmd = "%s %s %s -o %s" % (sys.executable, cython_py, pyx_file, c_file)
-        print cmd
-        os.system(cmd)
+    if nthreads:
+        # Requires multiprocessing (or Python >= 2.6)
+        try:
+            import multiprocessing
+        except ImportError:
+            print "multiprocessing required for parallel cythonization"
+            nthreads = 0
+        pool = multiprocessing.Pool(nthreads)
+        pool.map(cythonoize_one_helper, to_compile)
+    if not nthreads:
+        for priority, pyx_file, c_file in to_compile:
+            cythonoize_one(pyx_file, c_file)
     return module_list
+
+cython_py = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../cython.py'))
+
+def cythonoize_one(pyx_file, c_file):
+    cmd = "%s %s %s -o %s" % (sys.executable, cython_py, pyx_file, c_file)
+    print cmd
+    if os.system(cmd) != 0:
+        raise CompilerError, pyx_file
+
+def cythonoize_one_helper(m):
+    return cythonoize_one(*m[1:])
-- 
2.26.2