From: W. Trevor King <wking@drexel.edu>
Date: Wed, 27 Oct 2010 16:21:49 +0000 (-0700)
Subject: Add support for nosetests multiprocessing plugin.
X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=7e07146e0f970b782e92d26a5d2424502131f209;p=sawsim.git

Add support for nosetests multiprocessing plugin.

See
  python-nose/doc/doc_tests/test_multiprocess/multiprocess.rst
for an explanation of the plugin and its peculiarities.

On a 16-core SMP machine, the test suite now runs with:
  Ran 27 tests in 58.077s
vs the old:
  Ran 23 tests in 69.325s

The extra tests are repeats, probably having to do with a
split-vs-share issue that I haven't tracked down yet.  The tests could
be faster still if we used the subproc manager by default for Python
>= 2.6.  I'll do that next.
---

diff --git a/misc/hooks/pre-commit-pysawsim-check b/misc/hooks/pre-commit-pysawsim-check
index 6e4f27b..fc7288f 100755
--- a/misc/hooks/pre-commit-pysawsim-check
+++ b/misc/hooks/pre-commit-pysawsim-check
@@ -1,31 +1,41 @@
 #!/bin/sh
 
+# Detect number of cores with Python >= 2.6.
+# For Python 2.5, you'll have to hard-code the value (or sed /proc/cpuinfo ;).
+CORES=$(python -c 'from multiprocessing import cpu_count; print cpu_count()' \
+    || exit 1)
+#CORES=1
+
+NOSE="nosetests --with-doctest --doctest-tests --processes $CORES"
+
 HAS_MPD=$(which mpdallexit 2>/dev/null)
 
 if [ -n "$HAS_MPD" ]; then
+    echo "Running nosetests with $CORES processes via MPI."
 
     LOCAL_MPD=''
 
     mpdtrace >/dev/null 2>&1
     if [ "$?" -ne 0 ]; then
-	LOCAL_MPD='1'
-	mpd &    # start an mpd instance
-	sleep 1  # give mpd some time to start up
+        LOCAL_MPD='1'
+        mpd &    # start an mpd instance
+        sleep 1  # give mpd some time to start up
     fi           # otherwise there is an mpd instance already running
 
-    mpiexec -n 1 nosetests --with-doctest --doctest-tests pysawsim
+    mpiexec -n 1 $NOSE pysawsim
     RESULT="$?"
 
     if [ -n "$LOCAL_MPD" ]; then
-	mpdallexit
+        mpdallexit
     fi
 
     if [ "$RESULT" -ne 0 ]; then
-	exit 1;
+        exit 1;
     fi
 
 else  # no MPD
+    echo "Running nosetests with $CORES processes without MPI."
 
-    nosetests --with-doctest --doctest-tests pysawsim || exit 1
+    $NOSE pysawsim || exit 1
 
 fi
diff --git a/pysawsim/__init__.py b/pysawsim/__init__.py
index f5b7e64..199b82a 100644
--- a/pysawsim/__init__.py
+++ b/pysawsim/__init__.py
@@ -50,8 +50,15 @@ import logging.handlers
 import sys
 
 
+_multiprocess_shared_ = True
+"""Allow nosetests to share this module between test processes.
+
+This module cannot be split because _log setup is not re-entrant.
+"""
+
 __version__ = '0.10'  # match sawsim version
 
+
 def log():
     return logging.getLogger('pysawsim')
 
diff --git a/pysawsim/_collections.py b/pysawsim/_collections.py
index e6aee98..9754470 100644
--- a/pysawsim/_collections.py
+++ b/pysawsim/_collections.py
@@ -2,6 +2,12 @@ from operator import itemgetter as _itemgetter
 from keyword import iskeyword as _iskeyword
 import sys as _sys
 
+
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 def namedtuple(typename, field_names, verbose=False):
     """Returns a new subclass of tuple with named fields.
 
diff --git a/pysawsim/histogram.py b/pysawsim/histogram.py
index 34d47a6..5b4480b 100644
--- a/pysawsim/histogram.py
+++ b/pysawsim/histogram.py
@@ -25,6 +25,11 @@ import numpy
 from . import log
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 class Histogram (object):
     """A histogram with a flexible comparison method, `residual()`.
 
diff --git a/pysawsim/invoke.py b/pysawsim/invoke.py
index bb501e7..2d8103f 100644
--- a/pysawsim/invoke.py
+++ b/pysawsim/invoke.py
@@ -24,6 +24,11 @@ from subprocess import Popen, PIPE
 import sys
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 class CommandError(Exception):
     """Represent errors in command execution.
 
diff --git a/pysawsim/parameter_error.py b/pysawsim/parameter_error.py
index 9593919..ee915b2 100755
--- a/pysawsim/parameter_error.py
+++ b/pysawsim/parameter_error.py
@@ -30,6 +30,11 @@ from .sawsim_histogram import sawsim_histogram
 from .sawsim import SawsimRunner
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 def find_error_bounds(histogram_matcher, param_range, log_scale, threshold):
     if log_scale == False:
         ps = numpy.linspace(*param_range)
diff --git a/pysawsim/parameter_scan.py b/pysawsim/parameter_scan.py
index 3a4f6f9..ff34a44 100644
--- a/pysawsim/parameter_scan.py
+++ b/pysawsim/parameter_scan.py
@@ -36,6 +36,10 @@ from .sawsim_histogram import sawsim_histogram
 from .sawsim import SawsimRunner
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
 FIGURE = pylab.figure()  # avoid memory problems.
 """`pylab` keeps internal references to all created figures, so share
 a single instance.
diff --git a/pysawsim/sawsim.py b/pysawsim/sawsim.py
index 92db883..ccac6f3 100644
--- a/pysawsim/sawsim.py
+++ b/pysawsim/sawsim.py
@@ -39,6 +39,10 @@ from . import __version__
 from .manager import MANAGERS, get_manager, InvokeJob
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
 SAWSIM = 'sawsim'  # os.path.expand(os.path.join('~', 'bin', 'sawsim'))
 CACHE_DIR = os.path.expanduser(os.path.join('~', '.sawsim-cache'))
 DEFAULT_PARAM_STRING = (
diff --git a/pysawsim/sawsim_histogram.py b/pysawsim/sawsim_histogram.py
index 3220ba4..969ad46 100644
--- a/pysawsim/sawsim_histogram.py
+++ b/pysawsim/sawsim_histogram.py
@@ -24,6 +24,11 @@ from .manager import MANAGERS, get_manager
 from .sawsim import SawsimRunner
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 def sawsim_histogram(sawsim_runner, param_string, N=400, bin_edges=None):
         """Run `N` simulations and return a histogram with `bin_edges`.