Add support for nosetests multiprocessing plugin.

See
  python-nose/doc/doc_tests/test_multiprocess/multiprocess.rst
for an explanation of the plugin and its peculiarities.

On a 16-core SMP machine, the test suite now runs with:
  Ran 27 tests in 58.077s
vs the old:
  Ran 23 tests in 69.325s

The extra tests are repeats, probably having to do with a
split-vs-share issue that I haven't tracked down yet.  The tests could
be faster still if we used the subproc manager by default for Python
>= 2.6.  I'll do that next.

diff --git a/misc/hooks/pre-commit-pysawsim-check b/misc/hooks/pre-commit-pysawsim-check
index 6e4f27b13965d8a89bfe81bbd5040e48dd1ced8f..fc7288f1b80e6cda0e5e41dbfb588e471c898cb8 100755 (executable)
--- a/misc/hooks/pre-commit-pysawsim-check
+++ b/misc/hooks/pre-commit-pysawsim-check
@@ -1,31 +1,41 @@
 #!/bin/sh
 
+# Detect number of cores with Python >= 2.6.
+# For Python 2.5, you'll have to hard-code the value (or sed /proc/cpuinfo ;).
+CORES=$(python -c 'from multiprocessing import cpu_count; print cpu_count()' \
+    || exit 1)
+#CORES=1
+
+NOSE="nosetests --with-doctest --doctest-tests --processes $CORES"
+
 HAS_MPD=$(which mpdallexit 2>/dev/null)
 
 if [ -n "$HAS_MPD" ]; then
+    echo "Running nosetests with $CORES processes via MPI."
 
     LOCAL_MPD=''
 
     mpdtrace >/dev/null 2>&1
     if [ "$?" -ne 0 ]; then
-       LOCAL_MPD='1'
-       mpd &    # start an mpd instance
-       sleep 1  # give mpd some time to start up
+        LOCAL_MPD='1'
+        mpd &    # start an mpd instance
+        sleep 1  # give mpd some time to start up
     fi           # otherwise there is an mpd instance already running
 
-    mpiexec -n 1 nosetests --with-doctest --doctest-tests pysawsim
+    mpiexec -n 1 $NOSE pysawsim
     RESULT="$?"
 
     if [ -n "$LOCAL_MPD" ]; then
-       mpdallexit
+        mpdallexit
     fi
 
     if [ "$RESULT" -ne 0 ]; then
-       exit 1;
+        exit 1;
     fi
 
 else  # no MPD
+    echo "Running nosetests with $CORES processes without MPI."
 
-    nosetests --with-doctest --doctest-tests pysawsim || exit 1
+    $NOSE pysawsim || exit 1
 
 fi

diff --git a/pysawsim/__init__.py b/pysawsim/__init__.py
index f5b7e64447e260673fac11302ec1c43a8050382f..199b82a65acfdc78248cf441ffcb5de8cfd5fe5b 100644 (file)
--- a/pysawsim/__init__.py
+++ b/pysawsim/__init__.py
@@ -50,8 +50,15 @@ import logging.handlers
 import sys
 
 
+_multiprocess_shared_ = True
+"""Allow nosetests to share this module between test processes.
+
+This module cannot be split because _log setup is not re-entrant.
+"""
+
 __version__ = '0.10'  # match sawsim version
 
+
 def log():
     return logging.getLogger('pysawsim')
 

diff --git a/pysawsim/_collections.py b/pysawsim/_collections.py
index e6aee98fb49a065b4990488bd7019f559b75b925..9754470ada67a04a9c14539d52bd729a206c545d 100644 (file)
--- a/pysawsim/_collections.py
+++ b/pysawsim/_collections.py
@@ -2,6 +2,12 @@ from operator import itemgetter as _itemgetter
 from keyword import iskeyword as _iskeyword
 import sys as _sys
 
+
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 def namedtuple(typename, field_names, verbose=False):
     """Returns a new subclass of tuple with named fields.
 

diff --git a/pysawsim/histogram.py b/pysawsim/histogram.py
index 34d47a6e353807ec16435f2bfd535e08d134b0f1..5b4480b856fa1abd17623e022c08ef8e8c8a49ae 100644 (file)
--- a/pysawsim/histogram.py
+++ b/pysawsim/histogram.py
@@ -25,6 +25,11 @@ import numpy
 from . import log
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 class Histogram (object):
     """A histogram with a flexible comparison method, `residual()`.
 

diff --git a/pysawsim/invoke.py b/pysawsim/invoke.py
index bb501e768901702c12758c153aae820dc5bfb14d..2d8103fab178e6dab14ae1eb298e3e881c89ec23 100644 (file)
--- a/pysawsim/invoke.py
+++ b/pysawsim/invoke.py
@@ -24,6 +24,11 @@ from subprocess import Popen, PIPE
 import sys
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 class CommandError(Exception):
     """Represent errors in command execution.
 

diff --git a/pysawsim/parameter_error.py b/pysawsim/parameter_error.py
index 95939198820ed2618f576c88732e268666a1901b..ee915b2dcfdf5113a644a30f1084e2501806dd76 100755 (executable)
--- a/pysawsim/parameter_error.py
+++ b/pysawsim/parameter_error.py
@@ -30,6 +30,11 @@ from .sawsim_histogram import sawsim_histogram
 from .sawsim import SawsimRunner
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 def find_error_bounds(histogram_matcher, param_range, log_scale, threshold):
     if log_scale == False:
         ps = numpy.linspace(*param_range)

diff --git a/pysawsim/parameter_scan.py b/pysawsim/parameter_scan.py
index 3a4f6f978ed2b13cfdb9226d59692486bbe5212b..ff34a44661a34a99b4ba1366a631a3b3c5926990 100644 (file)
--- a/pysawsim/parameter_scan.py
+++ b/pysawsim/parameter_scan.py
@@ -36,6 +36,10 @@ from .sawsim_histogram import sawsim_histogram
 from .sawsim import SawsimRunner
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
 FIGURE = pylab.figure()  # avoid memory problems.
 """`pylab` keeps internal references to all created figures, so share
 a single instance.

diff --git a/pysawsim/sawsim.py b/pysawsim/sawsim.py
index 92db8839a633d1847468c91818e30fa220beca5f..ccac6f3308da100724ed7d97e0dd1eb7529f33ea 100644 (file)
--- a/pysawsim/sawsim.py
+++ b/pysawsim/sawsim.py
@@ -39,6 +39,10 @@ from . import __version__
 from .manager import MANAGERS, get_manager, InvokeJob
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
 SAWSIM = 'sawsim'  # os.path.expand(os.path.join('~', 'bin', 'sawsim'))
 CACHE_DIR = os.path.expanduser(os.path.join('~', '.sawsim-cache'))
 DEFAULT_PARAM_STRING = (

diff --git a/pysawsim/sawsim_histogram.py b/pysawsim/sawsim_histogram.py
index 3220ba4600ab9237f199a39bb168b8bc23198869..969ad4655a63a3ec5e4f06e4168ce11b5f59a199 100644 (file)
--- a/pysawsim/sawsim_histogram.py
+++ b/pysawsim/sawsim_histogram.py
@@ -24,6 +24,11 @@ from .manager import MANAGERS, get_manager
 from .sawsim import SawsimRunner
 
 
+_multiprocess_can_split_ = True
+"""Allow nosetests to split tests between processes.
+"""
+
+
 def sawsim_histogram(sawsim_runner, param_string, N=400, bin_edges=None):
         """Run `N` simulations and return a histogram with `bin_edges`.