From 7e07146e0f970b782e92d26a5d2424502131f209 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Oct 2010 09:21:49 -0700 Subject: [PATCH] Add support for nosetests multiprocessing plugin. See python-nose/doc/doc_tests/test_multiprocess/multiprocess.rst for an explanation of the plugin and its peculiarities. On a 16-core SMP machine, the test suite now runs with: Ran 27 tests in 58.077s vs the old: Ran 23 tests in 69.325s The extra tests are repeats, probably having to do with a split-vs-share issue that I haven't tracked down yet. The tests could be faster still if we used the subproc manager by default for Python >= 2.6. I'll do that next. --- misc/hooks/pre-commit-pysawsim-check | 24 +++++++++++++++++------- pysawsim/__init__.py | 7 +++++++ pysawsim/_collections.py | 6 ++++++ pysawsim/histogram.py | 5 +++++ pysawsim/invoke.py | 5 +++++ pysawsim/parameter_error.py | 5 +++++ pysawsim/parameter_scan.py | 4 ++++ pysawsim/sawsim.py | 4 ++++ pysawsim/sawsim_histogram.py | 5 +++++ 9 files changed, 58 insertions(+), 7 deletions(-) diff --git a/misc/hooks/pre-commit-pysawsim-check b/misc/hooks/pre-commit-pysawsim-check index 6e4f27b..fc7288f 100755 --- a/misc/hooks/pre-commit-pysawsim-check +++ b/misc/hooks/pre-commit-pysawsim-check @@ -1,31 +1,41 @@ #!/bin/sh +# Detect number of cores with Python >= 2.6. +# For Python 2.5, you'll have to hard-code the value (or sed /proc/cpuinfo ;). +CORES=$(python -c 'from multiprocessing import cpu_count; print cpu_count()' \ + || exit 1) +#CORES=1 + +NOSE="nosetests --with-doctest --doctest-tests --processes $CORES" + HAS_MPD=$(which mpdallexit 2>/dev/null) if [ -n "$HAS_MPD" ]; then + echo "Running nosetests with $CORES processes via MPI." LOCAL_MPD='' mpdtrace >/dev/null 2>&1 if [ "$?" -ne 0 ]; then - LOCAL_MPD='1' - mpd & # start an mpd instance - sleep 1 # give mpd some time to start up + LOCAL_MPD='1' + mpd & # start an mpd instance + sleep 1 # give mpd some time to start up fi # otherwise there is an mpd instance already running - mpiexec -n 1 nosetests --with-doctest --doctest-tests pysawsim + mpiexec -n 1 $NOSE pysawsim RESULT="$?" if [ -n "$LOCAL_MPD" ]; then - mpdallexit + mpdallexit fi if [ "$RESULT" -ne 0 ]; then - exit 1; + exit 1; fi else # no MPD + echo "Running nosetests with $CORES processes without MPI." - nosetests --with-doctest --doctest-tests pysawsim || exit 1 + $NOSE pysawsim || exit 1 fi diff --git a/pysawsim/__init__.py b/pysawsim/__init__.py index f5b7e64..199b82a 100644 --- a/pysawsim/__init__.py +++ b/pysawsim/__init__.py @@ -50,8 +50,15 @@ import logging.handlers import sys +_multiprocess_shared_ = True +"""Allow nosetests to share this module between test processes. + +This module cannot be split because _log setup is not re-entrant. +""" + __version__ = '0.10' # match sawsim version + def log(): return logging.getLogger('pysawsim') diff --git a/pysawsim/_collections.py b/pysawsim/_collections.py index e6aee98..9754470 100644 --- a/pysawsim/_collections.py +++ b/pysawsim/_collections.py @@ -2,6 +2,12 @@ from operator import itemgetter as _itemgetter from keyword import iskeyword as _iskeyword import sys as _sys + +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + + def namedtuple(typename, field_names, verbose=False): """Returns a new subclass of tuple with named fields. diff --git a/pysawsim/histogram.py b/pysawsim/histogram.py index 34d47a6..5b4480b 100644 --- a/pysawsim/histogram.py +++ b/pysawsim/histogram.py @@ -25,6 +25,11 @@ import numpy from . import log +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + + class Histogram (object): """A histogram with a flexible comparison method, `residual()`. diff --git a/pysawsim/invoke.py b/pysawsim/invoke.py index bb501e7..2d8103f 100644 --- a/pysawsim/invoke.py +++ b/pysawsim/invoke.py @@ -24,6 +24,11 @@ from subprocess import Popen, PIPE import sys +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + + class CommandError(Exception): """Represent errors in command execution. diff --git a/pysawsim/parameter_error.py b/pysawsim/parameter_error.py index 9593919..ee915b2 100755 --- a/pysawsim/parameter_error.py +++ b/pysawsim/parameter_error.py @@ -30,6 +30,11 @@ from .sawsim_histogram import sawsim_histogram from .sawsim import SawsimRunner +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + + def find_error_bounds(histogram_matcher, param_range, log_scale, threshold): if log_scale == False: ps = numpy.linspace(*param_range) diff --git a/pysawsim/parameter_scan.py b/pysawsim/parameter_scan.py index 3a4f6f9..ff34a44 100644 --- a/pysawsim/parameter_scan.py +++ b/pysawsim/parameter_scan.py @@ -36,6 +36,10 @@ from .sawsim_histogram import sawsim_histogram from .sawsim import SawsimRunner +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + FIGURE = pylab.figure() # avoid memory problems. """`pylab` keeps internal references to all created figures, so share a single instance. diff --git a/pysawsim/sawsim.py b/pysawsim/sawsim.py index 92db883..ccac6f3 100644 --- a/pysawsim/sawsim.py +++ b/pysawsim/sawsim.py @@ -39,6 +39,10 @@ from . import __version__ from .manager import MANAGERS, get_manager, InvokeJob +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + SAWSIM = 'sawsim' # os.path.expand(os.path.join('~', 'bin', 'sawsim')) CACHE_DIR = os.path.expanduser(os.path.join('~', '.sawsim-cache')) DEFAULT_PARAM_STRING = ( diff --git a/pysawsim/sawsim_histogram.py b/pysawsim/sawsim_histogram.py index 3220ba4..969ad46 100644 --- a/pysawsim/sawsim_histogram.py +++ b/pysawsim/sawsim_histogram.py @@ -24,6 +24,11 @@ from .manager import MANAGERS, get_manager from .sawsim import SawsimRunner +_multiprocess_can_split_ = True +"""Allow nosetests to split tests between processes. +""" + + def sawsim_histogram(sawsim_runner, param_string, N=400, bin_edges=None): """Run `N` simulations and return a histogram with `bin_edges`. -- 2.26.2