From 1149c914605bc4f71f63fea58761396f0c41f911 Mon Sep 17 00:00:00 2001 From: stevenknight Date: Tue, 10 Dec 2002 18:17:13 +0000 Subject: [PATCH] Add functions for getting and setting the number of jobs. (Anthony Roach) git-svn-id: http://scons.tigris.org/svn/scons/trunk@521 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- doc/man/scons.1 | 16 ++++++ src/CHANGES.txt | 5 ++ src/engine/SCons/Job.py | 2 +- src/engine/SCons/Script/SConscript.py | 14 +++++ src/engine/SCons/Script/__init__.py | 12 +++-- test/option-j.py | 75 ++++++++++++++++++++++++--- 6 files changed, 114 insertions(+), 10 deletions(-) diff --git a/doc/man/scons.1 b/doc/man/scons.1 index 3e75d774..5ef46a87 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -2394,6 +2394,13 @@ foo = FindFile('foo', ['dir1', 'dir2']) .\".RI GetLaunchDir( XXX ) .\"XXX +.TP +.RI GetJobs() +Get the number of jobs (commands) that will be run simultaneously. See also +.I -j +and +.IR SetJobs() . + .TP .RI Help( text ) This specifies help text to be printed if the @@ -2610,6 +2617,15 @@ command line will not cause files to be rebuilt. "MD5" signatures take longer to compute, but "timestamp" signatures are less accurate. The default is "MD5". +.TP +.RI SetJobs( num ) +Specifies the number of jobs (commands) to run simultaneously. +.I -j +overrides this function. See also +.I -j +and +.IR GetJobs() . + .TP .RI Split( arg ) Returns a list of file names or other objects. diff --git a/src/CHANGES.txt b/src/CHANGES.txt index ee6aa1d7..e3c48592 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -14,6 +14,11 @@ RELEASE 0.10 - XXX - Remove Python bytecode (*.pyc) files from the scons-local packages. + From Anthony Roach: + + - Add SetJobs() and GetJobs() methods to allow configuration of the + number of default jobs (still overridden by -j). + RELEASE 0.09 - Thu, 5 Dec 2002 04:48:25 -0600 diff --git a/src/engine/SCons/Job.py b/src/engine/SCons/Job.py index c06cf231..aedf7688 100644 --- a/src/engine/SCons/Job.py +++ b/src/engine/SCons/Job.py @@ -42,7 +42,7 @@ class Jobs: """ create 'num' jobs using the given taskmaster. - If 'num' is equal to 0, then a serial job will be used, + If 'num' is 1 or less, then a serial job will be used, otherwise 'num' parallel jobs will be used. """ diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py index 41368a58..15f116fa 100644 --- a/src/engine/SCons/Script/SConscript.py +++ b/src/engine/SCons/Script/SConscript.py @@ -303,6 +303,18 @@ def EnsurePythonVersion(major, minor): print "Python %d.%d or greater required, but you have Python %s" %(major,minor,v) sys.exit(2) +def GetJobs(): + return SCons.Script.get_num_jobs(SCons.Script.options) + +def SetJobs(num): + try: + tmp = int(num) + if tmp < 1: + raise ValueError + SCons.Script.num_jobs = tmp + except ValueError, x: + raise SCons.Errors.UserError, "A positive integer is required: %s"%repr(num) + def BuildDefaultGlobals(): """ Create a dictionary containing all the default globals for @@ -325,6 +337,7 @@ def BuildDefaultGlobals(): globals['FindFile'] = FindFile globals['GetBuildPath'] = GetBuildPath globals['GetCommandHandler'] = SCons.Action.GetCommandHandler + globals['GetJobs'] = GetJobs globals['GetLaunchDir'] = GetLaunchDir globals['Help'] = Help globals['Import'] = Import @@ -344,6 +357,7 @@ def BuildDefaultGlobals(): globals['SetBuildSignatureType'] = SetBuildSignatureType globals['SetCommandHandler'] = SCons.Action.SetCommandHandler globals['SetContentSignatureType'] = SetContentSignatureType + globals['SetJobs'] = SetJobs globals['SharedLibrary'] = SCons.Defaults.SharedLibrary globals['SharedObject'] = SCons.Defaults.SharedObject globals['Split'] = SCons.Util.Split diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 338b98fc..a0c2f7d1 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -178,6 +178,7 @@ exit_status = 0 # exit status, assume success by default profiling = 0 repositories = [] sig_module = None +num_jobs = 1 # this is modifed by SConscript.SetJobs() def print_it(text): print text @@ -190,6 +191,12 @@ class PrintHelp(Exception): # utility functions +def get_num_jobs(options): + if hasattr(options, 'num_jobs'): + return options.num_jobs + else: + return num_jobs + def get_all_children(node): return node.all_children(None) def get_derived_children(node): @@ -604,8 +611,6 @@ class OptParser(OptionParser): opt, arglist = OptionParser.parse_args(self, args, values) if opt.implicit_deps_changed or opt.implicit_deps_unchanged: opt.implicit_cache = 1 - if not hasattr(opt, "num_jobs"): - setattr(opt, "num_jobs", 1) return opt, arglist @@ -626,6 +631,7 @@ def _main(): # it's OK if there's no SCONSFLAGS pass parser = OptParser() + global options options, args = parser.parse_args(all_args) if options.help_msg: @@ -835,7 +841,7 @@ def _main(): display("scons: Building targets ...") taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, calc) - jobs = SCons.Job.Jobs(options.num_jobs, taskmaster) + jobs = SCons.Job.Jobs(get_num_jobs(options), taskmaster) try: jobs.run() diff --git a/test/option-j.py b/test/option-j.py index b886afb1..b6e83963 100644 --- a/test/option-j.py +++ b/test/option-j.py @@ -22,6 +22,11 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # +""" +This tests the -j command line option, and the SetJobs() and GetJobs() +SConscript functions. +""" + __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import string @@ -89,27 +94,85 @@ def RunTest(args, extra): return start2, finish1 -start2, finish1 = RunTest('-j 2 f1 f2', "first") - +# Test 2 parallel jobs. # fail if the second file was not started -# before the first one was finished +# before the first one was finished. +start2, finish1 = RunTest('-j 2 f1 f2', "first") test.fail_test(not (start2 < finish1)) -s2, f1 = RunTest('-j 2 f1 f2', "first") - # re-run the test with the same input, fail if we don't # get back the same times, which would indicate that # SCons rebuilt the files even though nothing changed +s2, f1 = RunTest('-j 2 f1 f2', "first") test.fail_test(start2 != s2) test.fail_test(finish1 != f1) +# Test a single serial job. +# fail if the second file was started +# before the first one was finished start2, finish1 = RunTest('f1 f2', "second") +test.fail_test(start2 < finish1) + +# Make sure that a parallel build using a list builder +# succeedes. +test.run(arguments='-j 2 out') + +# Test SetJobs() with no -j: +test.write('SConstruct', """ +MyBuild = Builder(action = r'%s build.py $TARGETS') +env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) +env.MyBuild(target = 'f1', source = 'f1.in') +env.MyBuild(target = 'f2', source = 'f2.in') + +def copyn(env, target, source): + import shutil + import time + time.sleep(1) + for t in target: + shutil.copy(str(source[0]), str(t)) + +t = env.Command(target=['foo/foo1.out', 'foo/foo2.out'], source='foo/foo.in', action=copyn) +env.Install('out', t) + +assert GetJobs() == 1 +SetJobs(2) +assert GetJobs() == 2 +""" % python) + +# This should be a prallel build because the SConscript sets jobs to 2. +# fail if the second file was not started +# before the first one was finished +start2, finish1 = RunTest('f1 f2', "third") +test.fail_test(not (start2 < finish1)) + +# Test SetJobs() with -j: +test.write('SConstruct', """ +MyBuild = Builder(action = r'%s build.py $TARGETS') +env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) +env.MyBuild(target = 'f1', source = 'f1.in') +env.MyBuild(target = 'f2', source = 'f2.in') +def copyn(env, target, source): + import shutil + import time + time.sleep(1) + for t in target: + shutil.copy(str(source[0]), str(t)) + +t = env.Command(target=['foo/foo1.out', 'foo/foo2.out'], source='foo/foo.in', action=copyn) +env.Install('out', t) + +assert GetJobs() == 1 +SetJobs(2) +assert GetJobs() == 1 +""" % python) + +# This should be a serial build since -j 1 overrides the call to SetJobs(). # fail if the second file was started # before the first one was finished +start2, finish1 = RunTest('-j 1 f1 f2', "fourth") test.fail_test(start2 < finish1) -test.run(arguments='-j 2 out') test.pass_test() -- 2.26.2