From: stevenknight Date: Thu, 6 Feb 2003 04:59:18 +0000 (+0000) Subject: Add the --random option. X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=b72b72bc7031dbea4e8929da2363df3477fbcf40;p=scons.git Add the --random option. git-svn-id: http://scons.tigris.org/svn/scons/trunk@574 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- diff --git a/doc/man/scons.1 b/doc/man/scons.1 index 1e9086f1..c1cb9e00 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -246,6 +246,24 @@ scons -j 4 builds four targets in parallel, for example. +.B scons +can maintain a cache of target (derived) files that can +be shared between multiple builds. When caching is enabled in a +SConscript file, any target files built by +.B scons +will be copied +to the cache. If an up-to-date target file is found in the cache, it +will be retrieved from the cache instead of being rebuilt locally. +Caching behavior may be disabled and controlled in other ways by the +.BR --cache-force , +.BR --cache-disable , +and +.B --cache-show +command-line options. The +.B --random +option is useful to prevent multiple builds +from trying to update the cache simultaneously. + Values of variables to be passed to the SConscript file(s) may be specified on the command line: @@ -265,24 +283,6 @@ else: env = Environment() .EE -.\" .B scons -.\" can maintain a cache of target (derived) files that can -.\" be shared between multiple builds. When caching is enabled in a -.\" SConscript file, any target files built by -.\" .B scons -.\" will be copied -.\" to the cache. If an up-to-date target file is found in the cache, it -.\" will be retrieved from the cache instead of being rebuilt locally. -.\" Caching behavior may be disabled and controlled in other ways by the -.\" .BR --cache-force , -.\" .BR --cache-disable , -.\" and -.\" .B --cache-show -.\" command-line options. The -.\" .B --random -.\" option is useful whenever multiple builds may be -.\" trying to update the cache simultaneously. - .B scons requires Python version 1.5.2 or later. There should be no other dependencies or requirements to run @@ -633,13 +633,13 @@ to rebuild target files are still printed. .\" -r, -R, --no-builtin-rules, --no-builtin-variables .\" Clear the default construction variables. Construction .\" environments that are created will be completely empty. -.\" -.\" .TP -.\" --random -.\" Build dependencies in a random order. This is useful when -.\" building multiple trees simultaneously with caching enabled as a -.\" way to prevent multiple builds from simultaneously trying to build -.\" or retrieve the same target files. + +.TP +--random +Build dependencies in a random order. This is useful when +building multiple trees simultaneously with caching enabled, +to prevent multiple builds from simultaneously trying to build +or retrieve the same target files. .TP -s, --silent, --quiet diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 08a47a7e..2057797f 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -727,7 +727,6 @@ class Dir(Entry): return self.entries['..'].root() def all_children(self, scan): - #XXX --random: randomize "dependencies?" keys = filter(lambda k: k != '.' and k != '..', self.entries.keys()) kids = map(lambda x, s=self: s.entries[x], keys) def c(one, two): diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index eed045de..41b95dcc 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -41,6 +41,7 @@ start_time = time.time() import os import os.path +import random import string import sys import traceback @@ -544,6 +545,9 @@ class OptParser(OptionParser): default=0, help="Don't print SCons progress messages.") + self.add_option('--random', dest="random", action="store_true", + default=0, help="Build dependencies in random order.") + self.add_option('-s', '--silent', '--quiet', action="store_true", default=0, help="Don't print commands.") @@ -607,10 +611,6 @@ class OptParser(OptionParser): callback=opt_not_yet, # help="Clear default environments and variables." help=SUPPRESS_HELP) - self.add_option('--random', action="callback", - callback=opt_not_yet, - # help="Build dependencies in random order." - help=SUPPRESS_HELP) self.add_option('-w', '--print-directory', action="callback", callback=opt_not_yet, # help="Print the current directory." @@ -868,8 +868,23 @@ def _main(): calc = SCons.Sig.default_calc + if options.random: + def order(dependencies): + """Randomize the dependencies.""" + # This is cribbed from the implementation of + # random.shuffle() in Python 2.X. + d = dependencies + for i in xrange(len(d)-1, 0, -1): + j = int(random.random() * (i+1)) + d[i], d[j] = d[j], d[i] + return d + else: + def order(dependencies): + """Leave the order of dependencies alone.""" + return dependencies + display("scons: Building targets ...") - taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, calc) + taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, calc, order) jobs = SCons.Job.Jobs(get_num_jobs(options), taskmaster) diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index e0d933ee..521dc19f 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -159,6 +159,10 @@ class Task: side_effect.set_state(state) t.set_state(state) +def order(dependencies): + """Re-order a list of dependencies (if we need to).""" + return dependencies + class Calc: def bsig(self, node): """ @@ -180,7 +184,7 @@ class Taskmaster: the base class method, so this class can do its thing. """ - def __init__(self, targets=[], tasker=Task, calc=Calc()): + def __init__(self, targets=[], tasker=Task, calc=Calc(), order=order): self.targets = targets # top level targets self.candidates = targets[:] # nodes that might be ready to be executed self.candidates.reverse() @@ -189,6 +193,7 @@ class Taskmaster: self.tasker = tasker self.ready = None # the next task that is ready to be executed self.calc = calc + self.order = order def _find_next_ready_node(self): """Find the next node that is ready to be built""" @@ -200,12 +205,12 @@ class Taskmaster: node = self.candidates[-1] state = node.get_state() - # Skip nodes that have already been executed: + # Skip this node if it has already been executed: if state != None and state != SCons.Node.stack: self.candidates.pop() continue - # keep track of which nodes are in the execution stack: + # Mark this node as being on the execution stack: node.set_state(SCons.Node.stack) try: @@ -221,7 +226,7 @@ class Taskmaster: self.ready = node break - # detect dependency cycles: + # Detect dependency cycles: def in_stack(node): return node.get_state() == SCons.Node.stack cycle = filter(in_stack, children) if cycle: @@ -230,17 +235,18 @@ class Taskmaster: desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ") raise SCons.Errors.UserError, desc - # Add non-derived files that have not been built + # Add derived files that have not been built # to the candidates list: def derived(node): return (node.has_builder() or node.side_effect) and node.get_state() == None derived = filter(derived, children) if derived: derived.reverse() - self.candidates.extend(derived) + self.candidates.extend(self.order(derived)) continue - # Skip nodes whose side-effects are currently being built: + # Skip this node if it has side-effects that are + # currently being built: cont = 0 for side_effect in node.side_effects: if side_effect.get_state() == SCons.Node.executing: @@ -251,14 +257,16 @@ class Taskmaster: break if cont: continue - # Skip nodes that are pending on a currently executing node: + # Skip this node if it is pending on a currently + # executing node: if node.depends_on(self.executing) or node.depends_on(self.pending): self.pending.append(node) node.set_state(SCons.Node.pending) self.candidates.pop() continue - # The default when we've gotten through all of the checks above. + # The default when we've gotten through all of the checks above: + # this node is ready to be built. self.candidates.pop() self.ready = node break diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index 2b8a362e..5a19c56c 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -352,6 +352,29 @@ class TaskmasterTestCase(unittest.TestCase): assert not tm.next_task() t.executed() + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3") + n4 = Node("n4", [n1,n2,n3]) + def reverse(dependencies): + dependencies.reverse() + return dependencies + tm = SCons.Taskmaster.Taskmaster([n4], order=reverse) + t = tm.next_task() + assert t.get_target() == n3, t.get_target() + t.executed() + t = tm.next_task() + assert t.get_target() == n2, t.get_target() + t.executed() + t = tm.next_task() + assert t.get_target() == n1, t.get_target() + t.executed() + t = tm.next_task() + assert t.get_target() == n4, t.get_target() + t.executed() + + + def test_make_ready_exception(self): """Test handling exceptions from Task.make_ready() """ diff --git a/test/option--random.py b/test/option--random.py index 205a8036..f806bbee 100644 --- a/test/option--random.py +++ b/test/option--random.py @@ -24,16 +24,52 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +Verify that we build correctly using the --random option. +""" + +import os.path + import TestSCons -import string -import sys test = TestSCons.TestSCons() -test.write('SConstruct', "") +test.write('SConstruct', """ +def cat(env, source, target): + target = str(target[0]) + source = map(str, source) + f = open(target, "wb") + for src in source: + f.write(open(src, "rb").read()) + f.close() +env = Environment(BUILDERS={'Cat':Builder(action=cat)}) +env.Cat('aaa.out', 'aaa.in') +env.Cat('bbb.out', 'bbb.in') +env.Cat('ccc.out', 'ccc.in') +env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out']) +""") + +test.write('aaa.in', "aaa.in\n") +test.write('bbb.in', "bbb.in\n") +test.write('ccc.in', "ccc.in\n") + +test.run(arguments = '--random .') + +test.fail_test(test.read('all') != "aaa.in\nbbb.in\nccc.in\n") + +test.run(arguments = '-q --random .') + +test.run(arguments = '-c --random .') + +test.fail_test(os.path.exists(test.workpath('aaa.out'))) +test.fail_test(os.path.exists(test.workpath('bbb.out'))) +test.fail_test(os.path.exists(test.workpath('ccc.out'))) +test.fail_test(os.path.exists(test.workpath('all'))) + +test.run(arguments = '-q --random .', status = 1) + +test.run(arguments = '--random .') -test.run(arguments = '--random .', - stderr = "Warning: the --random option is not yet implemented\n") +test.fail_test(test.read('all') != "aaa.in\nbbb.in\nccc.in\n") test.pass_test() -