From: stevenknight Date: Mon, 3 Feb 2003 17:57:01 +0000 (+0000) Subject: Add CacheDir support. X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=ea6ec41fa19d0e499121acfb5a12da3d7d9da021;p=scons.git Add CacheDir support. git-svn-id: http://scons.tigris.org/svn/scons/trunk@571 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- diff --git a/doc/man/scons.1 b/doc/man/scons.1 index a1eac4cd..7eac0e81 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -297,28 +297,40 @@ command is specified. Also remove any files or directories associated to the construction command using the Clean() function. -.\" .TP -.\" --cache-disable, --no-cache -.\" Disable caching. Will neither retrieve files from cache nor flush -.\" files to cache. Has no effect if use of caching is not specified -.\" in an SConscript file. -.\" -.\" .TP -.\" --cache-force, --cache-populate -.\" Populate a cache by forcing any already-existing up-to-date -.\" target files to the cache, in addition to files built by this -.\" invocation. This is useful to populate a new cache with -.\" appropriate target files, or to make available in the cache -.\" any target files recently built with caching disabled via the -.\" .B --cache-disable -.\" option. -.\" -.\" .TP -.\" --cache-show -.\" When retrieving a target file from a cache, show the command -.\" that would have been executed to build the file. This produces -.\" consistent output for build logs, regardless of whether a target -.\" file was rebuilt or retrieved from cache. +.TP +--cache-disable, --no-cache +Disable the derived-file caching specified by +.BR CacheDir (). +.B scons +will neither retrieve files from the cache +nor copy files to the cache. + +.TP +--cache-force, --cache-populate +When using +.BR CacheDir (), +populate a cache by copying any already-existing, up-to-date +derived files to the cache, +in addition to files built by this invocation. +This is useful to populate a new cache with +all the current derived files, +or to add to the cache any derived files +recently built with caching disabled via the +.B --cache-disable +option. + +.TP +--cache-show +When using +.BR CacheDir () +and retrieving a derived file from the cache, +show the command +that would have been executed to build the file, +instead of the usual report, +"Retrieved `file' from cache." +This will produce consistent output for build logs, +regardless of whether a target +file was rebuilt or retrieved from the cache. .TP .RI "-C" " directory" ", --directory=" directory @@ -2311,6 +2323,32 @@ also provides various additional functions, not associated with a construction environment, that SConscript files can use: +.TP +.RI AddPostAction ( target, action ) +Arranges for the specified +.I action +to be performed +after the specified +.I target +has been built. +The specified action(s) may be +an Action object, or anything that +can be converted into an Action object +(see below). + +.TP +.RI AddPreAction ( target, action ) +Arranges for the specified +.I action +to be performed +before the specified +.I target +is built. +The specified action(s) may be +an Action object, or anything that +can be converted into an Action object +(see below). + .TP .RI BuildDir( build_dir ", " src_dir ", [" duplicate ]) This specifies a build directory to use for all derived files. @@ -2356,17 +2394,82 @@ can be converted into an Action object (see below). .TP -.RI AddPreAction ( target, action ) -Arranges for the specified -.I action -to be performed -before the specified -.I target -is built. -The specified action(s) may be -an Action object, or anything that -can be converted into an Action object -(see below). +.RI CacheDir ( cache_dir ) +Specifies that +.B scons +will maintain a cache of derived files in +.I cache_dir . +The derived files in the cache will be shared +among all the builds using the same +.BR CacheDir () +call. + +When a +.BR CacheDir () +is being used and +.B scons +finds a derived file that needs to be rebuilt, +it will first look in the cache to see if a +derived file has already been built +from identical input files and an identical build action +(as incorporated into the MD5 build signature). +If so, +.B scons +will retrieve the file from the cache. +If the derived file is not present in the cache, +.B scons +will rebuild it and +then place a copy of the built file in the cache +(identified by its MD5 build signature), +so that it may be retrieved by other +builds that need to build the same derived file +from identical inputs. + +Use of a specified +.BR CacheDir() +may be disabled for any invocation +by using the +.B --cache-disable +option. + +If the +.B --cache-force +option is used, +.B scons +will place a copy of +.I all +derived files in the cache, +even if they already existed +and were not built by this invocation. +This is useful to populate a cache +the first time +.BR CacheDir () +is added to a build, +or after using the +.B --cache-disable +option. + +When using +.BR CacheDir (), +.B scons +will report, +"Retrieved `file' from cache," +unless the +.B --cache-show +option is being used. +When the +.B --cache-show +option is used, +.B scons +will print the action that +.I would +have been used to build the file, +without any indication that +the file was actually retrieved from the cache. +This is useful to generate build logs +that are equivalent regardless of whether +a given derived file has been built in-place +or retrieved from the cache. .TP .RI Clean ( target, files_or_dirs ) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index b29889e1..b2581995 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -54,6 +54,10 @@ RELEASE 0.11 - XXX objects have strfunction() methods, and the functions for building and returning a string both take the same arguments. + - Add support for new CacheDir() functionality to share derived files + between builds, with related options --cache-disable, --cache-force, + and --cache-show. + From Steve Leblanc: - Fix the output of -c -n when directories are involved, so it diff --git a/src/RELEASE.txt b/src/RELEASE.txt index d87808b6..ccb301c7 100644 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -187,21 +187,17 @@ RELEASE 0.10 - Thu, 16 Jan 2003 04:11:46 -0600 command < $SOURCE > $TARGET - If you don't put space (for example, "<$SOURCE"), SCons will not - recognize the redirection. + If you don't supply a space (for example, "<$SOURCE"), SCons will + not recognize the redirection. - Executing the -u or -U option from a source directory that has an associated BuildDir() does not build the targets in the BuildDir(). - - No support yet for the following future features: + - No support yet for the following planned command-line options: - - No support for caching built files. - - - No support yet for the following command-line options: - - -d -e -l --list-actions --list-derived --list-where - -o --override -p -r -R --random -w --write-filenames - -W --warn-undefined-variables + -d -e -l --list-actions --list-derived --list-where + -o --override -p -r -R --random -w --write-filenames + -W --warn-undefined-variables diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index 3ab910b6..960ffce9 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -155,7 +155,8 @@ class ActionBase: return cmp(self.__dict__, other.__dict__) def show(self, string): - print string + if print_actions: + print string def get_actions(self): return [self] diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index 2d3540c4..61cea537 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -187,6 +187,30 @@ class ActionBaseTestCase(unittest.TestCase): assert a1 != a3 assert a2 != a3 + def test_show(self): + """Test the show() method + """ + save = SCons.Action.print_actions + SCons.Action.print_actions = 0 + + sio = StringIO.StringIO() + sys.stdout = sio + a = SCons.Action.Action("x") + a.show("xyzzy") + s = sio.getvalue() + assert s == "", s + + SCons.Action.print_actions = 1 + + sio = StringIO.StringIO() + sys.stdout = sio + a.show("foobar") + s = sio.getvalue() + assert s == "foobar\n", s + + SCons.Action.print_actions = save + sys.stdout = StringIO.StringIO() + def test_get_actions(self): """Test the get_actions() method """ diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 923615ff..08a47a7e 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -38,6 +38,8 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import os.path +import shutil +import stat import string from UserDict import UserDict @@ -80,8 +82,6 @@ def LinkFunc(target, source, env): try : os.symlink(src, dest) except (AttributeError, OSError): - import shutil - import stat shutil.copy2(src, dest) st=os.stat(src) os.chmod(dest, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) @@ -106,6 +106,38 @@ def MkdirFunc(target, source, env): Mkdir = SCons.Action.Action(MkdirFunc, None) +def CacheRetrieveFunc(target, source, env): + t = target[0] + cachedir, cachefile = t.cachepath() + if os.path.exists(cachefile): + shutil.copy2(cachefile, t.path) + st = os.stat(cachefile) + os.chmod(t.path, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) + return 0 + return 1 + +def CacheRetrieveString(target, source, env): + t = target[0] + cachedir, cachefile = t.cachepath() + if os.path.exists(cachefile): + return "Retrieved `%s' from cache" % t.path + return None + +CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) + +CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) + +def CachePushFunc(target, source, env): + t = target[0] + cachedir, cachefile = t.cachepath() + if not os.path.isdir(cachedir): + os.mkdir(cachedir) + shutil.copy2(t.path, cachefile) + st = os.stat(t.path) + os.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) + +CachePush = SCons.Action.Action(CachePushFunc, None) + # class ParentOfRoot: """ @@ -283,6 +315,9 @@ class FS: self.pathTop = path self.Root = {} self.Top = None + self.CachePath = None + self.cache_force = None + self.cache_show = None def set_toplevel_dir(self, path): assert not self.Top, "You can only set the top-level path on an FS object that has not had its File, Dir, or Entry methods called yet." @@ -572,6 +607,9 @@ class FS: d = d.get_dir() return ret + def CacheDir(self, path): + self.CachePath = path + # XXX TODO? # Annotate with the creator # rel_path @@ -924,14 +962,48 @@ class File(Entry): except OSError: pass + def build(self): + """Actually build the file. + + This overrides the base class build() method to check for the + existence of derived files in a CacheDir before going ahead and + building them. + + This method is called from multiple threads in a parallel build, + so only do thread safe stuff here. Do thread unsafe stuff in + built(). + """ + if not self.has_builder(): + return + if self.fs.CachePath: + if self.fs.cache_show: + if CacheRetrieveSilent(self, None, None) == 0: + def do_print(action, targets, sources, env, self=self): + al = action.strfunction(targets, self.sources, env) + if not SCons.Util.is_List(al): + al = [al] + for a in al: + action.show(a) + self._for_each_action(do_print) + return + elif CacheRetrieve(self, None, None) == 0: + return + SCons.Node.Node.build(self) + def built(self): SCons.Node.Node.built(self) + if self.fs.CachePath and os.path.exists(self.path): + CachePush(self, None, None) self.found_includes = {} if hasattr(self, '_exists'): delattr(self, '_exists') if hasattr(self, '_rexists'): delattr(self, '_rexists') + def visited(self): + if self.fs.CachePath and self.fs.cache_force and os.path.exists(self.path): + CachePush(self, None, None) + def prepare(self): """Prepare for this file to be created.""" @@ -1020,6 +1092,13 @@ class File(Entry): def rstr(self): return str(self.rfile()) + def cachepath(self): + if self.fs.CachePath: + bsig = str(self.get_bsig()) + subdir = string.upper(bsig[0]) + dir = os.path.join(self.fs.CachePath, subdir) + return dir, os.path.join(dir, bsig) + return None, None default_fs = FS() diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 83bdccfd..fa32c25d 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -54,6 +54,10 @@ class Builder: global built_it built_it = 1 return 0 + def show(self, string): + pass + def strfunction(self, targets, sources, env): + return "" return [Action()] def targets(self, t): @@ -1206,6 +1210,116 @@ class get_actionsTestCase(unittest.TestCase): a = dir.get_actions() assert a == [], a +class CacheDirTestCase(unittest.TestCase): + def runTest(self): + """Test CacheDir functionality""" + global built_it + + fs = SCons.Node.FS.FS() + assert fs.CachePath is None, fs.CachePath + assert fs.cache_force is None, fs.cache_force + assert fs.cache_show is None, fs.cache_show + + fs.CacheDir('cache') + assert fs.CachePath == 'cache', fs.CachePath + + save_CacheRetrieve = SCons.Node.FS.CacheRetrieve + self.retrieved = [] + def retrieve_succeed(target, source, env, self=self): + self.retrieved.append(target) + return 0 + def retrieve_fail(target, source, env, self=self): + self.retrieved.append(target) + return 1 + + f1 = fs.File("cd.f1") + f1.builder_set(Builder(fs.File)) + f1.env_set(Environment()) + try: + SCons.Node.FS.CacheRetrieve = retrieve_succeed + self.retrieved = [] + built_it = None + + f1.build() + assert self.retrieved == [f1], self.retrieved + assert built_it is None, built_it + + SCons.Node.FS.CacheRetrieve = retrieve_fail + self.retrieved = [] + built_it = None + + f1.build() + assert self.retrieved == [f1], self.retrieved + assert built_it, built_it + finally: + SCons.Node.FS.CacheRetrieve = save_CacheRetrieve + + save_CacheRetrieveSilent = SCons.Node.FS.CacheRetrieveSilent + + fs.cache_show = 1 + + f2 = fs.File("cd.f2") + f2.builder_set(Builder(fs.File)) + f2.env_set(Environment()) + try: + SCons.Node.FS.CacheRetrieveSilent = retrieve_succeed + self.retrieved = [] + built_it = None + + f2.build() + assert self.retrieved == [f2], self.retrieved + assert built_it is None, built_it + + SCons.Node.FS.CacheRetrieveSilent = retrieve_fail + self.retrieved = [] + built_it = None + + f2.build() + assert self.retrieved == [f2], self.retrieved + assert built_it, built_it + finally: + SCons.Node.FS.CacheRetrieveSilent = save_CacheRetrieveSilent + + save_CachePush = SCons.Node.FS.CachePush + self.pushed = [] + def push(target, source, env, self=self): + self.pushed.append(target) + return 0 + SCons.Node.FS.CachePush = push + + try: + test = TestCmd(workdir='') + + cd_f3 = test.workpath("cd.f3") + f3 = fs.File(cd_f3) + f3.built() + assert self.pushed == [], self.pushed + test.write(cd_f3, "cd.f3\n") + f3.built() + assert self.pushed == [f3], self.pushed + + self.pushed = [] + + cd_f4 = test.workpath("cd.f4") + f4 = fs.File(cd_f4) + f4.visited() + assert self.pushed == [], self.pushed + test.write(cd_f4, "cd.f4\n") + f4.visited() + assert self.pushed == [], self.pushed + fs.cache_force = 1 + f4.visited() + assert self.pushed == [f4], self.pushed + finally: + SCons.Node.FS.CachePush = save_CachePush + + f5 = fs.File("cd.f5") + f5.set_bsig('a_fake_bsig') + cp = f5.cachepath() + dirname = os.path.join('cache', 'A') + filename = os.path.join(dirname, 'a_fake_bsig') + assert cp == (dirname, filename), cp + if __name__ == "__main__": suite = unittest.TestSuite() @@ -1216,5 +1330,6 @@ if __name__ == "__main__": suite.addTest(StringDirTestCase()) suite.addTest(prepareTestCase()) suite.addTest(get_actionsTestCase()) + suite.addTest(CacheDirTestCase()) if not unittest.TextTestRunner().run(suite).wasSuccessful(): sys.exit(1) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 3b66f431..48da843f 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -215,6 +215,14 @@ class NodeTestCase(unittest.TestCase): assert type(act.built_target[0]) == type(MyNode("bar")), type(act.built_target[0]) assert str(act.built_target[0]) == "xxx", str(act.built_target[0]) assert act.built_source == ["yyy", "zzz"], act.built_source + + def test_visited(self): + """Test the base visited() method + + Just make sure it's there and we can call it. + """ + n = SCons.Node.Node() + n.visited() def test_depends_on(self): """Test the depends_on() method diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 9641ea70..8b9a542e 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -108,13 +108,7 @@ class Node: def generate_build_env(self): return self.env.Override(self.overrides) - def build(self): - """Actually build the node. - - This method is called from multiple threads in a parallel build, - so only do thread safe stuff here. Do thread unsafe stuff in - built(). - """ + def _for_each_action(self, func): if not self.has_builder(): return None action_list = self.pre_actions + self.builder.get_actions() + \ @@ -124,10 +118,21 @@ class Node: targets = self.builder.targets(self) env = self.generate_build_env() for action in action_list: - stat = action(targets, self.sources, env) + func(action, targets, self.sources, env) + + def build(self): + """Actually build the node. + + This method is called from multiple threads in a parallel build, + so only do thread safe stuff here. Do thread unsafe stuff in + built(). + """ + def do_action(action, targets, sources, env, self=self): + stat = action(targets, sources, env) if stat: raise SCons.Errors.BuildError(node = self, errstr = "Error %d" % stat) + self._for_each_action(do_action) def built(self): """Called just after this node is sucessfully built.""" @@ -151,6 +156,11 @@ class Node: # node were presumably just changed: self.del_csig() + def visited(self): + """Called just after this node has been visited + without requiring a build..""" + pass + def depends_on(self, nodes): """Does this node depend on any of 'nodes'?""" for node in nodes: diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py index c5cccb62..7cfb3127 100644 --- a/src/engine/SCons/Script/SConscript.py +++ b/src/engine/SCons/Script/SConscript.py @@ -360,6 +360,7 @@ def BuildDefaultGlobals(): globals['ARGUMENTS'] = arguments globals['BuildDir'] = BuildDir globals['Builder'] = SCons.Builder.Builder + globals['CacheDir'] = SCons.Node.FS.default_fs.CacheDir globals['Clean'] = Clean globals['CScan'] = SCons.Defaults.CScan globals['Default'] = Default diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index a14c7ae4..8bc3da49 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -427,6 +427,18 @@ class OptParser(OptionParser): self.add_option('-C', '--directory', type="string", action = "append", help="Change to DIRECTORY before doing anything.") + self.add_option('--cache-disable', '--no-cache', + action="store_true", dest='cache_disable', default=0, + help="Do not retrieve built targets from CacheDir.") + + self.add_option('--cache-force', '--cache-populate', + action="store_true", dest='cache_force', default=0, + help="Copy already-built targets into the CacheDir.") + + self.add_option('--cache-show', + action="store_true", dest='cache_show', default=0, + help="Print build actions for files from CacheDir.") + def opt_not_yet(option, opt, value, parser): sys.stderr.write("Warning: the %s option is not yet implemented\n" % opt) sys.exit(0) @@ -555,18 +567,6 @@ class OptParser(OptionParser): self.add_option('-Y', '--repository', nargs=1, action="append", help="Search REPOSITORY for source and target files.") - self.add_option('--cache-disable', '--no-cache', action="callback", - callback=opt_not_yet, - # help = "Do not retrieve built targets from Cache." - help=SUPPRESS_HELP) - self.add_option('--cache-force', '--cache-populate', action="callback", - callback=opt_not_yet, - # help = "Copy already-built targets into the Cache." - help=SUPPRESS_HELP) - self.add_option('--cache-show', action="callback", - callback=opt_not_yet, - # help = "Print what would have built Cached targets.", - help=SUPPRESS_HELP) self.add_option('-e', '--environment-overrides', action="callback", callback=opt_not_yet, # help="Environment variables override makefiles." @@ -676,6 +676,13 @@ def _main(): display.set_mode(0) if options.silent: SCons.Action.print_actions = None + if options.cache_disable: + def disable(self): pass + SCons.Node.FS.default_fs.CacheDir = disable + if options.cache_force: + SCons.Node.FS.default_fs.cache_force = 1 + if options.cache_show: + SCons.Node.FS.default_fs.cache_show = 1 if options.directory: cdir = _create_path(options.directory) try: diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 10f074f5..e0d933ee 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -115,6 +115,9 @@ class Task: side_effect.set_state(None) t.set_state(SCons.Node.executed) t.built() + else: + for t in self.targets: + t.visited() self.tm.executed(self.node) diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index cc0b437d..2b8a362e 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -31,6 +31,7 @@ import SCons.Errors built_text = None +visited_nodes = [] executed = None scan_called = 0 @@ -64,6 +65,10 @@ class Node: global built_text built_text = built_text + " really" + def visited(self): + global visited_nodes + visited_nodes.append(self.name) + def prepare(self): self.prepared = 1 @@ -449,7 +454,32 @@ class TaskmasterTestCase(unittest.TestCase): def test_executed(self): """Test when a task has been executed """ - pass + global built_text + global visited_nodes + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + built_text = "xxx" + visited_nodes = [] + n1.set_state(SCons.Node.executing) + t.executed() + s = n1.get_state() + assert s == SCons.Node.executed, s + assert built_text == "xxx really", built_text + assert visited_nodes == [], visited_nodes + + n2 = Node("n2") + tm = SCons.Taskmaster.Taskmaster([n2]) + t = tm.next_task() + built_text = "should_not_change" + visited_nodes = [] + n2.set_state(None) + t.executed() + s = n1.get_state() + assert s == SCons.Node.executed, s + assert built_text == "should_not_change", built_text + assert visited_nodes == ["n2"], visited_nodes def test_prepare(self): """Test preparation of multiple Nodes for a task diff --git a/test/CacheDir.py b/test/CacheDir.py new file mode 100644 index 00000000..ffe74440 --- /dev/null +++ b/test/CacheDir.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test retrieving derived files from a CacheDir. +""" + +import os.path +import shutil + +import TestSCons + +test = TestSCons.TestSCons() + +test.subdir('cache', 'src') + +test.write(['src', 'SConstruct'], """ +def cat(env, source, target): + target = str(target[0]) + open('cat.out', 'ab').write(target + "\\n") + source = map(str, source) + f = open(target, "wb") + for src in source: + f.write(open(src, "rb").read()) + f.close() +env = Environment(BUILDERS={'Cat':Builder(action=cat)}) +env.Cat('aaa.out', 'aaa.in') +env.Cat('bbb.out', 'bbb.in') +env.Cat('ccc.out', 'ccc.in') +env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out']) +CacheDir(r'%s') +""" % test.workpath('cache')) + +test.write(['src', 'aaa.in'], "aaa.in\n") +test.write(['src', 'bbb.in'], "bbb.in\n") +test.write(['src', 'ccc.in'], "ccc.in\n") + +# Verify that a normal build works correctly, and clean up. +# This should populate the cache with our derived files. +test.run(chdir = 'src', arguments = '.') + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(test.read(['src', 'cat.out']) != "aaa.out\nbbb.out\nccc.out\nall\n") + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') +test.unlink(['src', 'cat.out']) + +# Verify that we now retrieve the derived files from cache, +# not rebuild them. Then clean up. +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) + +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') + +# Verify that rebuilding with -n reports that everything was retrieved +# from the cache, but that nothing really was. +test.run(chdir = 'src', arguments = '-n .', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) + +test.fail_test(os.path.exists(test.workpath('src', 'aaa.out'))) +test.fail_test(os.path.exists(test.workpath('src', 'bbb.out'))) +test.fail_test(os.path.exists(test.workpath('src', 'ccc.out'))) +test.fail_test(os.path.exists(test.workpath('src', 'all'))) + +# Verify that rebuilding with -s retrieves everything from the cache +# even though it doesn't report anything. +test.run(chdir = 'src', arguments = '-s .', stdout = "") + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') + +# Verify that updating one input file builds its derived file and +# dependency but that the other files are retrieved from cache. +test.write(['src', 'bbb.in'], "bbb.in 2\n") + +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +cat("bbb.out", "bbb.in") +Retrieved `ccc.out' from cache +cat("all", ["aaa.out", "bbb.out", "ccc.out"]) +""")) + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in 2\nccc.in\n") +test.fail_test(test.read(['src', 'cat.out']) != "bbb.out\nall\n") + +test.up_to_date(chdir = 'src', arguments = '.') + +# All done. +test.pass_test() diff --git a/test/option--cd.py b/test/option--cd.py index 0f83bd33..e68256bb 100644 --- a/test/option--cd.py +++ b/test/option--cd.py @@ -24,19 +24,106 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +Test the --cache-disable option when retrieving derived files from a +CacheDir. +""" + +import os.path +import shutil + import TestSCons -import string -import sys test = TestSCons.TestSCons() -test.write('SConstruct', "") +test.subdir('cache', 'src') + +test.write(['src', 'SConstruct'], """ +def cat(env, source, target): + target = str(target[0]) + open('cat.out', 'ab').write(target + "\\n") + source = map(str, source) + f = open(target, "wb") + for src in source: + f.write(open(src, "rb").read()) + f.close() +env = Environment(BUILDERS={'Cat':Builder(action=cat)}) +env.Cat('aaa.out', 'aaa.in') +env.Cat('bbb.out', 'bbb.in') +env.Cat('ccc.out', 'ccc.in') +env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out']) +CacheDir(r'%s') +""" % test.workpath('cache')) + +test.write(['src', 'aaa.in'], "aaa.in\n") +test.write(['src', 'bbb.in'], "bbb.in\n") +test.write(['src', 'ccc.in'], "ccc.in\n") + +# Verify that a normal build works correctly, and clean up. +# This should populate the cache with our derived files. +test.run(chdir = 'src', arguments = '.') + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(test.read(['src', 'cat.out']) != "aaa.out\nbbb.out\nccc.out\nall\n") + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') +test.unlink(['src', 'cat.out']) + +# Verify that we now retrieve the derived files from cache, +# not rebuild them. Then clean up. +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') + +# Verify that when run with --cache-disable, we rebuild the files even +# though they're available from the cache. Then clean up. +test.run(chdir = 'src', + arguments = '--cache-disable .', + stdout = test.wrap_stdout("""\ +cat("aaa.out", "aaa.in") +cat("bbb.out", "bbb.in") +cat("ccc.out", "ccc.in") +cat("all", ["aaa.out", "bbb.out", "ccc.out"]) +""")) + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(test.read(['src', 'cat.out']) != "aaa.out\nbbb.out\nccc.out\nall\n") + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') +test.unlink(['src', 'cat.out']) + +# Verify that when run with --no-cache, we rebuild the files even +# though they're available from the cache. Then clean up. +test.run(chdir = 'src', + arguments = '--no-cache .', + stdout = test.wrap_stdout("""\ +cat("aaa.out", "aaa.in") +cat("bbb.out", "bbb.in") +cat("ccc.out", "ccc.in") +cat("all", ["aaa.out", "bbb.out", "ccc.out"]) +""")) + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(test.read(['src', 'cat.out']) != "aaa.out\nbbb.out\nccc.out\nall\n") -test.run(arguments = '--cache-disable .', - stderr = "Warning: the --cache-disable option is not yet implemented\n") +test.up_to_date(chdir = 'src', arguments = '.') -test.run(arguments = '--no-cache .', - stderr = "Warning: the --no-cache option is not yet implemented\n") +test.run(chdir = 'src', arguments = '-c .') +test.unlink(['src', 'cat.out']) +# All done. test.pass_test() - diff --git a/test/option--cf.py b/test/option--cf.py index 94e1fdb7..47847d43 100644 --- a/test/option--cf.py +++ b/test/option--cf.py @@ -24,19 +24,100 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +Test populating a CacheDir with the --cache-force option. +""" + +import os.path +import shutil + import TestSCons -import string -import sys test = TestSCons.TestSCons() -test.write('SConstruct', "") +test.subdir('cache', 'src') + +test.write(['src', 'SConstruct'], """ +def cat(env, source, target): + target = str(target[0]) + open('cat.out', 'ab').write(target + "\\n") + source = map(str, source) + f = open(target, "wb") + for src in source: + f.write(open(src, "rb").read()) + f.close() +env = Environment(BUILDERS={'Cat':Builder(action=cat)}) +env.Cat('aaa.out', 'aaa.in') +env.Cat('bbb.out', 'bbb.in') +env.Cat('ccc.out', 'ccc.in') +env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out']) +CacheDir(r'%s') +""" % test.workpath('cache')) + +test.write(['src', 'aaa.in'], "aaa.in\n") +test.write(['src', 'bbb.in'], "bbb.in\n") +test.write(['src', 'ccc.in'], "ccc.in\n") + +# Verify that a normal build works correctly, and clean up. +# This should populate the cache with our derived files. +test.run(chdir = 'src', arguments = '.') + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(test.read(['src', 'cat.out']) != "aaa.out\nbbb.out\nccc.out\nall\n") + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') +test.unlink(['src', 'cat.out']) + +# Verify that we now retrieve the derived files from cache, +# not rebuild them. DO NOT CLEAN UP. +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) + +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.up_to_date(chdir = 'src', arguments = '.') + +# Blow away and recreate the CacheDir, then verify that --cache-force +# repopulates the cache with the local built targets. DO NOT CLEAN UP. +shutil.rmtree(test.workpath('cache')) +test.subdir('cache') + +test.run(chdir = 'src', arguments = '--cache-force .') + +test.run(chdir = 'src', arguments = '-c .') + +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) + +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +# Blow away and recreate the CacheDir, then verify that --cache-populate +# repopulates the cache with the local built targets. DO NOT CLEAN UP. +shutil.rmtree(test.workpath('cache')) +test.subdir('cache') + +test.run(chdir = 'src', arguments = '--cache-populate .') + +test.run(chdir = 'src', arguments = '-c .') -test.run(arguments = '--cache-force .', - stderr = "Warning: the --cache-force option is not yet implemented\n") +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) -test.run(arguments = '--cache-populate .', - stderr = "Warning: the --cache-populate option is not yet implemented\n") +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) +# All done. test.pass_test() - diff --git a/test/option--cs.py b/test/option--cs.py index 1ed6eb49..ff1b221a 100644 --- a/test/option--cs.py +++ b/test/option--cs.py @@ -24,16 +24,125 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +Test printing build actions when using the --cache-show option and +retrieving derived files from a CacheDir. +""" + +import os.path +import shutil + import TestSCons -import string -import sys + +python = TestSCons.python test = TestSCons.TestSCons() -test.write('SConstruct', "") +test.subdir('cache', 'src') + +test.write(['src', 'build.py'], r""" +import sys +open('cat.out', 'ab').write(sys.argv[1] + "\n") +file = open(sys.argv[1], 'wb') +for src in sys.argv[2:]: + file.write(open(src, 'rb').read()) +file.close() +""") + +test.write(['src', 'SConstruct'], """ +def cat(env, source, target): + target = str(target[0]) + open('cat.out', 'ab').write(target + "\\n") + source = map(str, source) + f = open(target, "wb") + for src in source: + f.write(open(src, "rb").read()) + f.close() +env = Environment(BUILDERS={'Internal':Builder(action=cat), + 'External':Builder(action='%s build.py $TARGET $SOURCES')}) +env.External('aaa.out', 'aaa.in') +env.External('bbb.out', 'bbb.in') +env.Internal('ccc.out', 'ccc.in') +env.Internal('all', ['aaa.out', 'bbb.out', 'ccc.out']) +CacheDir(r'%s') +""" % (python, test.workpath('cache'))) + +test.write(['src', 'aaa.in'], "aaa.in\n") +test.write(['src', 'bbb.in'], "bbb.in\n") +test.write(['src', 'ccc.in'], "ccc.in\n") + +# Verify that a normal build works correctly, and clean up. +# This should populate the cache with our derived files. +test.run(chdir = 'src', arguments = '.') + +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") + +test.fail_test(test.read(['src', 'cat.out']) != "aaa.out\nbbb.out\nccc.out\nall\n") + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') +test.unlink(['src', 'cat.out']) + +# Verify that we now retrieve the derived files from cache, +# not rebuild them. Then clean up. +test.run(chdir = 'src', arguments = '.', stdout = test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) + +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') + +# Verify that using --cache-show reports the files as being rebuilt, +# even though we actually fetch them from the cache. Then clean up. +test.run(chdir = 'src', + arguments = '--cache-show .', + stdout = test.wrap_stdout("""\ +%s build.py aaa.out aaa.in +%s build.py bbb.out bbb.in +cat("ccc.out", "ccc.in") +cat("all", ["aaa.out", "bbb.out", "ccc.out"]) +""" % (python, python))) + +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.up_to_date(chdir = 'src', arguments = '.') + +test.run(chdir = 'src', arguments = '-c .') + +# Verify that using --cache-show -n reports the files as being rebuilt, +# even though we don't actually fetch them from the cache. No need to +# clean up. +test.run(chdir = 'src', + arguments = '--cache-show -n .', + stdout = test.wrap_stdout("""\ +%s build.py aaa.out aaa.in +%s build.py bbb.out bbb.in +cat("ccc.out", "ccc.in") +cat("all", ["aaa.out", "bbb.out", "ccc.out"]) +""" % (python, python))) + +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) + +test.fail_test(os.path.exists(test.workpath('src', 'aaa.out'))) +test.fail_test(os.path.exists(test.workpath('src', 'bbb.out'))) +test.fail_test(os.path.exists(test.workpath('src', 'ccc.out'))) +test.fail_test(os.path.exists(test.workpath('src', 'all'))) + +# Verify that using --cache-show -s doesn't report anything, even though +# we do fetch the files from the cache. No need to clean up. +test.run(chdir = 'src', + arguments = '--cache-show -s .', + stdout = "") -test.run(arguments = '--cache-show .', - stderr = "Warning: the --cache-show option is not yet implemented\n") +test.fail_test(test.read(['src', 'all']) != "aaa.in\nbbb.in\nccc.in\n") +test.fail_test(os.path.exists(test.workpath('src', 'cat.out'))) +# All done. test.pass_test() -