From: stevenknight Date: Sat, 6 Mar 2004 14:09:36 +0000 (+0000) Subject: scons.0.92 - Implement a --duplicate= option. (Christoph Wiedemann) X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=5a920b431434ff2429687466ad9febe661fe82fe;p=scons.git scons.0.92 - Implement a --duplicate= option. (Christoph Wiedemann) git-svn-id: http://scons.tigris.org/svn/scons/trunk@918 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- diff --git a/doc/man/scons.1 b/doc/man/scons.1 index c4c0f0cf..bea43c32 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -601,6 +601,25 @@ targets specified on the command line will still be processed. .\" average is at least .\" .I N .\" (a floating-point number). + +.TP +.RI --duplicate= ORDER +There are three ways to duplicate files in a build tree: hard links, +soft (symbolic) links and copies. The default behaviour of SCons is to +prefer hard links to soft links to copies. You can specify different +behaviours with this option. +.IR ORDER +must be one of +.IR hard-soft-copy +(the default), +.IR soft-hard-copy , +.IR hard-copy , +.IR soft-copy +or +.IR copy . +SCons will attempt to duplicate files using +the mechanisms in the specified order. + .\" .\" .TP .\" --list-derived @@ -2095,7 +2114,9 @@ into and then build the derived files within the copied tree. (The duplication is performed by linking or copying, -depending on the platform.) +depending on the platform; see also the +.IR --duplicate +option.) This guarantees correct builds regardless of whether intermediate source files are generated during the build, @@ -3344,10 +3365,19 @@ SConsignFile("/home/me/SCons/signatures") .TP .RI env.SetOption( name ", " value ) This function provides a way to set a select subset of the scons command -line options from a SConscript file. The options supported are: clean which -cooresponds to -c, --clean, and --remove; implicit_cache which corresponds -to --implicit-cache; max_drift which corresponds to --max-drift; and -num_jobs which corresponds to -j and --jobs. See the documentation for the +line options from a SConscript file. The options supported are: +.B clean +which corresponds to -c, --clean, and --remove; +.B duplicate +which +corresponds to --duplicate; +.B implicit_cache +which corresponds to --implicit-cache; +.B max_drift +which corresponds to --max-drift; +.B num_jobs +which corresponds to -j and --jobs. +See the documentation for the corresponding command line object for information about each specific option. Example: diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 64e8d5be..e2d1b970 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -447,6 +447,8 @@ RELEASE 0.93 - Thu, 23 Oct 2003 07:26:55 -0500 - Handle interrupts better by eliminating all try:-except: blocks which caught any and all exceptions, including KeyboardInterrupt. + - Add a --duplicate= option to control how files are duplicated. + RELEASE 0.92 - Wed, 20 Aug 2003 03:45:28 -0500 diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index dbcaf8a1..3bf2ee4d 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -66,29 +66,77 @@ import SCons.Warnings # there should be *no* changes to the external file system(s)... # +def _copy_func(src, dest): + shutil.copy2(src, dest) + st=os.stat(src) + os.chmod(dest, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) + +Valid_Duplicates = ['hard-soft-copy', 'soft-hard-copy', + 'hard-copy', 'soft-copy', 'copy'] + +Link_Funcs = [] # contains the callables of the specified duplication style + +def set_duplicate(duplicate): + # Fill in the Link_Funcs list according to the argument + # (discarding those not available on the platform). + + # Set up the dictionary that maps the argument names to the + # underlying implementations. We do this inside this function, + # not in the top-level module code, so that we can remap os.link + # and os.symlink for testing purposes. + try: + _hardlink_func = os.link + except AttributeError: + _hardlink_func = None + + try: + _softlink_func = os.symlink + except AttributeError: + _softlink_func = None + + link_dict = { + 'hard' : _hardlink_func, + 'soft' : _softlink_func, + 'copy' : _copy_func + } + + if not duplicate in Valid_Duplicates: + raise SCons.Errors.InternalError, ("The argument of set_duplicate " + "should be in Valid_Duplicates") + global Link_Funcs + Link_Funcs = [] + for func in string.split(duplicate,'-'): + if link_dict[func]: + Link_Funcs.append(link_dict[func]) + def LinkFunc(target, source, env): - t = target[0] - dest = t.path - fs = t.fs - src = source[0].path + # Relative paths cause problems with symbolic links, so + # we use absolute paths, which may be a problem for people + # who want to move their soft-linked src-trees around. Those + # people should use the 'hard-copy' mode, softlinks cannot be + # used for that; at least I have no idea how ... + src = source[0].abspath + dest = target[0].abspath dir, file = os.path.split(dest) - if dir and not fs.isdir(dir): - fs.makedirs(dir) - # Now actually link the files. First try to make a hard link. If that - # fails, try a symlink. If that fails then just copy it. - try : - fs.link(src, dest) - except (AttributeError, OSError): - try : - fs.symlink(src, dest) - except (AttributeError, OSError): - fs.copy2(src, dest) - st = fs.stat(src) - fs.chmod(dest, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) + if dir and not os.path.isdir(dir): + os.makedirs(dir) + if not Link_Funcs: + # Set a default order of link functions. + set_duplicate('hard-soft-copy') + # Now link the files with the previously specified order. + for func in Link_Funcs: + try: + func(src,dest) + break + except OSError: + if func == Link_Funcs[-1]: + # exception of the last link method (copy) are fatal + raise + else: + pass return 0 Link = SCons.Action.Action(LinkFunc, None) - def LocalString(target, source, env): return 'Local copy of %s from %s' % (target[0], source[0]) diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 85ae4d9f..63ac29d1 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -413,15 +413,14 @@ class BuildDirTestCase(unittest.TestCase): class LinkSimulator : """A class to intercept os.[sym]link() calls and track them.""" - def __init__( self ) : + def __init__( self, duplicate ) : + self.duplicate = duplicate self._reset() def _reset( self ) : """Reset the simulator if necessary""" if not self._need_reset() : return # skip if not needed now - self.link_called = False - self.symlink_called = False - self.copy_called = False + self.links_to_be_called = self.duplicate def _need_reset( self ) : """ @@ -430,85 +429,98 @@ class BuildDirTestCase(unittest.TestCase): or if all three methods have been tried already. """ return ( - ( not hasattr( self , "link_called" ) ) + ( not hasattr( self , "links_to_be_called" ) ) or - ( self.link_called and - self.symlink_called and - self.copy_called ) + (self.links_to_be_called == "") ) def link_fail( self , src , dest ) : self._reset() - assert not self.symlink_called , \ - "Wrong link order: symlink tried before hard link." - assert not self.copy_called , \ - "Wrong link order: copy tried before hard link." - self.link_called = True + l = string.split(self.links_to_be_called, "-") + next_link = l[0] + assert next_link == "hard", \ + "Wrong link order: expected %s to be called "\ + "instead of hard" % next_link + self.links_to_be_called = string.join(l[1:], '-') raise OSError( "Simulating hard link creation error." ) def symlink_fail( self , src , dest ) : self._reset() - assert self.link_called , \ - "Wrong link order: hard link not tried before symlink." - assert not self.copy_called , \ - "Wrong link order: copy tried before symlink link." - self.symlink_called = True + l = string.split(self.links_to_be_called, "-") + next_link = l[0] + assert next_link == "soft", \ + "Wrong link order: expected %s to be called "\ + "instead of soft" % next_link + self.links_to_be_called = string.join(l[1:], '-') raise OSError( "Simulating symlink creation error." ) def copy( self , src , dest ) : self._reset() - assert self.link_called , \ - "Wrong link order: hard link not tried before copy." - assert self.symlink_called , \ - "Wrong link order: symlink not tried before copy." - self.copy_called = True + l = string.split(self.links_to_be_called, "-") + next_link = l[0] + assert next_link == "copy", \ + "Wrong link order: expected %s to be called "\ + "instead of copy" % next_link + self.links_to_be_called = string.join(l[1:], '-') # copy succeeds, but use the real copy self._real_copy(src, dest) # end class LinkSimulator - simulator = LinkSimulator() - # save the real functions for later restoration - real_link = None - real_symlink = None try: - real_link = os.link - except AttributeError: - pass - try: - real_symlink = os.symlink - except AttributeError: + SCons.Node.FS.set_duplicate("no-link-order") + assert 0, "Expected exception when passing an invalid duplicate to set_duplicate" + except SCons.Errors.InternalError: pass - real_copy = shutil.copy2 - simulator._real_copy = real_copy # the simulator needs the real one + + for duplicate in SCons.Node.FS.Valid_Duplicates: + simulator = LinkSimulator(duplicate) - # override the real functions with our simulation - os.link = simulator.link_fail - os.symlink = simulator.symlink_fail - shutil.copy2 = simulator.copy + # save the real functions for later restoration + real_link = None + real_symlink = None + try: + real_link = os.link + except AttributeError: + pass + try: + real_symlink = os.symlink + except AttributeError: + pass + real_copy = shutil.copy2 + simulator._real_copy = real_copy # the simulator needs the real one + + # override the real functions with our simulation + os.link = simulator.link_fail + os.symlink = simulator.symlink_fail + shutil.copy2 = simulator.copy + SCons.Node.FS.set_duplicate(duplicate) + + src_foo = test.workpath('src', 'foo') + build_foo = test.workpath('build', 'foo') - try: - test.write('src/foo', 'src/foo\n') try: - os.chmod(test.workpath('src/foo'), stat.S_IRUSR) - SCons.Node.FS.Link(fs.File(test.workpath('build/foo')), - fs.File(test.workpath('src/foo')), - None) - os.chmod(test.workpath('src/foo'), stat.S_IRUSR | stat.S_IWRITE) - finally: - test.unlink( "src/foo" ) - test.unlink( "build/foo" ) + test.write(src_foo, 'src/foo\n') + os.chmod(src_foo, stat.S_IRUSR) + try: + SCons.Node.FS.Link(fs.File(build_foo), + fs.File(src_foo), + None) + finally: + os.chmod(src_foo, stat.S_IRUSR | stat.S_IWRITE) + test.unlink(src_foo) + test.unlink(build_foo) - finally: - # restore the real functions - if real_link: - os.link = real_link - else: - delattr(os, 'link') - if real_symlink: - os.symlink = real_symlink - else: - delattr(os, 'symlink') - shutil.copy2 = real_copy + finally: + # restore the real functions + if real_link: + os.link = real_link + else: + delattr(os, 'link') + if real_symlink: + os.symlink = real_symlink + else: + delattr(os, 'symlink') + shutil.copy2 = real_copy class FSTestCase(unittest.TestCase): def runTest(self): diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 7067ab1a..47fef2b6 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -608,6 +608,17 @@ class OptParser(OptionParser): # "LOAD-AVERAGE." # type="int", help=SUPPRESS_HELP) + def opt_duplicate(option, opt, value, parser): + if not value in SCons.Node.FS.Valid_Duplicates: + raise OptionValueError("`%s' is not a valid duplication style." % value) + setattr(parser.values, 'duplicate', value) + # Set the duplicate stye right away so it can affect linking + # of SConscript files. + SCons.Node.FS.set_duplicate(value) + self.add_option('--duplicate', action="callback", type="string", + callback=opt_duplicate, nargs=1, dest="duplicate", + help="Set the preferred duplication methods. Must be one of " + + string.join(SCons.Node.FS.Valid_Duplicates, ", ")) self.add_option('--list-derived', action="callback", callback=opt_not_yet, # help="Don't build; list files that would be built." @@ -683,7 +694,8 @@ class SConscriptSettableOptions: self.settable = {'num_jobs':1, 'max_drift':SCons.Sig.default_max_drift, 'implicit_cache':0, - 'clean':0} + 'clean':0, + 'duplicate':'hard-soft-copy'} def get(self, name): if not self.settable.has_key(name): @@ -709,6 +721,16 @@ class SConscriptSettableOptions: value = int(value) except ValueError, x: raise SCons.Errors.UserError, "An integer is required: %s"%repr(value) + elif name == 'duplicate': + try: + value = str(value) + except ValueError: + raise SCons.Errors.UserError, "A string is required: %s"%repr(value) + if not value in SCons.Node.FS.Valid_Duplicates: + raise SCons.Errors.UserError, "Not a valid duplication style: %s" % value + # Set the duplicate stye right away so it can affect linking + # of SConscript files. + SCons.Node.FS.set_duplicate(value) self.settable[name] = value @@ -745,7 +767,7 @@ def _main(args, parser): CleanTask.execute = CleanTask.show if options.question: SCons.SConf.dryrun = 1 - + if options.no_progress or options.silent: progress_display.set_mode(0) if options.silent: @@ -877,6 +899,7 @@ def _main(args, parser): # Now that we've read the SConscripts we can set the options # that are SConscript settable: SCons.Node.implicit_cache = ssoptions.get('implicit_cache') + SCons.Node.FS.set_duplicate(ssoptions.get('duplicate')) lookup_top = None if targets: diff --git a/test/option--duplicate.py b/test/option--duplicate.py new file mode 100644 index 00000000..f589ae08 --- /dev/null +++ b/test/option--duplicate.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +This tests the --duplicate command line option, and the duplicate +SConscript settable option. +""" + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import string +import sys +import os +import os.path +import stat +import TestSCons + +python = TestSCons.python + +test = TestSCons.TestSCons() + +test.write('SConstruct', """ +try: + duplicate = ARGUMENTS['duplicate'] + SetOption('duplicate', duplicate) +except KeyError: + pass +BuildDir('build', '.', duplicate=1) +SConscript('build/SConscript') +""") + +test.write('SConscript', '') + +hard = hasattr(os,'link') +soft = hasattr(os,'symlink') +copy = 1 # should always work + +bss = test.workpath('build/SConscript') + +criterion = { + 'hard' : lambda nl, islink: nl == 2 and not islink, + 'soft' : lambda nl, islink: nl == 1 and islink, + 'copy' : lambda nl, islink: nl == 1 and not islink, +} + +description = { + 'hard' : 'a hard link', + 'soft' : 'a soft link', + 'copy' : 'copied', +} + +def testLink(file, type): + nl = os.stat(file)[stat.ST_NLINK] + islink = os.path.islink(file) + assert criterion[type](nl, islink), \ + "Expected %s to be %s (nl %d, islink %d)" \ + % (file, description[type], nl, islink) + +def RunTest(order, type, bss): + # Test the command-line --duplicate option. + test.run(arguments='--duplicate='+order) + testLink(bss, type) + + # Test setting the option in the SConstruct file. + test.run(arguments='duplicate='+order) + testLink(bss, type) + + # Clean up for next run. + os.unlink(bss) + +# test the default (hard-soft-copy) +if hard: type='hard' +elif soft: type='soft' +else: type='copy' +RunTest('hard-soft-copy', type, bss) + +if soft: type='soft' +elif hard: type='hard' +else: type='copy' +RunTest('soft-hard-copy', type, bss) + +if soft: type='soft' +else: type='copy' +RunTest('soft-copy', type, bss) + +if hard: type='hard' +else: type='copy' +RunTest('hard-copy', type, bss) + +type='copy' +RunTest('copy', type, bss) + +test.run(arguments='--duplicate=nonsense', status=2, stderr="""\ +usage: scons [OPTION] [TARGET] ... + +SCons error: `nonsense' is not a valid duplication style. +""") + +test.pass_test()