From 1fc834c4fa45e6ad423d53e8d0f10117b7bcdbd7 Mon Sep 17 00:00:00 2001 From: stevenknight Date: Sat, 1 May 2004 19:12:23 +0000 Subject: [PATCH] Performance improvement: memo-ize Node.FS string values when appropriate. git-svn-id: http://scons.tigris.org/svn/scons/trunk@964 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- src/CHANGES.txt | 3 ++ src/engine/SCons/Builder.py | 17 +++++--- src/engine/SCons/Node/FS.py | 54 ++++++++++++++++++++++--- src/engine/SCons/Node/FSTests.py | 62 +++++++++++++++++++++++++++++ src/engine/SCons/Node/NodeTests.py | 6 +++ src/engine/SCons/Node/__init__.py | 3 ++ src/engine/SCons/Script/__init__.py | 6 +++ 7 files changed, 141 insertions(+), 10 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index f34104d9..1d54390a 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -101,6 +101,9 @@ RELEASE 0.96 - XXX - Add support for the strfunction argument to all types of Actions: CommandAction, ListAction, and CommandGeneratorAction. + - Speed up turning file system Nodes into strings by caching the + values after we're finished reading the SConscript files. + From Gary Oberbrunner: - Add a --debug=presub option to print actions prior to substitution. diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index 5971056a..4bca1c81 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -667,9 +667,18 @@ class MultiStepBuilder(BuilderBase): src_suffixes = self.src_suffixes(env) for snode in slist: - base, ext = self.splitext(str(snode)) - if sdict.has_key(ext): - tgt = sdict[ext]._execute(env, None, snode, overwarn) + try: + get_suffix = snode.get_suffix + except AttributeError: + ext = self.splitext(str(snode)) + else: + ext = get_suffix() + try: + subsidiary_builder = sdict[ext] + except KeyError: + final_sources.append(snode) + else: + tgt = subsidiary_builder._execute(env, None, snode, overwarn) # Only supply the builder with sources it is capable # of building. if SCons.Util.is_List(tgt): @@ -680,8 +689,6 @@ class MultiStepBuilder(BuilderBase): final_sources.append(tgt) else: final_sources.extend(tgt) - else: - final_sources.append(snode) return BuilderBase._execute(self, env, target, final_sources, overwarn) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index a349f774..a30994d8 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -52,6 +52,30 @@ import SCons.Sig.MD5 import SCons.Util import SCons.Warnings +# +# We stringify these file system Nodes a lot. Turning a file system Node +# into a string is non-trivial, because the final string representation +# can depend on a lot of factors: whether it's a derived target or not, +# whether it's linked to a repository or source directory, and whether +# there's duplication going on. The normal technique for optimizing +# calculations like this is to memoize (cache) the string value, so you +# only have to do the calculation once. +# +# A number of the above factors, however, can be set after we've already +# been asked to return a string for a Node, because a Repository() or +# BuildDir() call or the like may not occur until later in SConscript +# files. So this variable controls whether we bother trying to save +# string values for Nodes. The wrapper interface can set this whenever +# they're done mucking with Repository and BuildDir and the other stuff, +# to let this module know it can start returning saved string values +# for Nodes. +# +Save_Strings = None + +def save_strings(val): + global Save_Strings + Save_Strings = val + # # SCons.Action objects for interacting with the outside world. # @@ -417,16 +441,32 @@ class Base(SCons.Node.Node): delattr(self, '_rexists') except AttributeError: pass + try: + delattr(self, '_str_val') + except AttributeError: + pass + self.relpath = {} def get_dir(self): return self.dir + def get_suffix(self): + return SCons.Util.splitext(self.name)[1] + def __str__(self): """A Node.FS.Base object's string representation is its path name.""" - if self.duplicate or self.is_derived(): - return self.get_path() - return self.srcnode().get_path() + try: + return self._str_val + except AttributeError: + global Save_Strings + if self.duplicate or self.is_derived(): + str_val = self.get_path() + else: + str_val = self.srcnode().get_path() + if Save_Strings: + self._str_val = str_val + return str_val def exists(self): try: @@ -573,7 +613,7 @@ class Entry(Base): return node.get_found_includes(env, scanner, target) def scanner_key(self): - return SCons.Util.splitext(self.name)[1] + return self.get_suffix() def get_contents(self): """Fetch the contents of the entry. @@ -1124,6 +1164,10 @@ class Dir(Base): del node._srcnode except AttributeError: pass + try: + del node._str_val + except AttributeError: + pass if duplicate != None: node.duplicate=duplicate @@ -1347,7 +1391,7 @@ class File(Base): return self.dir.root() def scanner_key(self): - return SCons.Util.splitext(self.name)[1] + return self.get_suffix() def get_contents(self): if not self.rexists(): diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 6fbcfdc8..d5e04e18 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -1735,23 +1735,29 @@ class clearTestCase(unittest.TestCase): e = fs.Entry('e') e._exists = 1 e._rexists = 1 + e._str_val = 'e' e.clear() assert not hasattr(e, '_exists') assert not hasattr(e, '_rexists') + assert not hasattr(e, '_str_val') d = fs.Dir('d') d._exists = 1 d._rexists = 1 + d._str_val = 'd' d.clear() assert not hasattr(d, '_exists') assert not hasattr(d, '_rexists') + assert not hasattr(d, '_str_val') f = fs.File('f') f._exists = 1 f._rexists = 1 + f._str_val = 'f' f.clear() assert not hasattr(f, '_exists') assert not hasattr(f, '_rexists') + assert not hasattr(f, '_str_val') class postprocessTestCase(unittest.TestCase): def runTest(self): @@ -1903,6 +1909,61 @@ class SpecialAttrTestCase(unittest.TestCase): caught = 1 assert caught, "did not catch expected AttributeError" +class SaveStringsTestCase(unittest.TestCase): + def runTest(self): + """Test caching string values of nodes.""" + test=TestCmd(workdir='') + + def setup(fs): + fs.Dir('src') + fs.Dir('d0') + fs.Dir('d1') + + d0_f = fs.File('d0/f') + d1_f = fs.File('d1/f') + d0_b = fs.File('d0/b') + d1_b = fs.File('d1/b') + d1_f.duplicate = 1 + d1_b.duplicate = 1 + d0_b.builder = 1 + d1_b.builder = 1 + + return [d0_f, d1_f, d0_b, d1_b] + + def modify(nodes): + d0_f, d1_f, d0_b, d1_b = nodes + d1_f.duplicate = 0 + d1_b.duplicate = 0 + d0_b.builder = 0 + d1_b.builder = 0 + + fs1 = SCons.Node.FS.FS(test.workpath('fs1')) + nodes = setup(fs1) + fs1.BuildDir('d0', 'src', duplicate=0) + fs1.BuildDir('d1', 'src', duplicate=1) + + s = map(str, nodes) + assert s == ['src/f', 'd1/f', 'd0/b', 'd1/b'], s + + modify(nodes) + + s = map(str, nodes) + assert s == ['src/f', 'src/f', 'd0/b', 'd1/b'], s + + SCons.Node.FS.save_strings(1) + fs2 = SCons.Node.FS.FS(test.workpath('fs2')) + nodes = setup(fs2) + fs2.BuildDir('d0', 'src', duplicate=0) + fs2.BuildDir('d1', 'src', duplicate=1) + + s = map(str, nodes) + assert s == ['src/f', 'd1/f', 'd0/b', 'd1/b'], s + + modify(nodes) + + s = map(str, nodes) + assert s == ['src/f', 'd1/f', 'd0/b', 'd1/b'], s + if __name__ == "__main__": @@ -1921,5 +1982,6 @@ if __name__ == "__main__": suite.addTest(clearTestCase()) suite.addTest(postprocessTestCase()) suite.addTest(SpecialAttrTestCase()) + suite.addTest(SaveStringsTestCase()) if not unittest.TextTestRunner().run(suite).wasSuccessful(): sys.exit(1) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index e9d779c2..4ea73abe 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -954,6 +954,12 @@ class NodeTestCase(unittest.TestCase): siginfo = n.get_prevsiginfo() assert siginfo == (None, None, None), siginfo + def test_get_suffix(self): + """Test the base Node get_suffix() method""" + n = SCons.Node.Node() + s = n.get_suffix() + assert s == '', s + def test_generate_build_dict(self): """Test the base Node generate_build_dict() method""" n = SCons.Node.Node() diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index a9581e40..ceac5caf 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -135,6 +135,9 @@ class Node: # what line in what file created the node, for example). Annotate(self) + def get_suffix(self): + return '' + def generate_build_dict(self): """Return an appropriate dictionary of values for building this Node.""" diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 245fea97..ee1ec81a 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -963,6 +963,12 @@ def _main(args, parser): sys.exit(0) progress_display("scons: done reading SConscript files.") + # Tell the Node.FS subsystem that we're all done reading the + # SConscript files and calling Repository() and BuildDir() and the + # like, so it can go ahead and start memoizing the string values of + # file system nodes. + SCons.Node.FS.save_strings(1) + if not memory_stats is None: memory_stats.append(SCons.Debug.memory()) fs.chdir(fs.Top) -- 2.26.2