From 8807e5069af0f2cb08738e164456745cd10ad8ea Mon Sep 17 00:00:00 2001 From: stevenknight Date: Wed, 12 Jan 2005 12:54:11 +0000 Subject: [PATCH] More performance improvements? git-svn-id: http://scons.tigris.org/svn/scons/trunk@1214 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- doc/man/scons.1 | 7 ++ src/CHANGES.txt | 4 + src/engine/SCons/Memoize.py | 155 ++++++++++++++++++++++++++++---- src/engine/SCons/Script/Main.py | 76 +++++++++------- test/option/debug-memoizer.py | 53 +++++++++++ 5 files changed, 248 insertions(+), 47 deletions(-) create mode 100644 test/option/debug-memoizer.py diff --git a/doc/man/scons.1 b/doc/man/scons.1 index 9fc5a6a1..0074dfb7 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -536,6 +536,13 @@ of a given derived file: $ scons --debug=includes foo.o .EE +.TP +--debug=memoizer +Prints a summary of hits and misses in the Memoizer, +the internal SCons subsystem for caching +various values in memory instead of +recomputing them each time they're needed. + .TP --debug=memory Prints how much memory SCons uses diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 04444f25..0b47dd92 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -174,6 +174,10 @@ RELEASE 0.97 - XXX all of the normally-available global functions and variables by saying "from SCons.Script import *". + - Add a --debug=memoizer option to print Memoizer hit/mass statistics. + + - Allow more than one --debug= option to be set at a time. + From Wayne Lee: - Avoid "maximum recursion limit" errors when removing $(-$) pairs diff --git a/src/engine/SCons/Memoize.py b/src/engine/SCons/Memoize.py index 72adbcd7..1dc95cb9 100644 --- a/src/engine/SCons/Memoize.py +++ b/src/engine/SCons/Memoize.py @@ -352,6 +352,128 @@ def Memoizer_cache_get_one(func, cdict, self, arg): return rval +# +# Caching stuff is tricky, because the tradeoffs involved are often so +# non-obvious, so we're going to support an alternate set of functions +# that also count the hits and misses, to try to get a concrete idea of +# which Memoizations seem to pay off. +# +# Because different configurations can have such radically different +# performance tradeoffs, interpreting the hit/miss results will likely be +# more of an art than a science. In other words, don't assume that just +# because you see no hits in one configuration that it's not worthwhile +# Memoizing that method. +# +# Note that these are essentially cut-and-paste copies of the above +# Memozer_cache_get*() implementations, with the addition of the +# counting logic. If the above implementations change, the +# corresponding change should probably be made down below as well, +# just to try to keep things in sync. +# + +class CounterEntry: + def __init__(self): + self.hit = 0 + self.miss = 0 + +import UserDict +class Counter(UserDict.UserDict): + def __call__(self, klass, code): + k = (klass.__name__, id(code)) + try: + return self[k] + except KeyError: + c = self[k] = CounterEntry() + return c + +CacheCount = Counter() +CacheCountSelf = Counter() +CacheCountOne = Counter() + +Code_to_Name = {} + +def Dump(): + items = CacheCount.items() + CacheCountSelf.items() + CacheCountOne.items() + def keyify(t): + return Code_to_Name[(t[0], t[1])] + items = map(lambda t, k=keyify: (k(t[0]), t[1]), items) + items.sort() + for k, v in items: + print " %7d hits %7d misses %s()" % (v.hit, v.miss, k) + +def Count_cache_get(func, cdict, args, kw): + """Called instead of name to see if this method call's return + value has been cached. If it has, just return the cached + value; if not, call the actual method and cache the return.""" + + obj = args[0] + + ckey = obj._MeMoIZeR_Key + ':' + _MeMoIZeR_gen_key(args, kw) + + c = CacheCount(obj.__class__, func) + rval = cdict.get(ckey, "_MeMoIZeR") + if rval is "_MeMoIZeR": + rval = cdict[ckey] = apply(func, args, kw) + c.miss = c.miss + 1 + else: + c.hit = c.hit + 1 + + return rval + +def Count_cache_get_self(func, cdict, self): + """Called instead of func(self) to see if this method call's + return value has been cached. If it has, just return the cached + value; if not, call the actual method and cache the return. + Optimized version of Memoizer_cache_get for methods that take the + object instance as the only argument.""" + + ckey = self._MeMoIZeR_Key + + c = CacheCountSelf(self.__class__, func) + rval = cdict.get(ckey, "_MeMoIZeR") + if rval is "_MeMoIZeR": + rval = cdict[ckey] = func(self) + c.miss = c.miss + 1 + else: + c.hit = c.hit + 1 + + return rval + +def Count_cache_get_one(func, cdict, self, arg): + """Called instead of func(self, arg) to see if this method call's + return value has been cached. If it has, just return the cached + value; if not, call the actual method and cache the return. + Optimized version of Memoizer_cache_get for methods that take the + object instance and one other argument only.""" + + ckey = self._MeMoIZeR_Key + ':' + \ + (getattr(arg, "_MeMoIZeR_Key", None) or repr(arg)) + + c = CacheCountOne(self.__class__, func) + rval = cdict.get(ckey, "_MeMoIZeR") + if rval is "_MeMoIZeR": + rval = cdict[ckey] = func(self, arg) + c.miss = c.miss + 1 + else: + c.hit = c.hit + 1 + + return rval + +MCG_dict = { + 'MCG' : Memoizer_cache_get, + 'MCGS' : Memoizer_cache_get_self, + 'MCGO' : Memoizer_cache_get_one, +} + +def EnableCounting(): + global MCG_dict + MCG_dict = { + 'MCG' : Count_cache_get, + 'MCGS' : Count_cache_get_self, + 'MCGO' : Count_cache_get_one, + } + + class _Memoizer_Simple: @@ -438,7 +560,7 @@ def Analyze_Class(klass): modelklass = _Memoizer_Simple lcldict = {} - klass.__dict__.update(memoize_classdict(modelklass, lcldict, D, R)) + klass.__dict__.update(memoize_classdict(klass, modelklass, lcldict, D, R)) return klass @@ -458,39 +580,38 @@ def whoami(memoizer_funcname, real_funcname): return '...'+os.sep+'SCons'+os.sep+'Memoizer-'+ \ memoizer_funcname+'-lambda<'+real_funcname+'>' -def memoize_classdict(modelklass, new_klassdict, cacheable, resetting): +def memoize_classdict(klass, modelklass, new_klassdict, cacheable, resetting): new_klassdict.update(modelklass.__dict__) new_klassdict['_MeMoIZeR_converted'] = 1 for name,code in cacheable.items(): + Code_to_Name[(klass.__name__, id(code))] = klass.__name__ + '.' + name + eval_dict = { + 'methcode' : code, + 'methcached' : {}, + } + eval_dict.update(MCG_dict) if code.func_code.co_argcount == 1 and \ not code.func_code.co_flags & 0xC: - newmethod = eval( + compiled = \ compile("\n"*1 + "lambda self: MCGS(methcode, methcached, self)", whoami('cache_get_self', name), - "eval"), - {'methcode':code, 'methcached':{}, - 'MCGS':Memoizer_cache_get_self}, - {}) + "eval") elif code.func_code.co_argcount == 2 and \ not code.func_code.co_flags & 0xC: - newmethod = eval( + compiled = \ compile("\n"*2 + "lambda self, arg: MCGO(methcode, methcached, self, arg)", whoami('cache_get_one', name), - "eval"), - {'methcode':code, 'methcached':{}, - 'MCGO':Memoizer_cache_get_one}, - {}) + "eval") else: - newmethod = eval( + compiled = \ compile("\n"*3 + "lambda *args, **kw: MCG(methcode, methcached, args, kw)", whoami('cache_get', name), - "eval"), - {'methcode':code, 'methcached':{}, - 'MCG':Memoizer_cache_get}, {}) + "eval") + newmethod = eval(compiled, eval_dict, {}) new_klassdict[name] = newmethod for name,code in resetting.items(): @@ -662,7 +783,7 @@ else: cls_dict['_MeMoIZeR_cmp'] = C else: modelklass = _Memoizer_Simple - klassdict = memoize_classdict(modelklass, cls_dict, D, R) + klassdict = memoize_classdict(cls, modelklass, cls_dict, D, R) init = klassdict.get('__init__', None) if not init: diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index 1bd6939c..1bfd2482 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -246,6 +246,7 @@ print_dtree = 0 print_explanations = 0 print_includes = 0 print_objects = 0 +print_memoizer = 0 print_stacktrace = 0 print_stree = 0 print_time = 0 @@ -407,7 +408,7 @@ def _SConstruct_exists(dirname=''): def _set_globals(options): global repositories, keep_going_on_error, ignore_errors global print_count, print_dtree - global print_explanations, print_includes + global print_explanations, print_includes, print_memoizer global print_objects, print_stacktrace, print_stree global print_time, print_tree global memory_outf, memory_stats @@ -416,34 +417,40 @@ def _set_globals(options): repositories.extend(options.repository) keep_going_on_error = options.keep_going try: - if options.debug: - if options.debug == "count": - print_count = 1 - elif options.debug == "dtree": - print_dtree = 1 - elif options.debug == "explain": - print_explanations = 1 - elif options.debug == "findlibs": - SCons.Scanner.Prog.print_find_libs = "findlibs" - elif options.debug == "includes": - print_includes = 1 - elif options.debug == "memory": - memory_stats = [] - memory_outf = sys.stdout - elif options.debug == "objects": - print_objects = 1 - elif options.debug == "presub": - SCons.Action.print_actions_presub = 1 - elif options.debug == "stacktrace": - print_stacktrace = 1 - elif options.debug == "stree": - print_stree = 1 - elif options.debug == "time": - print_time = 1 - elif options.debug == "tree": - print_tree = 1 + debug_values = options.debug + if debug_values is None: + debug_values = [] except AttributeError: pass + else: + if "count" in debug_values: + print_count = 1 + if "dtree" in debug_values: + print_dtree = 1 + if "explain" in debug_values: + print_explanations = 1 + if "findlibs" in debug_values: + SCons.Scanner.Prog.print_find_libs = "findlibs" + if "includes" in debug_values: + print_includes = 1 + if "memoizer" in debug_values: + SCons.Memoize.EnableCounting() + print_memoizer = 1 + if "memory" in debug_values: + memory_stats = [] + memory_outf = sys.stdout + if "objects" in debug_values: + print_objects = 1 + if "presub" in debug_values: + SCons.Action.print_actions_presub = 1 + if "stacktrace" in debug_values: + print_stacktrace = 1 + if "stree" in debug_values: + print_stree = 1 + if "time" in debug_values: + print_time = 1 + if "tree" in debug_values: + print_tree = 1 ignore_errors = options.ignore_errors def _create_path(plist): @@ -534,13 +541,18 @@ class OptParser(OptionParser): "build all Default() targets.") debug_options = ["count", "dtree", "explain", "findlibs", - "includes", "memory", "objects", + "includes", "memoizer", "memory", "objects", "pdb", "presub", "stacktrace", "stree", "time", "tree"] def opt_debug(option, opt, value, parser, debug_options=debug_options): if value in debug_options: - parser.values.debug = value + try: + if parser.values.debug is None: + parser.values.debug = [] + except AttributeError: + parser.values.debug = [] + parser.values.debug.append(value) else: raise OptionValueError("Warning: %s is not a valid debug type" % value) self.add_option('--debug', action="callback", type="string", @@ -1093,6 +1105,10 @@ def _main(args, parser): SCons.Debug.listLoggedInstances('*') #SCons.Debug.dumpLoggedInstances('*') + if print_memoizer: + print "Memoizer (memory cache) hits and misses:" + SCons.Memoize.Dump() + def _exec_main(): all_args = sys.argv[1:] try: @@ -1103,7 +1119,7 @@ def _exec_main(): parser = OptParser() global options options, args = parser.parse_args(all_args) - if options.debug == "pdb": + if type(options.debug) == type([]) and "pdb" in options.debug: import pdb pdb.Pdb().runcall(_main, args, parser) else: diff --git a/test/option/debug-memoizer.py b/test/option/debug-memoizer.py new file mode 100644 index 00000000..2b79c118 --- /dev/null +++ b/test/option/debug-memoizer.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test calling the --debug=memoizer option. +""" + +import string + +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """ +def cat(target, source, env): + open(str(target[0]), 'wb').write(open(str(source[0]), 'rb').read()) +env = Environment(BUILDERS={'Cat':Builder(action=Action(cat))}) +env.Cat('file.out', 'file.in') +""") + +test.write('file.in', "file.in\n") + +test.run(arguments = '--debug=memoizer') + +expect = "Memoizer (memory cache) hits and misses" +test.fail_test(string.find(test.stdout(), expect) == -1) + +test.must_match('file.out', "file.in\n") + +test.pass_test() -- 2.26.2