return rval
+#
+# Caching stuff is tricky, because the tradeoffs involved are often so
+# non-obvious, so we're going to support an alternate set of functions
+# that also count the hits and misses, to try to get a concrete idea of
+# which Memoizations seem to pay off.
+#
+# Because different configurations can have such radically different
+# performance tradeoffs, interpreting the hit/miss results will likely be
+# more of an art than a science. In other words, don't assume that just
+# because you see no hits in one configuration that it's not worthwhile
+# Memoizing that method.
+#
+# Note that these are essentially cut-and-paste copies of the above
+# Memozer_cache_get*() implementations, with the addition of the
+# counting logic. If the above implementations change, the
+# corresponding change should probably be made down below as well,
+# just to try to keep things in sync.
+#
+
+class CounterEntry:
+ def __init__(self):
+ self.hit = 0
+ self.miss = 0
+
+import UserDict
+class Counter(UserDict.UserDict):
+ def __call__(self, klass, code):
+ k = (klass.__name__, id(code))
+ try:
+ return self[k]
+ except KeyError:
+ c = self[k] = CounterEntry()
+ return c
+
+CacheCount = Counter()
+CacheCountSelf = Counter()
+CacheCountOne = Counter()
+
+Code_to_Name = {}
+
+def Dump():
+ items = CacheCount.items() + CacheCountSelf.items() + CacheCountOne.items()
+ def keyify(t):
+ return Code_to_Name[(t[0], t[1])]
+ items = map(lambda t, k=keyify: (k(t[0]), t[1]), items)
+ items.sort()
+ for k, v in items:
+ print " %7d hits %7d misses %s()" % (v.hit, v.miss, k)
+
+def Count_cache_get(func, cdict, args, kw):
+ """Called instead of name to see if this method call's return
+ value has been cached. If it has, just return the cached
+ value; if not, call the actual method and cache the return."""
+
+ obj = args[0]
+
+ ckey = obj._MeMoIZeR_Key + ':' + _MeMoIZeR_gen_key(args, kw)
+
+ c = CacheCount(obj.__class__, func)
+ rval = cdict.get(ckey, "_MeMoIZeR")
+ if rval is "_MeMoIZeR":
+ rval = cdict[ckey] = apply(func, args, kw)
+ c.miss = c.miss + 1
+ else:
+ c.hit = c.hit + 1
+
+ return rval
+
+def Count_cache_get_self(func, cdict, self):
+ """Called instead of func(self) to see if this method call's
+ return value has been cached. If it has, just return the cached
+ value; if not, call the actual method and cache the return.
+ Optimized version of Memoizer_cache_get for methods that take the
+ object instance as the only argument."""
+
+ ckey = self._MeMoIZeR_Key
+
+ c = CacheCountSelf(self.__class__, func)
+ rval = cdict.get(ckey, "_MeMoIZeR")
+ if rval is "_MeMoIZeR":
+ rval = cdict[ckey] = func(self)
+ c.miss = c.miss + 1
+ else:
+ c.hit = c.hit + 1
+
+ return rval
+
+def Count_cache_get_one(func, cdict, self, arg):
+ """Called instead of func(self, arg) to see if this method call's
+ return value has been cached. If it has, just return the cached
+ value; if not, call the actual method and cache the return.
+ Optimized version of Memoizer_cache_get for methods that take the
+ object instance and one other argument only."""
+
+ ckey = self._MeMoIZeR_Key + ':' + \
+ (getattr(arg, "_MeMoIZeR_Key", None) or repr(arg))
+
+ c = CacheCountOne(self.__class__, func)
+ rval = cdict.get(ckey, "_MeMoIZeR")
+ if rval is "_MeMoIZeR":
+ rval = cdict[ckey] = func(self, arg)
+ c.miss = c.miss + 1
+ else:
+ c.hit = c.hit + 1
+
+ return rval
+
+MCG_dict = {
+ 'MCG' : Memoizer_cache_get,
+ 'MCGS' : Memoizer_cache_get_self,
+ 'MCGO' : Memoizer_cache_get_one,
+}
+
+def EnableCounting():
+ global MCG_dict
+ MCG_dict = {
+ 'MCG' : Count_cache_get,
+ 'MCGS' : Count_cache_get_self,
+ 'MCGO' : Count_cache_get_one,
+ }
+
+
class _Memoizer_Simple:
modelklass = _Memoizer_Simple
lcldict = {}
- klass.__dict__.update(memoize_classdict(modelklass, lcldict, D, R))
+ klass.__dict__.update(memoize_classdict(klass, modelklass, lcldict, D, R))
return klass
return '...'+os.sep+'SCons'+os.sep+'Memoizer-'+ \
memoizer_funcname+'-lambda<'+real_funcname+'>'
-def memoize_classdict(modelklass, new_klassdict, cacheable, resetting):
+def memoize_classdict(klass, modelklass, new_klassdict, cacheable, resetting):
new_klassdict.update(modelklass.__dict__)
new_klassdict['_MeMoIZeR_converted'] = 1
for name,code in cacheable.items():
+ Code_to_Name[(klass.__name__, id(code))] = klass.__name__ + '.' + name
+ eval_dict = {
+ 'methcode' : code,
+ 'methcached' : {},
+ }
+ eval_dict.update(MCG_dict)
if code.func_code.co_argcount == 1 and \
not code.func_code.co_flags & 0xC:
- newmethod = eval(
+ compiled = \
compile("\n"*1 +
"lambda self: MCGS(methcode, methcached, self)",
whoami('cache_get_self', name),
- "eval"),
- {'methcode':code, 'methcached':{},
- 'MCGS':Memoizer_cache_get_self},
- {})
+ "eval")
elif code.func_code.co_argcount == 2 and \
not code.func_code.co_flags & 0xC:
- newmethod = eval(
+ compiled = \
compile("\n"*2 +
"lambda self, arg: MCGO(methcode, methcached, self, arg)",
whoami('cache_get_one', name),
- "eval"),
- {'methcode':code, 'methcached':{},
- 'MCGO':Memoizer_cache_get_one},
- {})
+ "eval")
else:
- newmethod = eval(
+ compiled = \
compile("\n"*3 +
"lambda *args, **kw: MCG(methcode, methcached, args, kw)",
whoami('cache_get', name),
- "eval"),
- {'methcode':code, 'methcached':{},
- 'MCG':Memoizer_cache_get}, {})
+ "eval")
+ newmethod = eval(compiled, eval_dict, {})
new_klassdict[name] = newmethod
for name,code in resetting.items():
cls_dict['_MeMoIZeR_cmp'] = C
else:
modelklass = _Memoizer_Simple
- klassdict = memoize_classdict(modelklass, cls_dict, D, R)
+ klassdict = memoize_classdict(cls, modelklass, cls_dict, D, R)
init = klassdict.get('__init__', None)
if not init:
print_explanations = 0
print_includes = 0
print_objects = 0
+print_memoizer = 0
print_stacktrace = 0
print_stree = 0
print_time = 0
def _set_globals(options):
global repositories, keep_going_on_error, ignore_errors
global print_count, print_dtree
- global print_explanations, print_includes
+ global print_explanations, print_includes, print_memoizer
global print_objects, print_stacktrace, print_stree
global print_time, print_tree
global memory_outf, memory_stats
repositories.extend(options.repository)
keep_going_on_error = options.keep_going
try:
- if options.debug:
- if options.debug == "count":
- print_count = 1
- elif options.debug == "dtree":
- print_dtree = 1
- elif options.debug == "explain":
- print_explanations = 1
- elif options.debug == "findlibs":
- SCons.Scanner.Prog.print_find_libs = "findlibs"
- elif options.debug == "includes":
- print_includes = 1
- elif options.debug == "memory":
- memory_stats = []
- memory_outf = sys.stdout
- elif options.debug == "objects":
- print_objects = 1
- elif options.debug == "presub":
- SCons.Action.print_actions_presub = 1
- elif options.debug == "stacktrace":
- print_stacktrace = 1
- elif options.debug == "stree":
- print_stree = 1
- elif options.debug == "time":
- print_time = 1
- elif options.debug == "tree":
- print_tree = 1
+ debug_values = options.debug
+ if debug_values is None:
+ debug_values = []
except AttributeError:
pass
+ else:
+ if "count" in debug_values:
+ print_count = 1
+ if "dtree" in debug_values:
+ print_dtree = 1
+ if "explain" in debug_values:
+ print_explanations = 1
+ if "findlibs" in debug_values:
+ SCons.Scanner.Prog.print_find_libs = "findlibs"
+ if "includes" in debug_values:
+ print_includes = 1
+ if "memoizer" in debug_values:
+ SCons.Memoize.EnableCounting()
+ print_memoizer = 1
+ if "memory" in debug_values:
+ memory_stats = []
+ memory_outf = sys.stdout
+ if "objects" in debug_values:
+ print_objects = 1
+ if "presub" in debug_values:
+ SCons.Action.print_actions_presub = 1
+ if "stacktrace" in debug_values:
+ print_stacktrace = 1
+ if "stree" in debug_values:
+ print_stree = 1
+ if "time" in debug_values:
+ print_time = 1
+ if "tree" in debug_values:
+ print_tree = 1
ignore_errors = options.ignore_errors
def _create_path(plist):
"build all Default() targets.")
debug_options = ["count", "dtree", "explain", "findlibs",
- "includes", "memory", "objects",
+ "includes", "memoizer", "memory", "objects",
"pdb", "presub", "stacktrace", "stree",
"time", "tree"]
def opt_debug(option, opt, value, parser, debug_options=debug_options):
if value in debug_options:
- parser.values.debug = value
+ try:
+ if parser.values.debug is None:
+ parser.values.debug = []
+ except AttributeError:
+ parser.values.debug = []
+ parser.values.debug.append(value)
else:
raise OptionValueError("Warning: %s is not a valid debug type" % value)
self.add_option('--debug', action="callback", type="string",
SCons.Debug.listLoggedInstances('*')
#SCons.Debug.dumpLoggedInstances('*')
+ if print_memoizer:
+ print "Memoizer (memory cache) hits and misses:"
+ SCons.Memoize.Dump()
+
def _exec_main():
all_args = sys.argv[1:]
try:
parser = OptParser()
global options
options, args = parser.parse_args(all_args)
- if options.debug == "pdb":
+ if type(options.debug) == type([]) and "pdb" in options.debug:
import pdb
pdb.Pdb().runcall(_main, args, parser)
else:
--- /dev/null
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+"""
+Test calling the --debug=memoizer option.
+"""
+
+import string
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+test.write('SConstruct', """
+def cat(target, source, env):
+ open(str(target[0]), 'wb').write(open(str(source[0]), 'rb').read())
+env = Environment(BUILDERS={'Cat':Builder(action=Action(cat))})
+env.Cat('file.out', 'file.in')
+""")
+
+test.write('file.in', "file.in\n")
+
+test.run(arguments = '--debug=memoizer')
+
+expect = "Memoizer (memory cache) hits and misses"
+test.fail_test(string.find(test.stdout(), expect) == -1)
+
+test.must_match('file.out', "file.in\n")
+
+test.pass_test()
projects / scons.git / commitdiff