--- /dev/null
+#!/usr/bin/env python
+#
+# A script to compare the --debug=memoizer output found int
+# two different files.
+
+import sys,string
+
+def memoize_output(fname):
+ mout = {}
+ lines=filter(lambda words:
+ len(words) == 5 and
+ words[1] == 'hits' and words[3] == 'misses',
+ map(string.split, open(fname,'r').readlines()))
+ for line in lines:
+ mout[line[-1]] = ( int(line[0]), int(line[2]) )
+ return mout
+
+
+def memoize_cmp(filea, fileb):
+ ma = memoize_output(filea)
+ mb = memoize_output(fileb)
+
+ print 'All output: %s / %s [delta]'%(filea, fileb)
+ print '----------HITS---------- ---------MISSES---------'
+ cfmt='%7d/%-7d [%d]'
+ ma_o = []
+ mb_o = []
+ mab = []
+ for k in ma.keys():
+ if k in mb.keys():
+ if k not in mab:
+ mab.append(k)
+ else:
+ ma_o.append(k)
+ for k in mb.keys():
+ if k in ma.keys():
+ if k not in mab:
+ mab.append(k)
+ else:
+ mb_o.append(k)
+
+ mab.sort()
+ ma_o.sort()
+ mb_o.sort()
+
+ for k in mab:
+ hits = cfmt%(ma[k][0], mb[k][0], mb[k][0]-ma[k][0])
+ miss = cfmt%(ma[k][1], mb[k][1], mb[k][1]-ma[k][1])
+ print '%-24s %-24s %s'%(hits, miss, k)
+
+ for k in ma_o:
+ hits = '%7d/ --'%(ma[k][0])
+ miss = '%7d/ --'%(ma[k][1])
+ print '%-24s %-24s %s'%(hits, miss, k)
+
+ for k in mb_o:
+ hits = ' -- /%-7d'%(mb[k][0])
+ miss = ' -- /%-7d'%(mb[k][1])
+ print '%-24s %-24s %s'%(hits, miss, k)
+
+ print '-'*(24+24+1+20)
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 3:
+ print """Usage: %s file1 file2
+
+Compares --debug=memomize output from file1 against file2."""%sys.argv[0]
+ sys.exit(1)
+
+ memoize_cmp(sys.argv[1], sys.argv[2])
+ sys.exit(0)
+
return 'TestScannerkey'
def instance(self, env):
return self
+ def select(self, node):
+ return self
name = 'TestScanner'
def __str__(self):
return self.name
name = 'EnvTestScanner'
def __str__(self):
return self.name
- def select(self, scanner):
+ def select(self, node):
return self
def path(self, env, dir=None):
return ()
lvars['__env__'] = self
return SCons.Util.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv)
- def subst_path(self, path, target=None):
+ def subst_path(self, path, target=None, source=None):
"""Substitute a path list, turning EntryProxies into Nodes
and leaving Nodes (and other objects) as-is."""
r = []
for p in path:
if SCons.Util.is_String(p):
- p = self.subst(p, target=target, conv=s)
+ p = self.subst(p, target=target, source=source, conv=s)
if SCons.Util.is_List(p):
if len(p) == 1:
p = p[0]
r = env.subst_path(['$FOO', 'xxx', '$BAR'])
assert r == ['foo', 'xxx', 'bar'], r
- r = env.subst_path(['$FOO', '$TARGET', '$BAR'])
- assert r == ['foo', '', 'bar'], r
+ r = env.subst_path(['$FOO', '$TARGET', '$SOURCE', '$BAR'])
+ assert r == ['foo', '', '', 'bar'], r
- r = env.subst_path(['$FOO', '$TARGET', '$BAR'], target=MyNode('yyy'))
- assert map(str, r) == ['foo', 'yyy', 'bar'], r
+ r = env.subst_path(['$FOO', '$TARGET', '$BAR'], target=MyNode('ttt'))
+ assert map(str, r) == ['foo', 'ttt', 'bar'], r
+
+ r = env.subst_path(['$FOO', '$SOURCE', '$BAR'], source=MyNode('sss'))
+ assert map(str, r) == ['foo', 'sss', 'bar'], r
n = MyObj()
return build_env
+ def get_build_scanner_path(self, scanner):
+ """
+ __cacheable__
+ """
+ env = self.get_build_env()
+ try:
+ cwd = self.targets[0].cwd
+ except (IndexError, AttributeError):
+ cwd = None
+ return scanner.path(env, cwd, self.targets, self.sources)
+
def do_nothing(self, target, errfunc, kw):
pass
self.action = MyAction()
class MyNode:
- def __init__(self, pre, post):
+ def __init__(self, name=None, pre=[], post=[]):
+ self.name = name
self.pre_actions = pre
self.post_actions = post
+ def __str__(self):
+ return self.name
def build(self, errfunc=None):
executor = SCons.Executor.Executor(MyAction(self.pre_actions +
[self.builder.action] +
[self],
['s1', 's2'])
apply(executor, (self, errfunc), {})
+
+class MyScanner:
+ def path(self, env, dir, target, source):
+ target = map(str, target)
+ source = map(str, source)
+ return "scanner: %s, %s, %s, %s" % (env['SCANNERVAL'], dir, target, source)
class ExecutorTestCase(unittest.TestCase):
assert be['O'] == 'ob3', be['O']
assert be['Y'] == 'yyy', be['Y']
+ def test_get_build_scanner_path(self):
+ """Test fetching the path for the specified scanner."""
+ t = MyNode('t')
+ t.cwd = 'here'
+ x = SCons.Executor.Executor(MyAction(),
+ MyEnvironment(SCANNERVAL='sss'),
+ [],
+ [t],
+ ['s1', 's2'])
+
+ s = MyScanner()
+
+ p = x.get_build_scanner_path(s)
+ assert p == "scanner: sss, here, ['t'], ['s1', 's2']", p
+
def test__call__(self):
"""Test calling an Executor"""
result = []
a = MyAction([action1, action2])
b = MyBuilder(env, {})
b.action = a
- n = MyNode([pre], [post])
+ n = MyNode('n', [pre], [post])
n.builder = b
n.build()
assert result == ['pre', 'action1', 'action2', 'post'], result
errfunc(1)
return 1
- n = MyNode([pre_err], [post])
+ n = MyNode('n', [pre_err], [post])
n.builder = b
n.build()
assert result == ['pre_err', 'action1', 'action2', 'post'], result
a = MyAction([action1])
x = SCons.Executor.Executor(a, env, [], ['t1', 't2'], ['s1', 's2'])
- x(MyNode([], []), None)
+ x(MyNode('', [], []), None)
assert result == ['action1'], result
s = str(x)
assert s[:10] == 'GENSTRING ', s
x.nullify()
assert result == [], result
- x(MyNode([], []), None)
+ x(MyNode('', [], []), None)
assert result == [], result
s = str(x)
assert s == '', s
self.clear()
return File.rfile(self)
- def get_found_includes(self, env, scanner, target):
+ def get_found_includes(self, env, scanner, path):
"""If we're looking for included files, it's because this Entry
is really supposed to be a File itself."""
node = self.rfile()
- return node.get_found_includes(env, scanner, target)
+ return node.get_found_includes(env, scanner, path)
def scanner_key(self):
return self.get_suffix()
except AttributeError:
return None
- def get_found_includes(self, env, scanner, target):
+ def get_found_includes(self, env, scanner, path):
"""Return the included implicit dependencies in this file.
- Cache results so we only scan the file once regardless of
- how many times this information is requested."""
+ Cache results so we only scan the file once per path
+ regardless of how many times this information is requested.
+ __cacheable__"""
if not scanner:
return []
-
- try:
- path = target.scanner_paths[scanner]
- except AttributeError:
- # The target had no scanner_paths attribute, which means
- # it's an Alias or some other node that's not actually a
- # file. In that case, back off and use the path for this
- # node itself.
- try:
- path = self.scanner_paths[scanner]
- except KeyError:
- path = scanner.path(env, self.cwd, target)
- self.scanner_paths[scanner] = path
- except KeyError:
- path = scanner.path(env, target.cwd, target)
- target.scanner_paths[scanner] = path
-
return scanner(self, env, path)
def _createDir(self):
scanner_count = scanner_count + 1
self.hash = scanner_count
self.node = node
- def path(self, env, dir, target=None):
+ def path(self, env, dir, target=None, source=None):
return ()
def __call__(self, node, env, path):
return [self.node]
def __call__(self, node):
self.called = 1
return node.found_includes
+ def path(self, env, dir, target=None, source=None):
+ return ()
def select(self, node):
return self
assert str(act.built_target[0]) == "xxx", str(act.built_target[0])
assert act.built_source == ["yyy", "zzz"], act.built_source
+ def test_get_build_scanner_path(self):
+ """Test the get_build_scanner_path() method"""
+ n = SCons.Node.Node()
+ class MyExecutor:
+ def get_build_scanner_path(self, scanner):
+ return 'executor would call %s' % scanner
+ x = MyExecutor()
+ n.set_executor(x)
+ p = n.get_build_scanner_path('fake_scanner')
+ assert p == "executor would call fake_scanner", p
+
def test_get_executor(self):
- """Test the reset_executor() method"""
+ """Test the get_executor() method"""
n = SCons.Node.Node()
try:
assert x.env == 'env2', x.env
def test_set_executor(self):
- """Test the reset_executor() method"""
+ """Test the set_executor() method"""
n = SCons.Node.Node()
n.set_executor(1)
assert n.executor == 1, n.executor
__cacheable__"""
return self.get_executor().get_build_env()
+ def get_build_scanner_path(self, scanner):
+ """Fetch the appropriate Environment to build this node.
+ __cacheable__"""
+ return self.get_executor().get_build_scanner_path(scanner)
+
def set_executor(self, executor):
"""Set the action executor for this node."""
self.executor = executor
This allows an internal Builder created by SCons to be marked
non-explicit, so that it can be overridden by an explicit
builder that the user supplies (the canonical example being
- directories).
- __cacheable__"""
+ directories)."""
return self.has_builder() and self.builder.is_explicit
def get_builder(self, default_builder=None):
"""
return [], None
- def get_found_includes(self, env, scanner, target):
+ def get_found_includes(self, env, scanner, path):
"""Return the scanned include lines (implicit dependencies)
found in this node.
"""
return []
- def get_implicit_deps(self, env, scanner, target):
+ def get_implicit_deps(self, env, scanner, path):
"""Return a list of implicit dependencies for this node.
This method exists to handle recursive invocation of the scanner
while nodes:
n = nodes.pop(0)
d = filter(lambda x, seen=seen: not seen.has_key(x),
- n.get_found_includes(env, scanner, target))
+ n.get_found_includes(env, scanner, path))
if d:
deps.extend(d)
for n in d:
This function may be called very often; it attempts to cache
the scanner found to improve performance.
- __cacheable__
"""
# Called from scan() for each child (node) of this node
# (self). The scan() may be called multiple times, so this
if not self.has_builder():
return None
+ scanner = None
try:
scanner = self.builder.source_scanner
- if scanner:
- return scanner
except AttributeError:
pass
# based on the node's scanner key (usually the file
# extension).
- scanner = self.get_build_env().get_scanner(node.scanner_key())
+ if not scanner:
+ scanner = self.get_build_env().get_scanner(node.scanner_key())
+ if scanner:
+ scanner = scanner.select(node)
return scanner
def scan(self):
self._children_reset()
self.del_binfo()
+ # Potential optimization for the N^2 problem if we can tie
+ # scanning to the Executor in some way so that we can scan
+ # source files onces and then spread the implicit dependencies
+ # to all of the targets at once.
+ #kids = self.children(scan=0)
+ #for child in filter(lambda n: n.implicit is None, kids):
for child in self.children(scan=0):
scanner = self.get_source_scanner(child)
if scanner:
- deps = child.get_implicit_deps(build_env, scanner, self)
+ path = self.get_build_scanner_path(scanner)
+ deps = child.get_implicit_deps(build_env, scanner, path)
self._add_child(self.implicit, self.implicit_dict, deps)
# scan this node itself for implicit dependencies
scanner = self.builder.target_scanner
if scanner:
- deps = self.get_implicit_deps(build_env, scanner, self)
+ path = self.get_build_scanner_path(scanner)
+ deps = self.get_implicit_deps(build_env, scanner, path)
self._add_child(self.implicit, self.implicit_dict, deps)
# XXX See note above re: --implicit-cache.
env = self.get_build_env()
for s in self.sources:
scanner = self.get_source_scanner(s)
- def f(node, env=env, scanner=scanner, target=self):
- return node.get_found_includes(env, scanner, target)
+ path = self.get_build_scanner_path(scanner)
+ def f(node, env=env, scanner=scanner, path=path):
+ return node.get_found_includes(env, scanner, path)
return SCons.Util.render_tree(s, f, 1)
else:
return None
return [self.data[strSubst[1:]]]
return [[strSubst]]
- def subst_path(self, path, target=None):
+ def subst_path(self, path, target=None, source=None):
if type(path) != type([]):
path = [path]
return map(self.subst, path)
return self[arg[1:]]
return arg
- def subst_path(self, path, target=None):
+ def subst_path(self, path, target=None, source=None):
if type(path) != type([]):
path = [path]
return map(self.subst, path)
def subst(self, arg):
return arg
- def subst_path(self, path, target=None):
+ def subst_path(self, path, target=None, source=None):
if type(path) != type([]):
path = [path]
return map(self.subst, path)
return ''
return s
- def subst_path(self, path, target=None):
+ def subst_path(self, path, target=None, source=None):
if type(path) != type([]):
path = [path]
return map(self.subst, path)
if strSubst[0] == '$':
return [self.data[strSubst[1:]]]
return [[strSubst]]
- def subst_path(self, path, target=None):
+ def subst_path(self, path, target=None, source=None):
if type(path) != type([]):
path = [path]
return map(self.subst, path)
else:
self.failIf(hasattr(self, "arg"), "an argument was given when it shouldn't have been")
+ def test_path(self):
+ """Test the Scanner.Base path() method"""
+ def pf(env, cwd, target, source, argument=None):
+ return "pf: %s %s %s %s %s" % \
+ (env.VARIABLE, cwd, target[0], source[0], argument)
+
+ env = DummyEnvironment()
+ env.VARIABLE = 'v1'
+ target = DummyNode('target')
+ source = DummyNode('source')
+
+ s = SCons.Scanner.Base(self.func, path_function=pf)
+ p = s.path(env, 'here', [target], [source])
+ assert p == "pf: v1 here target source None", p
+
+ s = SCons.Scanner.Base(self.func, path_function=pf, argument="xyz")
+ p = s.path(env, 'here', [target], [source])
+ assert p == "pf: v1 here target source xyz", p
+
def test_positional(self):
"""Test the Scanner.Base class using positional arguments"""
s = SCons.Scanner.Base(self.func, "Pos")
return apply(Base, (function,) + args, kw)
-class Binder:
- __metaclass__ = SCons.Memoize.Memoized_Metaclass
+class _Binder:
def __init__(self, bindval):
self._val = bindval
def __call__(self):
def __str__(self):
return str(self._val)
#debug: return 'B<%s>'%str(self._val)
-
+
+BinderDict = {}
+
+def Binder(path):
+ try:
+ return BinderDict[path]
+ except KeyError:
+ b = _Binder(path)
+ BinderDict[path] = b
+ return b
+
+
class FindPathDirs:
"""A class to bind a specific *PATH variable name and the fs object
to a function that will return all of the *path directories."""
- __metaclass__ = SCons.Memoize.Memoized_Metaclass
def __init__(self, variable, fs):
self.variable = variable
self.fs = fs
- def __call__(self, env, dir, target=None, argument=None):
- "__cacheable__"
+ def __call__(self, env, dir, target=None, source=None, argument=None):
+ # The goal is that we've made caching this unnecessary
+ # because the caching takes place at higher layers.
try:
path = env[self.variable]
except KeyError:
return ()
- path = env.subst_path(path, target=target)
+ path = env.subst_path(path, target=target, source=source)
path_tuple = tuple(self.fs.Rsearchall(path,
must_exist = 0, #kwq!
clazz = SCons.Node.FS.Dir,
cwd = dir))
return Binder(path_tuple)
-if not SCons.Memoize.has_metaclass:
- _FPD_Base = FindPathDirs
- class FindPathDirs(SCons.Memoize.Memoizer, _FPD_Base):
- "Cache-backed version of FindPathDirs"
- def __init__(self, *args, **kw):
- apply(_FPD_Base.__init__, (self,)+args, kw)
- SCons.Memoize.Memoizer.__init__(self)
- _BinderBase = Binder
- class Binder(SCons.Memoize.Memoizer, _BinderBase):
- "Cache-backed version of Binder"
- def __init__(self, *args, **kw):
- apply(_BinderBase.__init__, (self,)+args, kw)
- SCons.Memoize.Memoizer.__init__(self)
-
-
class Base:
"""
The base class for dependency scanners. This implements
self.scan_check = scan_check
self.recursive = recursive
- def path(self, env, dir=None, target=None):
- "__cacheable__"
+ def path(self, env, dir=None, target=None, source=None):
if not self.path_function:
return ()
if not self.argument is _null:
- return self.path_function(env, dir, target, self.argument)
+ return self.path_function(env, dir, target, source, self.argument)
else:
- return self.path_function(env, dir, target)
+ return self.path_function(env, dir, target, source)
def __call__(self, node, env, path = ()):
"""
# once before they're generated and once after. That's the
# next thing to fix here.
-# Note KWQ 01 Nov 2004: used to check for a one for all counts below;
-# this was indirectly a test that the caching method in use at the
-# time was working. With the introduction of Memoize-based caching,
-# the caching is performed right at the interface level, so the test
-# here cannot be run the same way; ergo real counts are used below.
-
test.must_match("MyCScan.out", """\
libg_1.c: 1
libg_2.c: 1
libg_3.c: 1
-libg_gx.h: 3
+libg_gx.h: 1
libg_gy.h: 1
libg_gz.h: 1
-libg_w.h: 3
+libg_w.h: 1
""")
test.pass_test()
projects / scons.git / commitdiff