From: stevenknight Date: Tue, 27 Dec 2005 22:24:55 +0000 (+0000) Subject: Handle scanning of the in-memory entries for a Dir with a scanner, not a hard-coded... X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=3f903d57991874b05687f042258684f56964700e;p=scons.git Handle scanning of the in-memory entries for a Dir with a scanner, not a hard-coded Python method. git-svn-id: http://scons.tigris.org/svn/scons/trunk@1407 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index 6b9b1150..cb628b82 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -109,6 +109,7 @@ ProgScan = SCons.Tool.ProgramScanner # should go. Leave it here for now. import SCons.Scanner.Dir DirScanner = SCons.Scanner.Dir.DirScanner() +DirEntryScanner = SCons.Scanner.Dir.DirEntryScanner() # Actions for common languages. CAction = SCons.Action.Action("$CCCOM", "$CCCOMSTR") diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index ad671521..d95fd813 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -182,7 +182,8 @@ class Executor: self.scan(scanner, self.targets) def scan_sources(self, scanner): - self.scan(scanner, self.sources) + if self.sources: + self.scan(scanner, self.sources) def scan(self, scanner, node_list): """Scan a list of this Executor's files (targets or sources) for @@ -191,16 +192,16 @@ class Executor: each individual target, which is a hell of a lot more efficient. """ env = self.get_build_env() - select_specific_scanner = lambda t: (t[0], t[0].select_scanner(t[1])) + select_specific_scanner = lambda t: (t[0], t[1].select(t[0])) remove_null_scanners = lambda t: not t[1] is None add_scanner_path = lambda t, s=self: \ (t[0], t[1], s.get_build_scanner_path(t[1])) if scanner: - scanner_list = map(lambda src, s=scanner: (src, s), node_list) + scanner_list = map(lambda n, s=scanner: (n, s), node_list) else: kw = self.get_kw() - get_initial_scanners = lambda src, e=env, kw=kw: \ - (src, src.get_scanner(e, kw)) + get_initial_scanners = lambda n, e=env, kw=kw: \ + (n, n.get_env_scanner(e, kw)) scanner_list = map(get_initial_scanners, node_list) scanner_list = filter(remove_null_scanners, scanner_list) diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py index a5ac9a65..44086d7c 100644 --- a/src/engine/SCons/ExecutorTests.py +++ b/src/engine/SCons/ExecutorTests.py @@ -78,7 +78,7 @@ class MyNode: [self], ['s1', 's2']) apply(executor, (self, errfunc), {}) - def get_scanner(self, env, kw): + def get_env_scanner(self, env, kw): return MyScanner('dep-') def get_implicit_deps(self, env, scanner, path): return [scanner.prefix + str(self)] @@ -88,8 +88,6 @@ class MyNode: return self.missing_val def calc_signature(self, calc): return 'cs-'+calc+'-'+self.name - def select_scanner(self, scanner): - return scanner.select(self) class MyScanner: def __init__(self, prefix): diff --git a/src/engine/SCons/Node/Alias.py b/src/engine/SCons/Node/Alias.py index bc7e53b4..e023ab78 100644 --- a/src/engine/SCons/Node/Alias.py +++ b/src/engine/SCons/Node/Alias.py @@ -76,7 +76,7 @@ class Alias(SCons.Node.Node): def get_contents(self): """The contents of an alias is the concatenation of all the contents of its sources""" - contents = map(lambda n: n.get_contents(), self.children(None)) + contents = map(lambda n: n.get_contents(), self.children()) return string.join(contents, '') def sconsign(self): diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 59ad7075..69936d22 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -198,12 +198,14 @@ def get_MkdirBuilder(): global MkdirBuilder if MkdirBuilder is None: import SCons.Builder + import SCons.Defaults # "env" will get filled in by Executor.get_build_env() # calling SCons.Defaults.DefaultEnvironment() when necessary. MkdirBuilder = SCons.Builder.Builder(action = Mkdir, env = None, explain = None, is_explicit = None, + target_scanner = SCons.Defaults.DirEntryScanner, name = "MkdirBuilder") return MkdirBuilder @@ -1287,21 +1289,11 @@ class Dir(Base): return string.join(path_elems, os.sep) - def scan(self): - if not self.implicit is None: - return - self.implicit = [] - self.implicit_dict = {} - self._children_reset() + def get_env_scanner(self, env, kw={}): + return SCons.Defaults.DirEntryScanner - dont_scan = lambda k: k not in ['.', '..', '.sconsign'] - deps = filter(dont_scan, self.entries.keys()) - # keys() is going to give back the entries in an internal, - # unsorted order. Sort 'em so the order is deterministic. - deps.sort() - entries = map(lambda n, e=self.entries: e[n], deps) - - self._add_child(self.implicit, self.implicit_dict, entries) + def get_target_scanner(self): + return SCons.Defaults.DirEntryScanner def get_found_includes(self, env, scanner, path): """Return the included implicit dependencies in this file. @@ -1310,7 +1302,7 @@ class Dir(Base): __cacheable__""" if not scanner: return [] - # Clear cached info for this Node. If we already visited this + # Clear cached info for this Dir. If we already visited this # directory on our walk down the tree (because we didn't know at # that point it was being used as the source for another Node) # then we may have calculated build signature before realizing diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index ba6cea83..8c95f87e 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -966,22 +966,6 @@ class FSTestCase(_tempdirTestCase): dir = fs.Dir(drive) assert str(dir) == drive + os.sep, str(dir) - # Test Dir.scan() - dir = fs.Dir('ddd') - fs.File(string.join(['ddd', 'f1'], sep)) - fs.File(string.join(['ddd', 'f2'], sep)) - fs.File(string.join(['ddd', 'f3'], sep)) - fs.Dir(string.join(['ddd', 'd1'], sep)) - fs.Dir(string.join(['ddd', 'd1', 'f4'], sep)) - fs.Dir(string.join(['ddd', 'd1', 'f5'], sep)) - dir.scan() - kids = map(lambda x: x.path, dir.children(None)) - kids.sort() - assert kids == [os.path.join('ddd', 'd1'), - os.path.join('ddd', 'f1'), - os.path.join('ddd', 'f2'), - os.path.join('ddd', 'f3')], kids - # Test for a bug in 0.04 that did not like looking up # dirs with a trailing slash on Win32. d=fs.Dir('./') @@ -1366,7 +1350,7 @@ class FSTestCase(_tempdirTestCase): assert str(t) == 'pre-z-suf', str(t) def test_same_name(self): - """Test that a local same-named file isn't found for # Dir lookup""" + """Test that a local same-named file isn't found for a Dir lookup""" test = self.test fs = self.fs @@ -1485,6 +1469,42 @@ class DirTestCase(_tempdirTestCase): assert a[0] == 'pre', a assert a[2] == 'post', a + def test_get_env_scanner(self): + """Test the Dir.get_env_scanner() method + """ + import SCons.Defaults + d = self.fs.Dir('ddd') + s = d.get_env_scanner(Environment()) + assert s is SCons.Defaults.DirEntryScanner, s + + def test_get_target_scanner(self): + """Test the Dir.get_target_scanner() method + """ + import SCons.Defaults + d = self.fs.Dir('ddd') + s = d.get_target_scanner() + assert s is SCons.Defaults.DirEntryScanner, s + + def test_scan(self): + """Test scanning a directory for in-memory entries + """ + fs = self.fs + + dir = fs.Dir('ddd') + fs.File(os.path.join('ddd', 'f1')) + fs.File(os.path.join('ddd', 'f2')) + fs.File(os.path.join('ddd', 'f3')) + fs.Dir(os.path.join('ddd', 'd1')) + fs.Dir(os.path.join('ddd', 'd1', 'f4')) + fs.Dir(os.path.join('ddd', 'd1', 'f5')) + dir.scan() + kids = map(lambda x: x.path, dir.children(None)) + kids.sort() + assert kids == [os.path.join('ddd', 'd1'), + os.path.join('ddd', 'f1'), + os.path.join('ddd', 'f2'), + os.path.join('ddd', 'f3')], kids + def test_entry_exists_on_disk(self): """Test the Dir.entry_exists_on_disk() method """ diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 3e2fd5c2..1cd5201e 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -930,17 +930,28 @@ class NodeTestCase(unittest.TestCase): deps = node.get_implicit_deps(env, s, target) assert deps == [d1, d2], map(str, deps) - def test_get_scanner(self): + def test_get_env_scanner(self): """Test fetching the environment scanner for a Node """ node = SCons.Node.Node() scanner = Scanner() env = Environment(SCANNERS = [scanner]) - s = node.get_scanner(env) + s = node.get_env_scanner(env) assert s == scanner, s - s = node.get_scanner(env, {'X':1}) + s = node.get_env_scanner(env, {'X':1}) assert s == scanner, s + def test_get_target_scanner(self): + """Test fetching the target scanner for a Node + """ + s = Scanner() + b = Builder() + b.target_scanner = s + n = SCons.Node.Node() + n.builder = b + x = n.get_target_scanner() + assert x is s, x + def test_get_source_scanner(self): """Test fetching the source scanner for a Node """ @@ -1044,12 +1055,6 @@ class NodeTestCase(unittest.TestCase): """Test that a scanner_key() method exists""" assert SCons.Node.Node().scanner_key() == None - def test_select_scanner(self): - """Test the base select_scanner() method returns its scanner""" - scanner = Scanner() - s = SCons.Node.Node().select_scanner(scanner) - assert scanner is s, s - def test_children(self): """Test fetching the non-ignored "children" of a Node. """ diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 7a29f95c..250c7140 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -474,9 +474,12 @@ class Node: return deps - def get_scanner(self, env, kw={}): + def get_env_scanner(self, env, kw={}): return env.get_scanner(self.scanner_key()) + def get_target_scanner(self): + return self.builder.target_scanner + def get_source_scanner(self, node): """Fetch the source scanner for the specified node @@ -498,7 +501,7 @@ class Node: # The builder didn't have an explicit scanner, so go look up # a scanner from env['SCANNERS'] based on the node's scanner # key (usually the file extension). - scanner = self.get_scanner(self.get_build_env()) + scanner = self.get_env_scanner(self.get_build_env()) if scanner: scanner = scanner.select(node) return scanner @@ -563,7 +566,7 @@ class Node: # If there's a target scanner, have the executor scan the target # node itself and associated targets that might be built. - scanner = self.builder.target_scanner + scanner = self.get_target_scanner() if scanner: executor.scan_targets(scanner) diff --git a/src/engine/SCons/Scanner/Dir.py b/src/engine/SCons/Scanner/Dir.py index 3da1661d..fb23d1b1 100644 --- a/src/engine/SCons/Scanner/Dir.py +++ b/src/engine/SCons/Scanner/Dir.py @@ -43,15 +43,31 @@ def DirEntryScanner(**kw): """Return a prototype Scanner instance for "scanning" directory Nodes for their in-memory entries""" kw['node_factory'] = SCons.Node.FS.Entry - kw['recursive'] = only_dirs + kw['recursive'] = None return apply(SCons.Scanner.Base, (scan_in_memory, "DirEntryScanner"), kw) -skip_entry = { - '.' : 1, - '..' : 1, - '.sconsign' : 1, - '.sconsign.dblite' : 1, -} +skip_entry = {} + +skip_entry_list = [ + '.', + '..', + '.sconsign', + # Used by the native dblite.py module. + '.sconsign.dblite', + # Used by dbm and dumbdbm. + '.sconsign.dir', + # Used by dbm. + '.sconsign.pag', + # Used by dumbdbm. + '.sconsign.dat', + '.sconsign.bak', + # Used by some dbm emulations using Berkeley DB. + '.sconsign.db', +] + +for skip in skip_entry_list: + skip_entry[skip] = 1 + skip_entry[SCons.Node.FS._my_normcase(skip)] = 1 do_not_scan = lambda k: not skip_entry.has_key(k) diff --git a/src/engine/SCons/Scanner/DirTests.py b/src/engine/SCons/Scanner/DirTests.py index e4e59a39..0dde95e3 100644 --- a/src/engine/SCons/Scanner/DirTests.py +++ b/src/engine/SCons/Scanner/DirTests.py @@ -62,11 +62,20 @@ class DirScannerTestBase(unittest.TestCase): self.test.write(['dir', 'f1'], "dir/f1\n") self.test.write(['dir', 'f2'], "dir/f2\n") self.test.write(['dir', '.sconsign'], "dir/.sconsign\n") + self.test.write(['dir', '.sconsign.bak'], "dir/.sconsign.bak\n") + self.test.write(['dir', '.sconsign.dat'], "dir/.sconsign.dat\n") + self.test.write(['dir', '.sconsign.db'], "dir/.sconsign.db\n") self.test.write(['dir', '.sconsign.dblite'], "dir/.sconsign.dblite\n") + self.test.write(['dir', '.sconsign.dir'], "dir/.sconsign.dir\n") + self.test.write(['dir', '.sconsign.pag'], "dir/.sconsign.pag\n") self.test.write(['dir', 'sub', 'f3'], "dir/sub/f3\n") self.test.write(['dir', 'sub', 'f4'], "dir/sub/f4\n") self.test.write(['dir', 'sub', '.sconsign'], "dir/.sconsign\n") + self.test.write(['dir', 'sub', '.sconsign.bak'], "dir/.sconsign.bak\n") + self.test.write(['dir', 'sub', '.sconsign.dat'], "dir/.sconsign.dat\n") self.test.write(['dir', 'sub', '.sconsign.dblite'], "dir/.sconsign.dblite\n") + self.test.write(['dir', 'sub', '.sconsign.dir'], "dir/.sconsign.dir\n") + self.test.write(['dir', 'sub', '.sconsign.pag'], "dir/.sconsign.pag\n") class DirScannerTestCase1(DirScannerTestBase): def runTest(self): @@ -74,13 +83,22 @@ class DirScannerTestCase1(DirScannerTestBase): s = SCons.Scanner.Dir.DirScanner() + expect = [ + os.path.join('dir', 'f1'), + os.path.join('dir', 'f2'), + os.path.join('dir', 'sub'), + ] deps = s(env.Dir('dir'), env, ()) sss = map(str, deps) - assert sss == ['dir/f1', 'dir/f2', 'dir/sub'], sss + assert sss == expect, sss + expect = [ + os.path.join('dir', 'sub', 'f3'), + os.path.join('dir', 'sub', 'f4'), + ] deps = s(env.Dir('dir/sub'), env, ()) sss = map(str, deps) - assert sss == ['dir/sub/f3', 'dir/sub/f4'], sss + assert sss == expect, sss class DirScannerTestCase2(DirScannerTestBase): def runTest(self): diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index cc7bf134..16e790ee 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -1236,7 +1236,7 @@ def _main(args, parser): if options.taskmastertrace_file == '-': tmtrace = sys.stdout elif options.taskmastertrace_file: - tmtrace = open(options.taskmastertrace_file, 'w') + tmtrace = open(options.taskmastertrace_file, 'wb') else: tmtrace = None taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace) diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index cbb3f89a..5d689898 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -320,29 +320,28 @@ class Taskmaster: else: S = None - if T: T.write('Taskmaster: %s' % repr(str(node))) + if T: T.write('Taskmaster: %s:' % repr(str(node))) # Skip this node if it has already been handled: if not state in [ SCons.Node.no_state, SCons.Node.stack ]: if S: S.already_handled = S.already_handled + 1 - if T: T.write(': already handled\n') + if T: T.write(' already handled\n') continue # Mark this node as being on the execution stack: node.set_state(SCons.Node.stack) try: - childinfo = map(lambda N: (N.get_state(), - N.is_derived() or N.is_pseudo_derived(), - N), node.children()) + children = node.children() except SystemExit: exc_value = sys.exc_info()[1] e = SCons.Errors.ExplicitExit(node, exc_value.code) self.ready_exc = (SCons.Errors.ExplicitExit, e) self.ready = node - if T: T.write(': SystemExit\n') + if T: T.write(' SystemExit\n') break except KeyboardInterrupt: + if T: T.write(' KeyboardInterrupt\n') raise except: # We had a problem just trying to figure out the @@ -352,8 +351,16 @@ class Taskmaster: self.ready_exc = sys.exc_info() self.ready = node if S: S.problem = S.problem + 1 - if T: T.write(': exception problem\n') + if T: T.write(' exception\n') break + else: + c = map(str, children) + c.sort() + if T: T.write(' children:\n %s\n ' % c) + + childinfo = map(lambda N: (N.get_state(), + N.is_derived() or N.is_pseudo_derived(), + N), children) # Skip this node if any of its children have failed. This # catches the case where we're descending a top-level target @@ -364,7 +371,10 @@ class Taskmaster: if failed_children: node.set_state(SCons.Node.failed) if S: S.child_failed = S.child_failed + 1 - if T: T.write(': children failed:\n %s\n' % map(str, failed_children)) + if T: + c = map(str, failed_children) + c.sort() + T.write(' children failed:\n %s\n' % c) continue # Detect dependency cycles: @@ -379,7 +389,7 @@ class Taskmaster: map(lambda I: I[2], cycle) nodes.reverse() desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ") - if T: T.write(': dependency cycle\n') + if T: T.write(' dependency cycle\n') raise SCons.Errors.UserError, desc # Select all of the dependencies that are derived targets @@ -413,7 +423,10 @@ class Taskmaster: not_started.reverse() self.candidates.extend(self.order(not_started)) if S: S.not_started = S.not_started + 1 - if T: T.write(': waiting on unstarted children:\n %s\n' % map(str, not_started)) + if T: + c = map(str, not_started) + c.sort() + T.write(' waiting on unstarted children:\n %s\n' % c) continue not_built = filter(lambda I: I[0] <= SCons.Node.executing, derived_children) @@ -433,7 +446,10 @@ class Taskmaster: self.pending.append(node) node.set_state(SCons.Node.pending) if S: S.not_built = S.not_built + 1 - if T: T.write(': waiting on unfinished children:\n %s\n' % map(str, not_built)) + if T: + c = map(str, not_built) + c.sort() + T.write(' waiting on unfinished children:\n %s\n' % c) continue # Skip this node if it has side-effects that are @@ -446,14 +462,17 @@ class Taskmaster: self.pending.append(node) node.set_state(SCons.Node.pending) if S: S.side_effects = S.side_effects + 1 - if T: T.write(': waiting on side effects:\n %s\n' % map(str, side_effects)) + if T: + c = map(str, side_effects) + c.sort() + T.write(' waiting on side effects:\n %s\n' % c) continue # The default when we've gotten through all of the checks above: # this node is ready to be built. self.ready = node if S: S.build = S.build + 1 - if T: T.write(': building\n') + if T: T.write(' evaluating\n') break def next_task(self): diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index dc231592..c35a89c1 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -1052,12 +1052,20 @@ class TaskmasterTestCase(unittest.TestCase): value = trace.getvalue() expect = """\ -Taskmaster: 'n1': building +Taskmaster: 'n1': children: + [] + evaluating Taskmaster: 'n1': already handled -Taskmaster: 'n3': waiting on unstarted children: +Taskmaster: 'n3': children: + ['n1', 'n2'] + waiting on unstarted children: ['n2'] -Taskmaster: 'n2': building -Taskmaster: 'n3': waiting on unfinished children: +Taskmaster: 'n2': children: + [] + evaluating +Taskmaster: 'n3': children: + ['n1', 'n2'] + waiting on unfinished children: ['n2'] """ assert value == expect, value diff --git a/test/option/taskmastertrace.py b/test/option/taskmastertrace.py index 066e1ffc..b1b8717d 100644 --- a/test/option/taskmastertrace.py +++ b/test/option/taskmastertrace.py @@ -41,13 +41,21 @@ env.Command('file.mid', 'file.in', Copy('$TARGET', '$SOURCE')) test.write('file.in', "file.in\n") expect_stdout = test.wrap_stdout("""\ -Taskmaster: '.': waiting on unstarted children: - ['file.out', 'file.mid'] -Taskmaster: 'file.mid': building +Taskmaster: '.': children: + ['SConstruct', 'file.in', 'file.mid', 'file.out'] + waiting on unstarted children: + ['file.mid', 'file.out'] +Taskmaster: 'file.mid': children: + ['file.in'] + evaluating Copy("file.mid", "file.in") -Taskmaster: 'file.out': building +Taskmaster: 'file.out': children: + ['file.mid'] + evaluating Copy("file.out", "file.mid") -Taskmaster: '.': building +Taskmaster: '.': children: + ['SConstruct', 'file.in', 'file.mid', 'file.out'] + evaluating """) test.run(arguments='--taskmastertrace=- .', stdout=expect_stdout) @@ -66,11 +74,19 @@ Copy("file.out", "file.mid") test.run(arguments='--taskmastertrace=trace.out .', stdout=expect_stdout) expect_trace = """\ -Taskmaster: '.': waiting on unstarted children: - ['file.out', 'file.mid'] -Taskmaster: 'file.mid': building -Taskmaster: 'file.out': building -Taskmaster: '.': building +Taskmaster: '.': children: + ['SConstruct', 'file.in', 'file.mid', 'file.out'] + waiting on unstarted children: + ['file.mid', 'file.out'] +Taskmaster: 'file.mid': children: + ['file.in'] + evaluating +Taskmaster: 'file.out': children: + ['file.mid'] + evaluating +Taskmaster: '.': children: + ['SConstruct', 'file.in', 'file.mid', 'file.out'] + evaluating """ test.must_match('trace.out', expect_trace)