From: stevenknight Date: Thu, 19 Nov 2009 08:03:26 +0000 (+0000) Subject: Capture initial infrastructure for working performance tests. X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=756e8b9958e0ea0f47c81f6644105e4d41ff00f6;p=scons.git Capture initial infrastructure for working performance tests. git-svn-id: http://scons.tigris.org/svn/scons/trunk@4463 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py index a2bf8fc9..1fed39a7 100644 --- a/QMTest/TestSCons.py +++ b/QMTest/TestSCons.py @@ -18,6 +18,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import re +import shutil import string import sys import time @@ -942,6 +943,129 @@ print py_ver else: alt_cpp_suffix = '.C' return alt_cpp_suffix + + +class TimeSCons(TestSCons): + """Class for timing SCons.""" + def __init__(self, *args, **kw): + """ + In addition to normal TestSCons.TestSCons intialization, + this enables verbose mode (which causes the command lines to + be displayed in the output) and copies the contents of the + directory containing the executing script to the temporary + working directory. + """ + if not kw.has_key('verbose'): + kw['verbose'] = True + TestSCons.__init__(self, *args, **kw) + + # TODO(sgk): better way to get the script dir than sys.argv[0] + test_dir = os.path.dirname(sys.argv[0]) + test_name = os.path.basename(test_dir) + + if not os.path.isabs(test_dir): + test_dir = os.path.join(self.orig_cwd, test_dir) + self.copy_timing_configuration(test_dir, self.workpath()) + + def main(self, *args, **kw): + """ + The main entry point for standard execution of timings. + + This method run SCons three times: + + Once with the --help option, to have it exit after just reading + the configuration. + + Once as a full build of all targets. + + Once again as a (presumably) null or up-to-date build of + all targets. + + The elapsed time to execute each build is printed after + it has finished. + """ + self.help(*args, **kw) + self.full(*args, **kw) + self.null(*args, **kw) + + def help(self, *args, **kw): + """ + Runs scons with the --help option. + + This serves as a way to isolate just the amount of time spent + reading up the configuration, since --help exits before any + "real work" is done. + """ + kw['options'] = kw.get('options', '') + ' --help' + self.run_build(*args, **kw) + sys.stdout.write(self.stdout()) + print "RESULT", self.elapsed_time() + + def full(self, *args, **kw): + """ + Runs a full build of SCons. + """ + self.run_build(*args, **kw) + sys.stdout.write(self.stdout()) + print "RESULT", self.elapsed_time() + + def null(self, *args, **kw): + """ + Runs an up-to-date null build of SCons. + """ + # TODO(sgk): allow the caller to specify the target (argument) + # that must be up-to-date. + self.up_to_date(arguments='.', **kw) + sys.stdout.write(self.stdout()) + print "RESULT", self.elapsed_time() + + def elapsed_time(self): + """ + Returns the elapsed time of the most recent command execution. + """ + return self.endTime - self.startTime + + def run_build(self, *args, **kw): + """ + Runs a single build command, capturing output in the specified file. + + Because this class is about timing SCons, we record the start + and end times of the elapsed execution, and also add the + --debug=memory and --debug=time options to have SCons report + its own memory and timing statistics. + """ + kw['options'] = kw.get('options', '') + ' --debug=memory --debug=time' + self.startTime = time.time() + try: + result = TestSCons.run(self, *args, **kw) + finally: + self.endTime = time.time() + return result + + def copy_timing_configuration(self, source_dir, dest_dir): + """ + Copies the timing configuration from the specified source_dir (the + directory in which the controlling script lives) to the specified + dest_dir (a temporary working directory). + + This ignores all files and directories that begin with the string + 'TimeSCons-', and all '.svn' subdirectories. + """ + for root, dirs, files in os.walk(source_dir): + if '.svn' in dirs: + dirs.remove('.svn') + dirs = [ d for d in dirs if not d.startswith('TimeSCons-') ] + files = [ f for f in files if not f.startswith('TimeSCons-') ] + for dirname in dirs: + source = os.path.join(root, dirname) + destination = source.replace(source_dir, dest_dir) + os.mkdir(destination) + if sys.platform != 'win32': + shutil.copystat(source, destination) + for filename in files: + source = os.path.join(root, filename) + destination = source.replace(source_dir, dest_dir) + shutil.copy2(source, destination) # In some environments, $AR will generate a warning message to stderr diff --git a/timings/CPPPATH/SConstruct b/timings/CPPPATH/SConstruct index 728db9cb..d49e3d63 100644 --- a/timings/CPPPATH/SConstruct +++ b/timings/CPPPATH/SConstruct @@ -21,45 +21,10 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -""" -This configuration is for testing the timing of searching long lists of -CPPPATH directories. +dir_count = int(ARGUMENTS['DIR_COUNT']) -We create 100 on-disk directories, with a single .h file in the last -directory in the list. We set CPPPATH to a list of Dir Nodes for the -created directories. The .c file we create #includes the .h file to be -found in the last directory in the list. -""" - -import os -import os.path - -dir_cnt = 100 - -dir_list = map(lambda t: 'inc_%03d' % t, xrange(dir_cnt)) - -for dir in dir_list: - if not os.path.isdir(dir): - os.mkdir(dir) - -foo_h = 'inc_099/foo.h' - -if not os.path.isfile(foo_h): - open(foo_h, 'w').write('#define FOO 1\n') - -contents = """\ -#include "foo.h" -void -foo(void) -{ - ; -} -""" - -if not os.path.isfile('foo.c'): - open('foo.c', 'w').write(contents) - -inc_list = map(lambda d: Dir(d), dir_list) +inc_list = [ Dir('inc_%04d' % t) for t in xrange(dir_count) ] +inc_list.append(Dir('include')) env = Environment(CPPPATH = inc_list) diff --git a/timings/CPPPATH/st.conf b/timings/CPPPATH/TimeSCons-run.py similarity index 61% rename from timings/CPPPATH/st.conf rename to timings/CPPPATH/TimeSCons-run.py index 6507ea24..d88042e6 100644 --- a/timings/CPPPATH/st.conf +++ b/timings/CPPPATH/TimeSCons-run.py @@ -19,26 +19,26 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# """ -scons-time.py configuration file for the "CPPPATH" timing test. +This configuration times searching long lists of CPPPATH directories. + +We create 5000 on-disk directories. A single checked-in .h file exists +in the 'include' directory. The SConstruct sets CPPPATH to a list of Dir +Nodes for the created directories, followed by 'include'. A checked-in .c +file #includes the .h file to be found in the last directory in the list. """ -archive_list = [ 'SConstruct' ] -subdir = '.' +import TestSCons + +test = TestSCons.TimeSCons() + +dir_count = 5000 -import sys -sys.path.insert(0, '..') -import SCons_Bars +for d in xrange(dir_count): + test.subdir('inc_%04d' % d) -revs = [ - 1224, # Don't create a Node for every file we try to find during scan. - 1349, # More efficient checking for on-disk file entries. - 1407, # Use a Dir scanner instead of a hard-coded method. - 1433, # Remove unnecessary creation of RCS and SCCS Node.Dir nodes. - 1703, # Lobotomize Memoizer. - 2380, # The Big Signature Refactoring hits branches/core. -] +test.main(options='DIR_COUNT=%s' % dir_count) -vertical_bars = SCons_Bars.Release_Bars.gnuplot(labels=True) + \ - SCons_Bars.Revision_Bars.gnuplot(labels=False, revs=revs) +test.pass_test() diff --git a/timings/CPPPATH/foo.c b/timings/CPPPATH/foo.c new file mode 100644 index 00000000..3cbb0362 --- /dev/null +++ b/timings/CPPPATH/foo.c @@ -0,0 +1,6 @@ +#include "foo.h" +void +foo(void) +{ + ; +} diff --git a/timings/CPPPATH/include/foo.h b/timings/CPPPATH/include/foo.h new file mode 100644 index 00000000..b1e1311e --- /dev/null +++ b/timings/CPPPATH/include/foo.h @@ -0,0 +1 @@ +#define FOO 1 diff --git a/timings/JTimer/SConstruct b/timings/JTimer/SConstruct index e1e38d20..95764a63 100644 --- a/timings/JTimer/SConstruct +++ b/timings/JTimer/SConstruct @@ -21,33 +21,18 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -""" -This configuration is for timing how we evaluate long chains of -dependencies, specifically when -j is used. - -We set up a chain of 100 targets that get built from a Python function -action with no source files (equivalent to "echo junk > $TARGET"). -Each target explicitly depends on the next target in turn, so the -Taskmaster will do a deep walk of the dependency graph. - -This test case was contributed by Kevin Massey. Prior to revision 1468, -we had a serious O(N^2) problem in the Taskmaster when handling long -dependency chains like this. That was fixed by adding reference counts -to the Taskmaster so it could be smarter about not re-evaluating Nodes. -""" - -target_cnt = 100 +target_count = int(ARGUMENTS['TARGET_COUNT']) env = Environment() def write_file( env, target, source ): - path_target = env.File( target ).path + path_target = env.File( target )[0].path outfile = open( path_target, 'w' ) outfile.write( 'junk' ) outfile.close() list = [] -for i in range( target_cnt ): +for i in range( target_count ): target = 'target_%03d' % i env.Command( target, [], write_file ) env.Depends( target, list ) diff --git a/timings/hundred/st.conf b/timings/JTimer/TimeSCons-run.py similarity index 57% rename from timings/hundred/st.conf rename to timings/JTimer/TimeSCons-run.py index adfb09eb..36b016f1 100644 --- a/timings/hundred/st.conf +++ b/timings/JTimer/TimeSCons-run.py @@ -22,26 +22,22 @@ # """ -scons-time.py configuration file for the "hundred" timing test. -""" +This configuration is for timing how we evaluate long chains of +dependencies, specifically when -j is used. + +We set up a chain of 500 targets that get built from a Python function +action with no source files (equivalent to "echo junk > $TARGET"). +Each target explicitly depends on the next target in turn, so the +Taskmaster will do a deep walk of the dependency graph. -archive_list = [ 'SConstruct' ] -subdir = '.' +This test case was contributed by Kevin Massey. Prior to revision 1468, +we had a serious O(N^2) problem in the Taskmaster when handling long +dependency chains like this. That was fixed by adding reference counts +to the Taskmaster so it could be smarter about not re-evaluating Nodes. +""" -import sys -sys.path.insert(0, '..') -import SCons_Bars +import TestSCons -revs = [ - 1220, # Use WeakValueDicts in the Memoizer to reduce memory use. - 1307, # Move signature Node tranlation of rel_paths into the class. - 1435, # Fix Debug.caller() directory separators. - 1477, # Delay disambiguation of Node.FS.Entry into File/Dir. - 1655, # Reduce unnecessary calls to Node.FS.disambiguate(). - 1703, # Lobotomize Memoizer. - 1727, # Cache Executor methods, reduce calls when scanning. - 2380, # The Big Signature Refactoring hits branches/core. -] +target_count = 500 -vertical_bars = SCons_Bars.Release_Bars.gnuplot(labels=True) + \ - SCons_Bars.Revision_Bars.gnuplot(labels=False, revs=revs) +TestSCons.TimeSCons().main(options='TARGET_COUNT=%d' % target_count) diff --git a/timings/README.txt b/timings/README.txt new file mode 100644 index 00000000..840c7f80 --- /dev/null +++ b/timings/README.txt @@ -0,0 +1,32 @@ +This directory contains timing configurations for SCons. + +Each configuration exists in a subdirectory. The controlling script +is named TimeSCons-run.py for the configuration. The TimeSCons-run.py +scripts use TestSCons.TimeSCons, a subclass of TestSCons.TestSCons (both +defined in ../QMTest/TestScons.py), to manage execution of the timing +runs. + +Unlike the TestSCons.TestSCons base class, the TestSCons.TimeSCons +subclass copies the contents of its containing directory to the temporary +working directory. (It avoids copying the .svn directory, and any files +or directories that start with the string "TimeSCons-".) This allows +the timing configuration files to be checked in directly to our source +code management system, instead of requiring that they be created from +in-line data inside the script. + +The simplest-possible TimeSCons-run.py script would look like: + + import TestSCons + TestSCons.TimeSCons().main() + +The above script would end up executing a SConstruct file configuration +in a temporary directory. The main() method is the standard interface +for a timing run. See its docstring for precisely what it does. + +Although the TestSCons.TimeSCons subclass copies its directory contents to +a temporary working directory for the timing run, because it is a subclass +of TestSCons.TestSCons, it *can* also create files or directories from +in-line data. This is typically done when it's necessary to create +hundreds of identical input files or directories before running the +timing test, to avoid cluttering our SCM system with hundreds of otherwise +meaningless files. diff --git a/timings/hundred/SConstruct b/timings/hundred/SConstruct index 2332d732..648c26ac 100644 --- a/timings/hundred/SConstruct +++ b/timings/hundred/SConstruct @@ -21,32 +21,15 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -""" -This configuration is for timing how we handle the NxM interaction when -we build a lot of targets from a lot of source files. +target_count = int(ARGUMENTS['TARGET_COUNT']) -We create a list of 100 target files that will each be built by copying -a file from a corresponding list of 100 source files. The source -files themselves are each built by a Python function action that's the -equivalent of "echo contents > $TARGET". -""" +def copy_files( env, target, source ): + for t, s in zip(target, source): + open(str(t), 'wb').write(open(str(s), 'rb').read()) -target_cnt = 100 +source_list = map(lambda t: 'source_%04d' % t, xrange(target_count)) +target_list = map(lambda t: 'target_%04d' % t, xrange(target_count)) env = Environment() -def create_file( env, target, source ): - t = str(target[0]) - open( t, 'w' ).write('contents\n') - -source_list = map(lambda t: 'source_%03d' % t, xrange(target_cnt)) -target_list = map(lambda t: 'target_%03d' % t, xrange(target_cnt)) - -for source in source_list: - env.Command( source, [], create_file ) - -def copy_files( env, target, source ): - for t, s in zip(target, source): - open(str(t), 'w').write(open(str(s), 'r').read()) - env.Command( target_list, source_list, copy_files ) diff --git a/timings/JTimer/st.conf b/timings/hundred/TimeSCons-run.py similarity index 57% rename from timings/JTimer/st.conf rename to timings/hundred/TimeSCons-run.py index cf1ecbfe..d21db8a5 100644 --- a/timings/JTimer/st.conf +++ b/timings/hundred/TimeSCons-run.py @@ -22,27 +22,24 @@ # """ -scons-time.py configuration file for the "JTimer" timing test. +This configuration is for timing how we handle the NxM interaction when +we build a lot of targets from a lot of source files. + +We create a list of 500 target files that will each be built by copying +a file from a corresponding list of 500 source files. The source +files themselves are each built by a Python function action that's the +equivalent of "echo contents > $TARGET". """ -archive_list = [ 'SConstruct' ] -subdir = '.' -targets = '-j2' - -import sys -sys.path.insert(0, '..') -import SCons_Bars - -revs = [ - 1261, # Fix -j re-scanning built files for implicit deps. - 1307, # Move signature Node tranlation of rel_paths into the class. - 1407, # Use a Dir scanner instead of a hard-coded method. - 1435, # Don't prep .sconsign dependencies until needed. - 1468, # Use waiting-Node reference counts to speed up Taskmaster. - 1703, # Lobotomize Memoizer. - 1706, # Fix _doLookup value-cache misspellings. - 2380, # The Big Signature Refactoring hits branches/core. -] - -vertical_bars = SCons_Bars.Release_Bars.gnuplot(labels=True) + \ - SCons_Bars.Revision_Bars.gnuplot(labels=False, revs=revs) +import TestSCons + +test = TestSCons.TimeSCons() + +target_count = 500 + +for t in xrange(target_count): + open('source_%04d' % t, 'wb' ).write('contents\n') + +test.main(options='TARGET_COUNT=%s' % target_count) + +test.pass_test()