5 # runtest.py - wrapper script for running SCons tests
7 # This script mainly exists to set PYTHONPATH to the right list of
8 # directories to test the SCons modules.
10 # By default, it directly uses the modules in the local tree:
11 # ./src/ (source files we ship) and ./QMTest/ (other modules we don't).
13 # HOWEVER, now that SCons has Repository support, we don't have
14 # Aegis copy all of the files into the local tree. So if you're
15 # using Aegis and want to run tests by hand using this script, you
16 # must "aecp ." the entire source tree into your local directory
17 # structure. When you're done with your change, you can then
18 # "aecpu -unch ." to un-copy any files that you haven't changed.
20 # When any -p option is specified, this script assumes it's in a
21 # directory in which a build has been performed, and sets PYTHONPATH
22 # so that it *only* references the modules that have unpacked from
23 # the specified built package, to test whether the packages are good.
27 # -a Run all tests; does a virtual 'find' for
28 # all SCons tests under the current directory.
30 # --aegis Print test results to an output file (specified
31 # by the -o option) in the format expected by
32 # aetest(5). This is intended for use in the
33 # batch_test_command field in the Aegis project
36 # -d Debug. Runs the script under the Python
37 # debugger (pdb.py) so you don't have to
38 # muck with PYTHONPATH yourself.
40 # -f file Only execute the tests listed in the specified
43 # -h Print the help and exit.
45 # -l List available tests and exit.
47 # -n No execute, just print command lines.
49 # -o file Print test results to the specified file.
50 # The --aegis and --xml options specify the
53 # -P Python Use the specified Python interpreter.
55 # -p package Test against the specified package.
57 # --passed In the final summary, also report which tests
58 # passed. The default is to only report tests
59 # which failed or returned NO RESULT.
61 # -q Quiet. By default, runtest.py prints the
62 # command line it will execute before
63 # executing it. This suppresses that print.
65 # --sp The Aegis search path.
67 # --spe The Aegis executable search path.
69 # -t Print the execution time of each test.
71 # -X The scons "script" is an executable; don't
74 # -x scons The scons script to use for tests.
76 # --xml Print test results to an output file (specified
77 # by the -o option) in an SCons-specific XML format.
78 # This is (will be) used for reporting results back
79 # to a central SCons test monitoring infrastructure.
81 # (Note: There used to be a -v option that specified the SCons
82 # version to be tested, when we were installing in a version-specific
83 # library directory. If we ever resurrect that as the default, then
84 # you can find the appropriate code in the 0.04 version of this script,
85 # rather than reinventing that wheel.)
87 from __future__ import generators ### KEEP FOR COMPATIBILITY FIXERS
97 if not hasattr(os, 'WEXITSTATUS'):
98 os.WEXITSTATUS = lambda x: x
103 # Pre-2.4 Python has no sorted() function.
105 # The pre-2.4 Python list.sort() method does not support
106 # list.sort(key=) nor list.sort(reverse=) keyword arguments, so
107 # we must implement the functionality of those keyword arguments
108 # by hand instead of passing them to list.sort().
109 def sorted(iterable, cmp=None, key=None, reverse=0):
111 result = [(key(x), x) for x in iterable]
115 # Pre-2.3 Python does not support list.sort(None).
120 result = [t1 for t0,t1 in result]
129 builddir = os.path.join(cwd, 'build')
136 print_passed_summary = None
148 Usage: runtest.py [OPTIONS] [TEST ...]
150 -a, --all Run all tests.
151 --aegis Print results in Aegis format.
152 -b BASE, --baseline BASE Run test scripts against baseline BASE.
153 --builddir DIR Directory in which packages were built.
154 -d, --debug Run test scripts under the Python debugger.
155 -f FILE, --file FILE Run tests in specified FILE.
156 -h, --help Print this message and exit.
157 -l, --list List available tests and exit.
158 -n, --no-exec No execute, just print command lines.
159 --noqmtest Execute tests directly, not using QMTest.
160 -o FILE, --output FILE Print test results to FILE.
161 -P Python Use the specified Python interpreter.
162 -p PACKAGE, --package PACKAGE
163 Test against the specified PACKAGE:
165 local-tar-gz .tar.gz standalone package
166 local-zip .zip standalone package
168 src-tar-gz .tar.gz source package
169 src-zip .zip source package
170 tar-gz .tar.gz distribution
171 zip .zip distribution
172 --passed Summarize which tests passed.
173 --qmtest Run using the QMTest harness.
174 -q, --quiet Don't print the test being executed.
175 --sp PATH The Aegis search path.
176 --spe PATH The Aegis executable search path.
177 -t, --time Print test execution time.
178 -v version Specify the SCons version.
179 --verbose=LEVEL Set verbose level: 1 = print executed commands,
180 2 = print commands and non-zero output,
181 3 = print commands and all output.
182 -X Test script is executable, don't feed to Python.
183 -x SCRIPT, --exec SCRIPT Test SCRIPT.
184 --xml Print results in SCons XML format.
187 opts, args = getopt.getopt(sys.argv[1:], "ab:df:hlno:P:p:qv:Xx:t",
188 ['all', 'aegis', 'baseline=', 'builddir=',
189 'debug', 'file=', 'help',
190 'list', 'no-exec', 'noqmtest', 'output=',
191 'package=', 'passed', 'python=', 'qmtest',
192 'quiet', 'sp=', 'spe=', 'time',
197 if o in ['-a', '--all']:
199 elif o in ['-b', '--baseline']:
201 elif o in ['--builddir']:
203 if not os.path.isabs(builddir):
204 builddir = os.path.normpath(os.path.join(cwd, builddir))
205 elif o in ['-d', '--debug']:
207 pdb = os.path.join(dir, 'pdb.py')
208 if os.path.exists(pdb):
211 elif o in ['-f', '--file']:
212 if not os.path.isabs(a):
213 a = os.path.join(cwd, a)
215 elif o in ['-h', '--help']:
218 elif o in ['-l', '--list']:
220 elif o in ['-n', '--no-exec']:
222 elif o in ['--noqmtest']:
224 elif o in ['-o', '--output']:
225 if a != '-' and not os.path.isabs(a):
226 a = os.path.join(cwd, a)
228 elif o in ['-p', '--package']:
230 elif o in ['--passed']:
231 print_passed_summary = 1
232 elif o in ['-P', '--python']:
234 elif o in ['--qmtest']:
235 if sys.platform == 'win32':
236 # typically in c:/PythonXX/Scripts
240 elif o in ['-q', '--quiet']:
243 sp = a.split(os.pathsep)
245 spe = a.split(os.pathsep)
246 elif o in ['-t', '--time']:
248 elif o in ['--verbose']:
249 os.environ['TESTCMD_VERBOSE'] = a
250 elif o in ['-v', '--version']:
254 elif o in ['-x', '--exec']:
256 elif o in ['--aegis', '--xml']:
259 if not args and not all and not testlistfile:
260 sys.stderr.write("""\
261 runtest.py: No tests were specified.
262 List one or more tests on the command line, use the
263 -f option to specify a file containing a list of tests,
264 or use the -a option to find and run all tests.
269 if sys.platform in ('win32', 'cygwin'):
272 pathext = [''] + os.environ['PATHEXT'].split(os.pathsep)
273 for dir in os.environ['PATH'].split(os.pathsep):
274 f = os.path.join(dir, file)
277 if os.path.isfile(fext):
284 for dir in os.environ['PATH'].split(os.pathsep):
285 f = os.path.join(dir, file)
286 if os.path.isfile(f):
291 if stat.S_IMODE(st[stat.ST_MODE]) & 0111:
295 # See if --qmtest or --noqmtest specified
300 # Old code for using QMTest by default if it's installed.
301 # We now default to not using QMTest unless explicitly asked for.
302 #for q in ['qmtest', 'qmtest.py']:
305 # # The name was found on $PATH; just execute the found name so
306 # # we don't have to worry about paths containing white space.
310 # msg = ('Warning: found neither qmtest nor qmtest.py on $PATH;\n' +
311 # '\tassuming --noqmtest option.\n')
312 # sys.stderr.write(msg)
315 aegis = whereis('aegis')
317 if format == '--aegis' and aegis:
318 change = os.popen("aesub '$c' 2>/dev/null", "r").read()
321 paths = os.popen("aesub '$sp' 2>/dev/null", "r").read()[:-1]
322 sp = paths.split(os.pathsep)
324 spe = os.popen("aesub '$spe' 2>/dev/null", "r").read()[:-1]
325 spe = spe.split(os.pathsep)
338 _ws = re.compile('\s')
343 s = s.replace('\\', '\\\\')
346 # Set up lowest-common-denominator spawning of a process on both Windows
347 # and non-Windows systems that works all the way back to Python 1.5.2.
350 except AttributeError:
351 def spawn_it(command_args):
354 os.execv(command_args[0], command_args)
356 pid, status = os.waitpid(pid, 0)
359 def spawn_it(command_args):
360 command = command_args[0]
361 command_args = list(map(escape, command_args))
362 return os.spawnv(os.P_WAIT, command, command_args)
365 def __init__(self, path, spe=None):
367 self.abspath = os.path.abspath(path)
370 f = os.path.join(dir, path)
371 if os.path.isfile(f):
376 class SystemExecutor(Base):
378 s = spawn_it(self.command_args)
381 sys.stdout.write("Unexpected exit status %d\n" % s)
389 except AttributeError:
390 class PopenExecutor(Base):
392 (tochild, fromchild, childerr) = os.popen3(self.command_str)
394 self.stderr = childerr.read()
395 self.stdout = fromchild.read()
397 self.status = childerr.close()
401 self.status = self.status >> 8
403 class PopenExecutor(Base):
405 p = popen2.Popen3(self.command_str, 1)
407 self.stdout = p.fromchild.read()
408 self.stderr = p.childerr.read()
409 self.status = p.wait()
410 self.status = self.status >> 8
412 class PopenExecutor(Base):
414 p = subprocess.Popen(self.command_str,
415 stdout=subprocess.PIPE,
416 stderr=subprocess.PIPE,
418 self.stdout = p.stdout.read()
419 self.stderr = p.stderr.read()
420 self.status = p.wait()
422 class Aegis(SystemExecutor):
424 f.write('test_result = [\n')
426 f.write(' { file_name = "%s";\n' % self.path)
427 f.write(' exit_status = %d; },\n' % self.status)
431 class XML(PopenExecutor):
433 f.write(' <results>\n')
436 f.write(' <file_name>%s</file_name>\n' % self.path)
437 f.write(' <command_line>%s</command_line>\n' % self.command_str)
438 f.write(' <exit_status>%s</exit_status>\n' % self.status)
439 f.write(' <stdout>%s</stdout>\n' % self.stdout)
440 f.write(' <stderr>%s</stderr>\n' % self.stderr)
441 f.write(' <time>%.1f</time>\n' % self.test_time)
442 f.write(' </test>\n')
444 f.write(' <time>%.1f</time>\n' % self.total_time)
445 f.write(' </results>\n')
448 None : SystemExecutor,
453 Test = format_class[format]
459 'local-tar-gz' : None,
468 # The hard-coded "python2.1" here is the library directory
469 # name on Debian systems, not an executable, so it's all right.
471 'deb' : os.path.join('python2.1', 'site-packages')
474 if package not in dir:
475 sys.stderr.write("Unknown package '%s'\n" % package)
478 test_dir = os.path.join(builddir, 'test-%s' % package)
480 if dir[package] is None:
481 scons_script_dir = test_dir
482 globs = glob.glob(os.path.join(test_dir, 'scons-local-*'))
484 sys.stderr.write("No `scons-local-*' dir in `%s'\n" % test_dir)
487 pythonpath_dir = globs[len(globs)-1]
488 elif sys.platform == 'win32':
489 scons_script_dir = os.path.join(test_dir, dir[package], 'Scripts')
490 scons_lib_dir = os.path.join(test_dir, dir[package])
491 pythonpath_dir = scons_lib_dir
493 scons_script_dir = os.path.join(test_dir, dir[package], 'bin')
494 l = lib.get(package, 'scons')
495 scons_lib_dir = os.path.join(test_dir, dir[package], 'lib', l)
496 pythonpath_dir = scons_lib_dir
498 scons_runtest_dir = builddir
504 # XXX: Logic like the following will be necessary once
505 # we fix runtest.py to run tests within an Aegis change
506 # without symlinks back to the baseline(s).
511 # d = os.path.join(dir, 'src', 'script')
512 # f = os.path.join(d, 'scons.py')
513 # if os.path.isfile(f):
516 # spe = map(lambda x: os.path.join(x, 'src', 'engine'), spe)
517 # ld = os.pathsep.join(spe)
519 if not baseline or baseline == '.':
521 elif baseline == '-':
522 # Tentative code for fetching information directly from the
523 # QMTest context file.
526 #import qm.test.context
528 #context = qm.test.context.Context()
529 #context.Read('context')
532 svn_info = os.popen("svn info 2>&1", "r").read()
533 match = re.search('URL: (.*)', svn_info)
537 sys.stderr.write('runtest.py: could not find a URL:\n')
538 sys.stderr.write(svn_info)
541 base = tempfile.mkdtemp(prefix='runtest-tmp-')
543 command = 'cd %s && svn co -q %s' % (base, url)
545 base = os.path.join(base, os.path.split(url)[1])
553 scons_runtest_dir = base
555 scons_script_dir = sd or os.path.join(base, 'src', 'script')
557 scons_lib_dir = ld or os.path.join(base, 'src', 'engine')
559 pythonpath_dir = scons_lib_dir
562 # Let the version of SCons that the -x option pointed to find
564 os.environ['SCONS'] = scons
566 # Because SCons is really aggressive about finding its modules,
567 # it sometimes finds SCons modules elsewhere on the system.
568 # This forces SCons to use the modules that are being tested.
569 os.environ['SCONS_LIB_DIR'] = scons_lib_dir
572 os.environ['SCONS_EXEC'] = '1'
574 os.environ['SCONS_RUNTEST_DIR'] = scons_runtest_dir
575 os.environ['SCONS_SCRIPT_DIR'] = scons_script_dir
576 os.environ['SCONS_CWD'] = cwd
578 os.environ['SCONS_VERSION'] = version
580 old_pythonpath = os.environ.get('PYTHONPATH')
582 # FIXME: the following is necessary to pull in half of the testing
583 # harness from $srcdir/etc. Those modules should be transfered
584 # to QMTest/ once we completely cut over to using that as
585 # the harness, in which case this manipulation of PYTHONPATH
586 # should be able to go away.
587 pythonpaths = [ pythonpath_dir ]
590 if format == '--aegis':
591 q = os.path.join(dir, 'build', 'QMTest')
593 q = os.path.join(dir, 'QMTest')
594 pythonpaths.append(q)
596 os.environ['SCONS_SOURCE_PATH_EXECUTABLE'] = os.pathsep.join(spe)
598 os.environ['PYTHONPATH'] = os.pathsep.join(pythonpaths)
601 os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + \
607 def find_Tests_py(tdict, dirname, names):
608 for n in [n for n in names if n[-8:] == "Tests.py"]:
609 tdict[os.path.join(dirname, n)] = 1
611 def find_py(tdict, dirname, names):
612 tests = [n for n in names if n[-3:] == ".py"]
614 excludes = open(os.path.join(dirname,".exclude_tests")).readlines()
615 except (OSError, IOError):
618 for exclude in excludes:
619 exclude = exclude.split('#' , 1)[0]
620 exclude = exclude.strip()
621 if not exclude: continue
622 tests = [n for n in tests if n != exclude]
624 tdict[os.path.join(dirname, n)] = 1
630 tests.extend(glob.glob(a))
633 x = os.path.join(dir, a)
640 for path in glob.glob(a):
641 if os.path.isdir(path):
643 if path[:3] == 'src':
644 os.path.walk(path, find_Tests_py, tdict)
645 elif path[:4] == 'test':
646 os.path.walk(path, find_py, tdict)
647 tests.extend(sorted(tdict.keys()))
651 tests = open(testlistfile, 'r').readlines()
652 tests = [x for x in tests if x[0] != '#']
653 tests = [x[:-1] for x in tests]
654 elif all and not qmtest:
655 # Find all of the SCons functional tests in the local directory
656 # tree. This is anything under the 'src' subdirectory that ends
657 # with 'Tests.py', or any Python script (*.py) under the 'test'
660 # Note that there are some tests under 'src' that *begin* with
661 # 'test_', but they're packaging and installation tests, not
662 # functional tests, so we don't execute them by default. (They can
663 # still be executed by hand, though, and are routinely executed
664 # by the Aegis packaging build to make sure that we're building
667 os.path.walk('src', find_Tests_py, tdict)
668 os.path.walk('test', find_py, tdict)
669 if format == '--aegis' and aegis:
670 cmd = "aegis -list -unf pf 2>/dev/null"
671 for line in os.popen(cmd, "r").readlines():
673 if a[0] == "test" and a[-1] not in tdict:
674 tdict[a[-1]] = Test(a[-1], spe)
675 cmd = "aegis -list -unf cf 2>/dev/null"
676 for line in os.popen(cmd, "r").readlines():
681 elif a[-1] not in tdict:
682 tdict[a[-1]] = Test(a[-1], spe)
684 tests = sorted(tdict.keys())
688 aegis_result_stream = 'scons_tdb.AegisBaselineStream'
689 qmr_file = 'baseline.qmr'
691 aegis_result_stream = 'scons_tdb.AegisChangeStream'
692 qmr_file = 'results.qmr'
695 aegis_result_stream = aegis_result_stream + "(print_time='1')"
697 qmtest_args = [ qmtest, ]
699 if format == '--aegis':
701 if not os.path.isdir(dir):
703 qmtest_args.extend(['-D', dir])
707 '--output %s' % qmr_file,
709 '--result-stream="%s"' % aegis_result_stream,
713 qmtest_args.append('--context python="%s"' % python)
716 if format == '--xml':
717 rsclass = 'scons_tdb.SConsXMLResultStream'
719 rsclass = 'scons_tdb.AegisBatchStream'
720 qof = "r'" + outputfile + "'"
721 rs = '--result-stream="%s(filename=%s)"' % (rsclass, qof)
722 qmtest_args.append(rs)
724 if format == '--aegis':
725 tests = [x.replace(cwd+os.sep, '') for x in tests]
727 os.environ['SCONS'] = os.path.join(cwd, 'src', 'script', 'scons.py')
729 cmd = ' '.join(qmtest_args + tests)
731 sys.stdout.write(cmd + '\n')
735 status = os.WEXITSTATUS(os.system(cmd))
739 # os.chdir(scons_script_dir)
743 tests = list(map(Test, tests))
746 def __init__(self, file):
748 def write(self, arg):
751 def __getattr__(self, attr):
752 return getattr(self.file, attr)
754 sys.stdout = Unbuffered(sys.stdout)
755 sys.stderr = Unbuffered(sys.stderr)
759 sys.stdout.write(t.path + "\n")
764 if os.name == 'java':
765 python = os.path.join(sys.prefix, 'jython')
767 python = sys.executable
769 # time.clock() is the suggested interface for doing benchmarking timings,
770 # but time.time() does a better job on Linux systems, so let that be
771 # the non-Windows default.
773 if sys.platform == 'win32':
774 time_func = time.clock
776 time_func = time.time
779 print_time_func = lambda fmt, time: sys.stdout.write(fmt % time)
781 print_time_func = lambda fmt, time: None
783 total_start_time = time_func()
785 command_args = ['-tt']
787 command_args.append(debug)
788 command_args.append(t.path)
789 t.command_args = [python] + command_args
790 t.command_str = " ".join([escape(python)] + command_args)
792 sys.stdout.write(t.command_str + "\n")
793 test_start_time = time_func()
796 t.test_time = time_func() - test_start_time
797 print_time_func("Test execution time: %.1f seconds\n", t.test_time)
799 tests[0].total_time = time_func() - total_start_time
800 print_time_func("Total execution time for all tests: %.1f seconds\n", tests[0].total_time)
802 passed = [t for t in tests if t.status == 0]
803 fail = [t for t in tests if t.status == 1]
804 no_result = [t for t in tests if t.status == 2]
806 if len(tests) != 1 and execute_tests:
807 if passed and print_passed_summary:
809 sys.stdout.write("\nPassed the following test:\n")
811 sys.stdout.write("\nPassed the following %d tests:\n" % len(passed))
812 paths = [x.path for x in passed]
813 sys.stdout.write("\t" + "\n\t".join(paths) + "\n")
816 sys.stdout.write("\nFailed the following test:\n")
818 sys.stdout.write("\nFailed the following %d tests:\n" % len(fail))
819 paths = [x.path for x in fail]
820 sys.stdout.write("\t" + "\n\t".join(paths) + "\n")
822 if len(no_result) == 1:
823 sys.stdout.write("\nNO RESULT from the following test:\n")
825 sys.stdout.write("\nNO RESULT from the following %d tests:\n" % len(no_result))
826 paths = [x.path for x in no_result]
827 sys.stdout.write("\t" + "\n\t".join(paths) + "\n")
830 if outputfile == '-':
833 f = open(outputfile, 'w')
835 #f.write("test_result = [\n")
840 if outputfile != '-':
843 if format == '--aegis':
854 # indent-tabs-mode:nil
856 # vim: set expandtab tabstop=4 shiftwidth=4: