2 TestCmd.py: a testing framework for commands and scripts.
4 The TestCmd module provides a framework for portable automated testing
5 of executable commands and scripts (in any language, not just Python),
6 especially commands and scripts that require file system interaction.
8 In addition to running tests and evaluating conditions, the TestCmd
9 module manages and cleans up one or more temporary workspace
10 directories, and provides methods for creating files and directories in
11 those workspace directories from in-line data, here-documents), allowing
12 tests to be completely self-contained.
14 A TestCmd environment object is created via the usual invocation:
17 test = TestCmd.TestCmd()
19 There are a bunch of keyword arguments that you can use at instantiation
22 test = TestCmd.TestCmd(description = 'string',
23 program = 'program_or_script_to_test',
24 interpreter = 'script_interpreter',
28 match = default_match_function,
31 There are a bunch of methods that let you do a bunch of different
32 things. Here is an overview of them:
36 test.description_set('string')
38 test.program_set('program_or_script_to_test')
40 test.interpreter_set('script_interpreter')
42 test.workdir_set('prefix')
46 test.workpath('subdir', 'file')
48 test.subdir('subdir', ...)
50 test.write('file', "contents\n")
51 test.write(['subdir', 'file'], "contents\n")
54 test.read(['subdir', 'file'])
55 test.read('file', mode)
56 test.read(['subdir', 'file'], mode)
58 test.writable('dir', 1)
59 test.writable('dir', None)
61 test.preserve(condition, ...)
63 test.cleanup(condition)
65 test.run(program = 'program_or_script_to_run',
66 interpreter = 'script_interpreter',
67 arguments = 'arguments to pass to program',
68 chdir = 'directory_to_chdir_to',
69 stdin = 'input to feed to the program\n')
72 test.pass_test(condition)
73 test.pass_test(condition, function)
76 test.fail_test(condition)
77 test.fail_test(condition, function)
78 test.fail_test(condition, function, skip)
81 test.no_result(condition)
82 test.no_result(condition, function)
83 test.no_result(condition, function, skip)
91 test.symlink(target, link)
93 test.match(actual, expected)
95 test.match_exact("actual 1\nactual 2\n", "expected 1\nexpected 2\n")
96 test.match_exact(["actual 1\n", "actual 2\n"],
97 ["expected 1\n", "expected 2\n"])
99 test.match_re("actual 1\nactual 2\n", regex_string)
100 test.match_re(["actual 1\n", "actual 2\n"], list_of_regexes)
102 test.match_re_dotall("actual 1\nactual 2\n", regex_string)
103 test.match_re_dotall(["actual 1\n", "actual 2\n"], list_of_regexes)
109 test.where_is('foo', 'PATH1:PATH2')
110 test.where_is('foo', 'PATH1;PATH2', '.suffix3;.suffix4')
113 test.unlink('subdir', 'file')
115 The TestCmd module provides pass_test(), fail_test(), and no_result()
116 unbound functions that report test results for use with the Aegis change
117 management system. These methods terminate the test immediately,
118 reporting PASSED, FAILED, or NO RESULT respectively, and exiting with
119 status 0 (success), 1 or 2 respectively. This allows for a distinction
120 between an actual failed test and a test that could not be properly
121 evaluated because of an external condition (such as a full file system
122 or incorrect permissions).
127 TestCmd.pass_test(condition)
128 TestCmd.pass_test(condition, function)
131 TestCmd.fail_test(condition)
132 TestCmd.fail_test(condition, function)
133 TestCmd.fail_test(condition, function, skip)
136 TestCmd.no_result(condition)
137 TestCmd.no_result(condition, function)
138 TestCmd.no_result(condition, function, skip)
140 The TestCmd module also provides unbound functions that handle matching
141 in the same way as the match_*() methods described above.
145 test = TestCmd.TestCmd(match = TestCmd.match_exact)
147 test = TestCmd.TestCmd(match = TestCmd.match_re)
149 test = TestCmd.TestCmd(match = TestCmd.match_re_dotall)
151 Lastly, the where_is() method also exists in an unbound function
156 TestCmd.where_is('foo')
157 TestCmd.where_is('foo', 'PATH1:PATH2')
158 TestCmd.where_is('foo', 'PATH1;PATH2', '.suffix3;.suffix4')
161 # Copyright 2000, 2001, 2002, 2003, 2004 Steven Knight
162 # This module is free software, and you may redistribute it and/or modify
163 # it under the same terms as Python itself, so long as this copyright message
164 # and disclaimer are retained in their original form.
166 # IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
167 # SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF
168 # THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
171 # THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
172 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
173 # PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS,
174 # AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
175 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
177 __author__ = "Steven Knight <knight at baldmt dot com>"
178 __revision__ = "TestCmd.py 0.13.D002 2004/11/20 08:34:16 knight"
195 __all__ = [ 'fail_test', 'no_result', 'pass_test',
196 'match_exact', 'match_re', 'match_re_dotall',
197 'python_executable', 'TestCmd' ]
200 return type(e) is types.ListType \
201 or isinstance(e, UserList.UserList)
204 from UserString import UserString
209 if hasattr(types, 'UnicodeType'):
211 return type(e) is types.StringType \
212 or type(e) is types.UnicodeType \
213 or isinstance(e, UserString)
216 return type(e) is types.StringType or isinstance(e, UserString)
218 tempfile.template = 'testcmd.'
220 if os.name == 'posix':
223 "escape shell special characters"
227 arg = string.replace(arg, slash, slash+slash)
229 arg = string.replace(arg, c, slash+c)
231 return '"' + arg + '"'
235 # Windows does not allow special characters in file names
236 # anyway, so no need for an escape function, we will just quote
238 escape_cmd = lambda x: '"' + x + '"'
250 sys.exitfunc = _clean
252 def _caller(tblist, skip):
255 for file, line, name, text in tblist:
256 if file[-10:] == "TestCmd.py":
258 arr = [(file, line, name, text)] + arr
260 for file, line, name, text in arr[skip:]:
264 name = " (" + name + ")"
265 string = string + ("%s line %d of %s%s\n" % (atfrom, line, file, name))
269 def fail_test(self = None, condition = 1, function = None, skip = 0):
270 """Cause the test to fail.
272 By default, the fail_test() method reports that the test FAILED
273 and exits with a status of 1. If a condition argument is supplied,
274 the test fails only if the condition is true.
278 if not function is None:
285 of = " of " + self.program
288 desc = " [" + self.description + "]"
291 at = _caller(traceback.extract_stack(), skip)
292 sys.stderr.write("FAILED test" + of + desc + sep + at)
296 def no_result(self = None, condition = 1, function = None, skip = 0):
297 """Causes a test to exit with no valid result.
299 By default, the no_result() method reports NO RESULT for the test
300 and exits with a status of 2. If a condition argument is supplied,
301 the test fails only if the condition is true.
305 if not function is None:
312 of = " of " + self.program
315 desc = " [" + self.description + "]"
318 at = _caller(traceback.extract_stack(), skip)
319 sys.stderr.write("NO RESULT for test" + of + desc + sep + at)
323 def pass_test(self = None, condition = 1, function = None):
324 """Causes a test to pass.
326 By default, the pass_test() method reports PASSED for the test
327 and exits with a status of 0. If a condition argument is supplied,
328 the test passes only if the condition is true.
332 if not function is None:
334 sys.stderr.write("PASSED\n")
337 def match_exact(lines = None, matches = None):
340 if not is_List(lines):
341 lines = string.split(lines, "\n")
342 if not is_List(matches):
343 matches = string.split(matches, "\n")
344 if len(lines) != len(matches):
346 for i in range(len(lines)):
347 if lines[i] != matches[i]:
351 def match_re(lines = None, res = None):
354 if not is_List(lines):
355 lines = string.split(lines, "\n")
357 res = string.split(res, "\n")
358 if len(lines) != len(res):
360 for i in range(len(lines)):
361 if not re.compile("^" + res[i] + "$").search(lines[i]):
365 def match_re_dotall(lines = None, res = None):
368 if not type(lines) is type(""):
369 lines = string.join(lines, "\n")
370 if not type(res) is type(""):
371 res = string.join(res, "\n")
372 if re.compile("^" + res + "$", re.DOTALL).match(lines):
375 if os.name == 'java':
377 python_executable = os.path.join(sys.prefix, 'jython')
381 python_executable = sys.executable
383 if sys.platform == 'win32':
385 default_sleep_seconds = 2
387 def where_is(file, path=None, pathext=None):
389 path = os.environ['PATH']
391 path = string.split(path, os.pathsep)
393 pathext = os.environ['PATHEXT']
394 if is_String(pathext):
395 pathext = string.split(pathext, os.pathsep)
397 if string.lower(ext) == string.lower(file[-len(ext):]):
401 f = os.path.join(dir, file)
404 if os.path.isfile(fext):
410 def where_is(file, path=None, pathext=None):
412 path = os.environ['PATH']
414 path = string.split(path, os.pathsep)
416 f = os.path.join(dir, file)
417 if os.path.isfile(f):
422 if stat.S_IMODE(st[stat.ST_MODE]) & 0111:
426 default_sleep_seconds = 1
432 def __init__(self, description = None,
440 self._cwd = os.getcwd()
441 self.description_set(description)
442 self.program_set(program)
443 self.interpreter_set(interpreter)
444 self.verbose_set(verbose)
445 self.combine = combine
446 if not match is None:
447 self.match_func = match
449 self.match_func = match_re
451 self._preserve = {'pass_test': 0, 'fail_test': 0, 'no_result': 0}
452 if os.environ.has_key('PRESERVE') and not os.environ['PRESERVE'] is '':
453 self._preserve['pass_test'] = os.environ['PRESERVE']
454 self._preserve['fail_test'] = os.environ['PRESERVE']
455 self._preserve['no_result'] = os.environ['PRESERVE']
458 self._preserve['pass_test'] = os.environ['PRESERVE_PASS']
462 self._preserve['fail_test'] = os.environ['PRESERVE_FAIL']
466 self._preserve['no_result'] = os.environ['PRESERVE_NO_RESULT']
472 self.condition = 'no_result'
473 self.workdir_set(workdir)
480 return "%x" % id(self)
482 def cleanup(self, condition = None):
483 """Removes any temporary working directories for the specified
484 TestCmd environment. If the environment variable PRESERVE was
485 set when the TestCmd environment was created, temporary working
486 directories are not removed. If any of the environment variables
487 PRESERVE_PASS, PRESERVE_FAIL, or PRESERVE_NO_RESULT were set
488 when the TestCmd environment was created, then temporary working
489 directories are not removed if the test passed, failed, or had
490 no result, respectively. Temporary working directories are also
491 preserved for conditions specified via the preserve method.
493 Typically, this method is not called directly, but is used when
494 the script exits to clean up temporary working directories as
495 appropriate for the exit status.
497 if not self._dirlist:
501 if condition is None:
502 condition = self.condition
503 if self._preserve[condition]:
504 for dir in self._dirlist:
505 print "Preserved directory", dir
507 list = self._dirlist[:]
510 self.writable(dir, 1)
511 shutil.rmtree(dir, ignore_errors = 1)
516 _Cleanup.remove(self)
517 except (AttributeError, ValueError):
520 def description_set(self, description):
521 """Set the description of the functionality being tested.
523 self.description = description
526 # """Diff two arrays.
529 def fail_test(self, condition = 1, function = None, skip = 0):
530 """Cause the test to fail.
534 self.condition = 'fail_test'
535 fail_test(self = self,
536 condition = condition,
540 def interpreter_set(self, interpreter):
541 """Set the program to be used to interpret the program
542 under test as a script.
544 self.interpreter = interpreter
546 def match(self, lines, matches):
547 """Compare actual and expected file contents.
549 return self.match_func(lines, matches)
551 def match_exact(self, lines, matches):
552 """Compare actual and expected file contents.
554 return match_exact(lines, matches)
556 def match_re(self, lines, res):
557 """Compare actual and expected file contents.
559 return match_re(lines, res)
561 def match_re_dotall(self, lines, res):
562 """Compare actual and expected file contents.
564 return match_re_dotall(lines, res)
566 def no_result(self, condition = 1, function = None, skip = 0):
567 """Report that the test could not be run.
571 self.condition = 'no_result'
572 no_result(self = self,
573 condition = condition,
577 def pass_test(self, condition = 1, function = None):
578 """Cause the test to pass.
582 self.condition = 'pass_test'
583 pass_test(self = self, condition = condition, function = function)
585 def preserve(self, *conditions):
586 """Arrange for the temporary working directories for the
587 specified TestCmd environment to be preserved for one or more
588 conditions. If no conditions are specified, arranges for
589 the temporary working directories to be preserved for all
593 conditions = ('pass_test', 'fail_test', 'no_result')
594 for cond in conditions:
595 self._preserve[cond] = 1
597 def program_set(self, program):
598 """Set the executable program or script to be tested.
600 if program and not os.path.isabs(program):
601 program = os.path.join(self._cwd, program)
602 self.program = program
604 def read(self, file, mode = 'rb'):
605 """Reads and returns the contents of the specified file name.
606 The file name may be a list, in which case the elements are
607 concatenated with the os.path.join() method. The file is
608 assumed to be under the temporary working directory unless it
609 is an absolute path name. The I/O mode for the file may
610 be specified; it must begin with an 'r'. The default is
614 file = apply(os.path.join, tuple(file))
615 if not os.path.isabs(file):
616 file = os.path.join(self.workdir, file)
618 raise ValueError, "mode must begin with 'r'"
619 return open(file, mode).read()
621 def run(self, program = None,
626 """Runs a test of the program or script for the test
627 environment. Standard output and error output are saved for
628 future retrieval via the stdout() and stderr() methods.
632 if not os.path.isabs(chdir):
633 chdir = os.path.join(self.workpath(chdir))
635 sys.stderr.write("chdir(" + chdir + ")\n")
639 if not os.path.isabs(program):
640 program = os.path.join(self._cwd, program)
641 cmd = escape_cmd(program)
643 cmd = interpreter + " " + cmd
645 cmd = escape_cmd(self.program)
647 cmd = self.interpreter + " " + cmd
649 cmd = cmd + " " + arguments
651 sys.stderr.write(cmd + "\n")
653 p = popen2.Popen3(cmd, 1)
654 except AttributeError:
655 (tochild, fromchild, childerr) = os.popen3(cmd)
663 out = fromchild.read()
664 err = childerr.read()
666 self._stdout.append(out + err)
668 self._stdout.append(out)
669 self._stderr.append(err)
671 self.status = childerr.close()
680 p.tochild.write(line)
682 p.tochild.write(stdin)
684 out = p.fromchild.read()
685 err = p.childerr.read()
687 self._stdout.append(out + err)
689 self._stdout.append(out)
690 self._stderr.append(err)
691 self.status = p.wait()
695 def sleep(self, seconds = default_sleep_seconds):
696 """Sleeps at least the specified number of seconds. If no
697 number is specified, sleeps at least the minimum number of
698 seconds necessary to advance file time stamps on the current
699 system. Sleeping more seconds is all right.
703 def stderr(self, run = None):
704 """Returns the error output from the specified run number.
705 If there is no specified run number, then returns the error
706 output of the last run. If the run number is less than zero,
707 then returns the error output from that many runs back from the
711 run = len(self._stderr)
713 run = len(self._stderr) + run
715 return self._stderr[run]
717 def stdout(self, run = None):
718 """Returns the standard output from the specified run number.
719 If there is no specified run number, then returns the standard
720 output of the last run. If the run number is less than zero,
721 then returns the standard output from that many runs back from
725 run = len(self._stdout)
727 run = len(self._stdout) + run
729 return self._stdout[run]
731 def subdir(self, *subdirs):
732 """Create new subdirectories under the temporary working
733 directory, one for each argument. An argument may be a list,
734 in which case the list elements are concatenated using the
735 os.path.join() method. Subdirectories multiple levels deep
736 must be created using a separate argument for each level:
738 test.subdir('sub', ['sub', 'dir'], ['sub', 'dir', 'ectory'])
740 Returns the number of subdirectories actually created.
747 sub = apply(os.path.join, tuple(sub))
748 new = os.path.join(self.workdir, sub)
757 def symlink(self, target, link):
758 """Creates a symlink to the specified target.
759 The link name may be a list, in which case the elements are
760 concatenated with the os.path.join() method. The link is
761 assumed to be under the temporary working directory unless it
762 is an absolute path name. The target is *not* assumed to be
763 under the temporary working directory.
766 link = apply(os.path.join, tuple(link))
767 if not os.path.isabs(link):
768 link = os.path.join(self.workdir, link)
769 os.symlink(target, link)
771 def unlink(self, file):
772 """Unlinks the specified file name.
773 The file name may be a list, in which case the elements are
774 concatenated with the os.path.join() method. The file is
775 assumed to be under the temporary working directory unless it
776 is an absolute path name.
779 file = apply(os.path.join, tuple(file))
780 if not os.path.isabs(file):
781 file = os.path.join(self.workdir, file)
784 def verbose_set(self, verbose):
785 """Set the verbose level.
787 self.verbose = verbose
789 def where_is(self, file, path=None, pathext=None):
790 """Find an executable file.
793 file = apply(os.path.join, tuple(file))
794 if not os.path.isabs(file):
795 file = where_is(file, path, pathext)
798 def workdir_set(self, path):
799 """Creates a temporary working directory with the specified
800 path name. If the path is a null string (''), a unique
801 directory name is created.
805 path = tempfile.mktemp()
808 # We'd like to set self.workdir like this:
809 # self.workdir = path
810 # But symlinks in the path will report things
811 # differently from os.getcwd(), so chdir there
812 # and back to fetch the canonical path.
815 self.workdir = os.getcwd()
817 # Uppercase the drive letter since the case of drive
818 # letters is pretty much random on win32:
819 drive,rest = os.path.splitdrive(self.workdir)
821 self.workdir = string.upper(drive) + rest
823 self._dirlist.append(self.workdir)
828 _Cleanup.append(self)
832 def workpath(self, *args):
833 """Returns the absolute path name to a subdirectory or file
834 within the current temporary working directory. Concatenates
835 the temporary working directory name with the specified
836 arguments using the os.path.join() method.
838 return apply(os.path.join, (self.workdir,) + tuple(args))
840 def writable(self, top, write):
841 """Make the specified directory tree writable (write == 1)
842 or not (write == None).
845 def _walk_chmod(arg, dirname, names):
846 st = os.stat(dirname)
847 os.chmod(dirname, arg(st[stat.ST_MODE]))
849 n = os.path.join(dirname, name)
851 os.chmod(n, arg(st[stat.ST_MODE]))
853 def _mode_writable(mode):
854 return stat.S_IMODE(mode|0200)
856 def _mode_non_writable(mode):
857 return stat.S_IMODE(mode&~0200)
862 f = _mode_non_writable
863 if os.path.isfile(top):
865 os.chmod(top, f(st[stat.ST_MODE]))
868 os.path.walk(top, _walk_chmod, f)
870 pass # ignore any problems changing modes
872 def write(self, file, content, mode = 'wb'):
873 """Writes the specified content text (second argument) to the
874 specified file name (first argument). The file name may be
875 a list, in which case the elements are concatenated with the
876 os.path.join() method. The file is created under the temporary
877 working directory. Any subdirectories in the path must already
878 exist. The I/O mode for the file may be specified; it must
879 begin with a 'w'. The default is 'wb' (binary write).
882 file = apply(os.path.join, tuple(file))
883 if not os.path.isabs(file):
884 file = os.path.join(self.workdir, file)
886 raise ValueError, "mode must begin with 'w'"
887 open(file, mode).write(content)