runtests.py

   1 #!/usr/bin/python
   2
   3 import os, sys, re, shutil, unittest, doctest
   4
   5 WITH_CYTHON = True
   6
   7 from distutils.dist import Distribution
   8 from distutils.core import Extension
   9 from distutils.command.build_ext import build_ext
  10 distutils_distro = Distribution()
  11
  12 TEST_DIRS = ['compile', 'errors', 'run']
  13 TEST_RUN_DIRS = ['run']
  14
  15 INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
  16 CFLAGS = os.getenv('CFLAGS', '').split()
  17
  18
  19 class ErrorWriter(object):
  20     match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
  21     def __init__(self):
  22         self.output = []
  23         self.write = self.output.append
  24
  25     def _collect(self, collect_errors, collect_warnings):
  26         s = ''.join(self.output)
  27         result = []
  28         for line in s.split('\n'):
  29             match = self.match_error(line)
  30             if match:
  31                 is_warning, line, column, message = match.groups()
  32                 if (is_warning and collect_warnings) or \
  33                         (not is_warning and collect_errors):
  34                     result.append( (int(line), int(column), message.strip()) )
  35         result.sort()
  36         return [ "%d:%d: %s" % values for values in result ]
  37
  38     def geterrors(self):
  39         return self._collect(True, False)
  40
  41     def getwarnings(self):
  42         return self._collect(False, True)
  43
  44     def getall(self):
  45         return self._collect(True, True)
  46
  47 class TestBuilder(object):
  48     def __init__(self, rootdir, workdir, selectors, annotate, cleanup_workdir):
  49         self.rootdir = rootdir
  50         self.workdir = workdir
  51         self.selectors = selectors
  52         self.annotate = annotate
  53         self.cleanup_workdir = cleanup_workdir
  54
  55     def build_suite(self):
  56         suite = unittest.TestSuite()
  57         filenames = os.listdir(self.rootdir)
  58         filenames.sort()
  59         for filename in filenames:
  60             if not WITH_CYTHON and filename == "errors":
  61                 # we won't get any errors without running Cython
  62                 continue
  63             path = os.path.join(self.rootdir, filename)
  64             if os.path.isdir(path) and filename in TEST_DIRS:
  65                 suite.addTest(
  66                     self.handle_directory(path, filename))
  67         return suite
  68
  69     def handle_directory(self, path, context):
  70         workdir = os.path.join(self.workdir, context)
  71         if not os.path.exists(workdir):
  72             os.makedirs(workdir)
  73         if workdir not in sys.path:
  74             sys.path.insert(0, workdir)
  75
  76         expect_errors = (context == 'errors')
  77         suite = unittest.TestSuite()
  78         filenames = os.listdir(path)
  79         filenames.sort()
  80         for filename in filenames:
  81             if not filename.endswith(".pyx"):
  82                 continue
  83             module = filename[:-4]
  84             fqmodule = "%s.%s" % (context, module)
  85             if not [ 1 for match in self.selectors
  86                      if match(fqmodule) ]:
  87                 continue
  88             if context in TEST_RUN_DIRS:
  89                 test = CythonRunTestCase(
  90                     path, workdir, module,
  91                     annotate=self.annotate,
  92                     cleanup_workdir=self.cleanup_workdir)
  93             else:
  94                 test = CythonCompileTestCase(
  95                     path, workdir, module,
  96                     expect_errors=expect_errors,
  97                     annotate=self.annotate,
  98                     cleanup_workdir=self.cleanup_workdir)
  99             suite.addTest(test)
 100         return suite
 101
 102 class CythonCompileTestCase(unittest.TestCase):
 103     def __init__(self, directory, workdir, module,
 104                  expect_errors=False, annotate=False, cleanup_workdir=True):
 105         self.directory = directory
 106         self.workdir = workdir
 107         self.module = module
 108         self.expect_errors = expect_errors
 109         self.annotate = annotate
 110         self.cleanup_workdir = cleanup_workdir
 111         unittest.TestCase.__init__(self)
 112
 113     def shortDescription(self):
 114         return "compiling " + self.module
 115
 116     def tearDown(self):
 117         cleanup_c_files = WITH_CYTHON and self.cleanup_workdir
 118         if os.path.exists(self.workdir):
 119             for rmfile in os.listdir(self.workdir):
 120                 if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
 121                     continue
 122                 if self.annotate and rmfile.endswith(".html"):
 123                     continue
 124                 try:
 125                     rmfile = os.path.join(self.workdir, rmfile)
 126                     if os.path.isdir(rmfile):
 127                         shutil.rmtree(rmfile, ignore_errors=True)
 128                     else:
 129                         os.remove(rmfile)
 130                 except IOError:
 131                     pass
 132         else:
 133             os.makedirs(self.workdir)
 134
 135     def runTest(self):
 136         self.compile(self.directory, self.module, self.workdir,
 137                      self.directory, self.expect_errors, self.annotate)
 138
 139     def split_source_and_output(self, directory, module, workdir):
 140         source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
 141         out = open(os.path.join(workdir, module + '.pyx'), 'w')
 142         for line in source_and_output:
 143             last_line = line
 144             if line.startswith("_ERRORS"):
 145                 out.close()
 146                 out = ErrorWriter()
 147             else:
 148                 out.write(line)
 149         try:
 150             geterrors = out.geterrors
 151         except AttributeError:
 152             return []
 153         else:
 154             return geterrors()
 155
 156     def run_cython(self, directory, module, targetdir, incdir, annotate):
 157         include_dirs = INCLUDE_DIRS[:]
 158         if incdir:
 159             include_dirs.append(incdir)
 160         source = os.path.join(directory, module + '.pyx')
 161         target = os.path.join(targetdir, module + '.c')
 162         options = CompilationOptions(
 163             pyrex_default_options,
 164             include_path = include_dirs,
 165             output_file = target,
 166             annotate = annotate,
 167             use_listing_file = False, cplus = False, generate_pxi = False)
 168         cython_compile(source, options=options,
 169                        full_module_name=module)
 170
 171     def run_distutils(self, module, workdir, incdir):
 172         cwd = os.getcwd()
 173         os.chdir(workdir)
 174         try:
 175             build_extension = build_ext(distutils_distro)
 176             build_extension.include_dirs = INCLUDE_DIRS[:]
 177             if incdir:
 178                 build_extension.include_dirs.append(incdir)
 179             build_extension.finalize_options()
 180
 181             extension = Extension(
 182                 module,
 183                 sources = [module + '.c'],
 184                 extra_compile_args = CFLAGS,
 185                 )
 186             build_extension.extensions = [extension]
 187             build_extension.build_temp = workdir
 188             build_extension.build_lib  = workdir
 189             build_extension.run()
 190         finally:
 191             os.chdir(cwd)
 192
 193     def compile(self, directory, module, workdir, incdir,
 194                 expect_errors, annotate):
 195         expected_errors = errors = ()
 196         if expect_errors:
 197             expected_errors = self.split_source_and_output(
 198                 directory, module, workdir)
 199             directory = workdir
 200
 201         if WITH_CYTHON:
 202             old_stderr = sys.stderr
 203             try:
 204                 sys.stderr = ErrorWriter()
 205                 self.run_cython(directory, module, workdir, incdir, annotate)
 206                 errors = sys.stderr.geterrors()
 207             finally:
 208                 sys.stderr = old_stderr
 209
 210         if errors or expected_errors:
 211             for expected, error in zip(expected_errors, errors):
 212                 self.assertEquals(expected, error)
 213             if len(errors) < len(expected_errors):
 214                 expected_error = expected_errors[len(errors)]
 215                 self.assertEquals(expected_error, None)
 216             elif len(errors) > len(expected_errors):
 217                 unexpected_error = errors[len(expected_errors)]
 218                 self.assertEquals(None, unexpected_error)
 219         else:
 220             self.run_distutils(module, workdir, incdir)
 221
 222 class CythonRunTestCase(CythonCompileTestCase):
 223     def shortDescription(self):
 224         return "compiling and running " + self.module
 225
 226     def run(self, result=None):
 227         if result is None:
 228             result = self.defaultTestResult()
 229         result.startTest(self)
 230         try:
 231             self.runTest()
 232             doctest.DocTestSuite(self.module).run(result)
 233         except Exception:
 234             result.addError(self, sys.exc_info())
 235             result.stopTest(self)
 236         try:
 237             self.tearDown()
 238         except Exception:
 239             pass
 240
 241 def collect_unittests(path, suite, selectors):
 242     def file_matches(filename):
 243         return filename.startswith("Test") and filename.endswith(".py")
 244
 245     def package_matches(dirname):
 246         return dirname == "Tests"
 247
 248     loader = unittest.TestLoader()
 249
 250     for dirpath, dirnames, filenames in os.walk(path):
 251         parentname = os.path.split(dirpath)[-1]
 252         if package_matches(parentname):
 253             for f in filenames:
 254                 if file_matches(f):
 255                     filepath = os.path.join(dirpath, f)[:-len(".py")]
 256                     modulename = filepath[len(path)+1:].replace(os.path.sep, '.')
 257                     if not [ 1 for match in selectors if match(modulename) ]:
 258                         continue
 259                     module = __import__(modulename)
 260                     for x in modulename.split('.')[1:]:
 261                         module = getattr(module, x)
 262                     suite.addTests(loader.loadTestsFromModule(module))
 263
 264 if __name__ == '__main__':
 265     from optparse import OptionParser
 266     parser = OptionParser()
 267     parser.add_option("--no-cleanup", dest="cleanup_workdir",
 268                       action="store_false", default=True,
 269                       help="do not delete the generated C files (allows passing --no-cython on next run)")
 270     parser.add_option("--no-cython", dest="with_cython",
 271                       action="store_false", default=True,
 272                       help="do not run the Cython compiler, only the C compiler")
 273     parser.add_option("--no-unit", dest="unittests",
 274                       action="store_false", default=True,
 275                       help="do not run the unit tests")
 276     parser.add_option("--no-file", dest="filetests",
 277                       action="store_false", default=True,
 278                       help="do not run the file based tests")
 279     parser.add_option("-C", "--coverage", dest="coverage",
 280                       action="store_true", default=False,
 281                       help="collect source coverage data for the Compiler")
 282     parser.add_option("-A", "--annotate", dest="annotate_source",
 283                       action="store_true", default=False,
 284                       help="generate annotated HTML versions of the test source files")
 285     parser.add_option("-v", "--verbose", dest="verbosity",
 286                       action="count", default=0,
 287                       help="display test progress, pass twice to print test names")
 288
 289     options, cmd_args = parser.parse_args()
 290
 291     if options.coverage:
 292         import coverage
 293         coverage.erase()
 294         coverage.start()
 295
 296     WITH_CYTHON = options.with_cython
 297
 298     if WITH_CYTHON:
 299         from Cython.Compiler.Main import \
 300             CompilationOptions, \
 301             default_options as pyrex_default_options, \
 302             compile as cython_compile
 303         from Cython.Compiler import Errors
 304         Errors.LEVEL = 0 # show all warnings
 305
 306     # RUN ALL TESTS!
 307     ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
 308     WORKDIR = os.path.join(os.getcwd(), 'BUILD')
 309     if WITH_CYTHON:
 310         if os.path.exists(WORKDIR):
 311             shutil.rmtree(WORKDIR, ignore_errors=True)
 312     if not os.path.exists(WORKDIR):
 313         os.makedirs(WORKDIR)
 314
 315     if WITH_CYTHON:
 316         from Cython.Compiler.Version import version
 317         print("Running tests against Cython %s" % version)
 318     else:
 319         print("Running tests without Cython.")
 320     print("Python %s" % sys.version)
 321     print("")
 322
 323     import re
 324     selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
 325     if not selectors:
 326         selectors = [ lambda x:True ]
 327
 328     test_suite = unittest.TestSuite()
 329
 330     if options.unittests:
 331         collect_unittests(os.getcwd(), test_suite, selectors)
 332
 333     if options.filetests:
 334         filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
 335                                 options.annotate_source, options.cleanup_workdir)
 336         test_suite.addTests(filetests.build_suite())
 337
 338     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 339
 340     if options.coverage:
 341         coverage.stop()
 342         ignored_modules = ('Options', 'Version', 'DebugFlags')
 343         modules = [ module for name, module in sys.modules.items()
 344                     if module is not None and
 345                     name.startswith('Cython.Compiler.') and
 346                     name[len('Cython.Compiler.'):] not in ignored_modules ]
 347         coverage.report(modules, show_missing=0)