runtests.py

   1 #!/usr/bin/python
   2
   3 import os, sys, re, shutil, unittest, doctest
   4
   5 WITH_CYTHON = True
   6
   7 from distutils.dist import Distribution
   8 from distutils.core import Extension
   9 from distutils.command.build_ext import build_ext
  10 distutils_distro = Distribution()
  11
  12 TEST_DIRS = ['compile', 'errors', 'run', 'pyregr']
  13 TEST_RUN_DIRS = ['run', 'pyregr']
  14
  15 INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
  16 CFLAGS = os.getenv('CFLAGS', '').split()
  17
  18
  19 class ErrorWriter(object):
  20     match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
  21     def __init__(self):
  22         self.output = []
  23         self.write = self.output.append
  24
  25     def _collect(self, collect_errors, collect_warnings):
  26         s = ''.join(self.output)
  27         result = []
  28         for line in s.split('\n'):
  29             match = self.match_error(line)
  30             if match:
  31                 is_warning, line, column, message = match.groups()
  32                 if (is_warning and collect_warnings) or \
  33                         (not is_warning and collect_errors):
  34                     result.append( (int(line), int(column), message.strip()) )
  35         result.sort()
  36         return [ "%d:%d: %s" % values for values in result ]
  37
  38     def geterrors(self):
  39         return self._collect(True, False)
  40
  41     def getwarnings(self):
  42         return self._collect(False, True)
  43
  44     def getall(self):
  45         return self._collect(True, True)
  46
  47 class TestBuilder(object):
  48     def __init__(self, rootdir, workdir, selectors, annotate,
  49                  cleanup_workdir, cleanup_sharedlibs, with_pyregr):
  50         self.rootdir = rootdir
  51         self.workdir = workdir
  52         self.selectors = selectors
  53         self.annotate = annotate
  54         self.cleanup_workdir = cleanup_workdir
  55         self.cleanup_sharedlibs = cleanup_sharedlibs
  56         self.with_pyregr = with_pyregr
  57
  58     def build_suite(self):
  59         suite = unittest.TestSuite()
  60         filenames = os.listdir(self.rootdir)
  61         filenames.sort()
  62         for filename in filenames:
  63             if not WITH_CYTHON and filename == "errors":
  64                 # we won't get any errors without running Cython
  65                 continue
  66             path = os.path.join(self.rootdir, filename)
  67             if os.path.isdir(path) and filename in TEST_DIRS:
  68                 if filename == 'pyregr' and not self.with_pyregr:
  69                     continue
  70                 suite.addTest(
  71                     self.handle_directory(path, filename))
  72         return suite
  73
  74     def handle_directory(self, path, context):
  75         workdir = os.path.join(self.workdir, context)
  76         if not os.path.exists(workdir):
  77             os.makedirs(workdir)
  78         if workdir not in sys.path:
  79             sys.path.insert(0, workdir)
  80
  81         expect_errors = (context == 'errors')
  82         suite = unittest.TestSuite()
  83         filenames = os.listdir(path)
  84         filenames.sort()
  85         for filename in filenames:
  86             if not (filename.endswith(".pyx") or filename.endswith(".py")):
  87                 continue
  88             if context == 'pyregr' and not filename.startswith('test_'):
  89                 continue
  90             module = os.path.splitext(filename)[0]
  91             fqmodule = "%s.%s" % (context, module)
  92             if not [ 1 for match in self.selectors
  93                      if match(fqmodule) ]:
  94                 continue
  95             if context in TEST_RUN_DIRS:
  96                 if module.startswith("test_"):
  97                     build_test = CythonUnitTestCase
  98                 else:
  99                     build_test = CythonRunTestCase
 100                 test = build_test(
 101                     path, workdir, module,
 102                     annotate=self.annotate,
 103                     cleanup_workdir=self.cleanup_workdir,
 104                     cleanup_sharedlibs=self.cleanup_sharedlibs)
 105             else:
 106                 test = CythonCompileTestCase(
 107                     path, workdir, module,
 108                     expect_errors=expect_errors,
 109                     annotate=self.annotate,
 110                     cleanup_workdir=self.cleanup_workdir,
 111                     cleanup_sharedlibs=self.cleanup_sharedlibs)
 112             suite.addTest(test)
 113         return suite
 114
 115 class CythonCompileTestCase(unittest.TestCase):
 116     def __init__(self, directory, workdir, module,
 117                  expect_errors=False, annotate=False, cleanup_workdir=True,
 118                  cleanup_sharedlibs=True):
 119         self.directory = directory
 120         self.workdir = workdir
 121         self.module = module
 122         self.expect_errors = expect_errors
 123         self.annotate = annotate
 124         self.cleanup_workdir = cleanup_workdir
 125         self.cleanup_sharedlibs = cleanup_sharedlibs
 126         unittest.TestCase.__init__(self)
 127
 128     def shortDescription(self):
 129         return "compiling " + self.module
 130
 131     def tearDown(self):
 132         cleanup_c_files = WITH_CYTHON and self.cleanup_workdir
 133         cleanup_lib_files = self.cleanup_sharedlibs
 134         if os.path.exists(self.workdir):
 135             for rmfile in os.listdir(self.workdir):
 136                 if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
 137                     continue
 138                 if not cleanup_lib_files and rmfile.endswith(".so") or rmfile.endswith(".dll"):
 139                     continue
 140                 if self.annotate and rmfile.endswith(".html"):
 141                     continue
 142                 try:
 143                     rmfile = os.path.join(self.workdir, rmfile)
 144                     if os.path.isdir(rmfile):
 145                         shutil.rmtree(rmfile, ignore_errors=True)
 146                     else:
 147                         os.remove(rmfile)
 148                 except IOError:
 149                     pass
 150         else:
 151             os.makedirs(self.workdir)
 152
 153     def runTest(self):
 154         self.runCompileTest()
 155
 156     def runCompileTest(self):
 157         self.compile(self.directory, self.module, self.workdir,
 158                      self.directory, self.expect_errors, self.annotate)
 159
 160     def find_module_source_file(self, source_file):
 161         if not os.path.exists(source_file):
 162             source_file = source_file[:-1]
 163         return source_file
 164
 165     def split_source_and_output(self, directory, module, workdir):
 166         source_file = os.path.join(directory, module) + '.pyx'
 167         source_and_output = open(
 168             self.find_module_source_file(source_file), 'rU')
 169         out = open(os.path.join(workdir, module + '.pyx'), 'w')
 170         for line in source_and_output:
 171             last_line = line
 172             if line.startswith("_ERRORS"):
 173                 out.close()
 174                 out = ErrorWriter()
 175             else:
 176                 out.write(line)
 177         try:
 178             geterrors = out.geterrors
 179         except AttributeError:
 180             return []
 181         else:
 182             return geterrors()
 183
 184     def run_cython(self, directory, module, targetdir, incdir, annotate):
 185         include_dirs = INCLUDE_DIRS[:]
 186         if incdir:
 187             include_dirs.append(incdir)
 188         source = self.find_module_source_file(
 189             os.path.join(directory, module + '.pyx'))
 190         target = os.path.join(targetdir, module + '.c')
 191         options = CompilationOptions(
 192             pyrex_default_options,
 193             include_path = include_dirs,
 194             output_file = target,
 195             annotate = annotate,
 196             use_listing_file = False, cplus = False, generate_pxi = False)
 197         cython_compile(source, options=options,
 198                        full_module_name=module)
 199
 200     def run_distutils(self, module, workdir, incdir):
 201         cwd = os.getcwd()
 202         os.chdir(workdir)
 203         try:
 204             build_extension = build_ext(distutils_distro)
 205             build_extension.include_dirs = INCLUDE_DIRS[:]
 206             if incdir:
 207                 build_extension.include_dirs.append(incdir)
 208             build_extension.finalize_options()
 209
 210             extension = Extension(
 211                 module,
 212                 sources = [module + '.c'],
 213                 extra_compile_args = CFLAGS,
 214                 )
 215             build_extension.extensions = [extension]
 216             build_extension.build_temp = workdir
 217             build_extension.build_lib  = workdir
 218             build_extension.run()
 219         finally:
 220             os.chdir(cwd)
 221
 222     def compile(self, directory, module, workdir, incdir,
 223                 expect_errors, annotate):
 224         expected_errors = errors = ()
 225         if expect_errors:
 226             expected_errors = self.split_source_and_output(
 227                 directory, module, workdir)
 228             directory = workdir
 229
 230         if WITH_CYTHON:
 231             old_stderr = sys.stderr
 232             try:
 233                 sys.stderr = ErrorWriter()
 234                 self.run_cython(directory, module, workdir, incdir, annotate)
 235                 errors = sys.stderr.geterrors()
 236             finally:
 237                 sys.stderr = old_stderr
 238
 239         if errors or expected_errors:
 240             for expected, error in zip(expected_errors, errors):
 241                 self.assertEquals(expected, error)
 242             if len(errors) < len(expected_errors):
 243                 expected_error = expected_errors[len(errors)]
 244                 self.assertEquals(expected_error, None)
 245             elif len(errors) > len(expected_errors):
 246                 unexpected_error = errors[len(expected_errors)]
 247                 self.assertEquals(None, unexpected_error)
 248         else:
 249             self.run_distutils(module, workdir, incdir)
 250
 251 class CythonRunTestCase(CythonCompileTestCase):
 252     def shortDescription(self):
 253         return "compiling and running " + self.module
 254
 255     def run(self, result=None):
 256         if result is None:
 257             result = self.defaultTestResult()
 258         result.startTest(self)
 259         try:
 260             self.runCompileTest()
 261             doctest.DocTestSuite(self.module).run(result)
 262         except Exception:
 263             result.addError(self, sys.exc_info())
 264             result.stopTest(self)
 265         try:
 266             self.tearDown()
 267         except Exception:
 268             pass
 269
 270 class CythonUnitTestCase(CythonCompileTestCase):
 271     def shortDescription(self):
 272         return "compiling and running unit tests in " + self.module
 273
 274     def run(self, result=None):
 275         if result is None:
 276             result = self.defaultTestResult()
 277         result.startTest(self)
 278         try:
 279             self.runCompileTest()
 280             unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
 281         except Exception:
 282             result.addError(self, sys.exc_info())
 283             result.stopTest(self)
 284         try:
 285             self.tearDown()
 286         except Exception:
 287             pass
 288
 289 def collect_unittests(path, suite, selectors):
 290     def file_matches(filename):
 291         return filename.startswith("Test") and filename.endswith(".py")
 292
 293     def package_matches(dirname):
 294         return dirname == "Tests"
 295
 296     loader = unittest.TestLoader()
 297
 298     for dirpath, dirnames, filenames in os.walk(path):
 299         parentname = os.path.split(dirpath)[-1]
 300         if package_matches(parentname):
 301             for f in filenames:
 302                 if file_matches(f):
 303                     filepath = os.path.join(dirpath, f)[:-len(".py")]
 304                     modulename = filepath[len(path)+1:].replace(os.path.sep, '.')
 305                     if not [ 1 for match in selectors if match(modulename) ]:
 306                         continue
 307                     module = __import__(modulename)
 308                     for x in modulename.split('.')[1:]:
 309                         module = getattr(module, x)
 310                     suite.addTests([loader.loadTestsFromModule(module)])
 311
 312 if __name__ == '__main__':
 313     from optparse import OptionParser
 314     parser = OptionParser()
 315     parser.add_option("--no-cleanup", dest="cleanup_workdir",
 316                       action="store_false", default=True,
 317                       help="do not delete the generated C files (allows passing --no-cython on next run)")
 318     parser.add_option("--no-cleanup-sharedlibs", dest="cleanup_sharedlibs",
 319                       action="store_false", default=True,
 320                       help="do not delete the generated shared libary files (allows manual module experimentation)")
 321     parser.add_option("--no-cython", dest="with_cython",
 322                       action="store_false", default=True,
 323                       help="do not run the Cython compiler, only the C compiler")
 324     parser.add_option("--no-unit", dest="unittests",
 325                       action="store_false", default=True,
 326                       help="do not run the unit tests")
 327     parser.add_option("--no-file", dest="filetests",
 328                       action="store_false", default=True,
 329                       help="do not run the file based tests")
 330     parser.add_option("--no-pyregr", dest="pyregr",
 331                       action="store_false", default=True,
 332                       help="do not run the regression tests of CPython in tests/pyregr/")
 333     parser.add_option("-C", "--coverage", dest="coverage",
 334                       action="store_true", default=False,
 335                       help="collect source coverage data for the Compiler")
 336     parser.add_option("-A", "--annotate", dest="annotate_source",
 337                       action="store_true", default=False,
 338                       help="generate annotated HTML versions of the test source files")
 339     parser.add_option("-v", "--verbose", dest="verbosity",
 340                       action="count", default=0,
 341                       help="display test progress, pass twice to print test names")
 342
 343     options, cmd_args = parser.parse_args()
 344
 345     if options.coverage:
 346         import coverage
 347         coverage.erase()
 348         coverage.start()
 349
 350     WITH_CYTHON = options.with_cython
 351
 352     if WITH_CYTHON:
 353         from Cython.Compiler.Main import \
 354             CompilationOptions, \
 355             default_options as pyrex_default_options, \
 356             compile as cython_compile
 357         from Cython.Compiler import Errors
 358         Errors.LEVEL = 0 # show all warnings
 359
 360     # RUN ALL TESTS!
 361     ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
 362     WORKDIR = os.path.join(os.getcwd(), 'BUILD')
 363     if WITH_CYTHON:
 364         if os.path.exists(WORKDIR):
 365             shutil.rmtree(WORKDIR, ignore_errors=True)
 366     if not os.path.exists(WORKDIR):
 367         os.makedirs(WORKDIR)
 368
 369     if WITH_CYTHON:
 370         from Cython.Compiler.Version import version
 371         print("Running tests against Cython %s" % version)
 372     else:
 373         print("Running tests without Cython.")
 374     print("Python %s" % sys.version)
 375     print("")
 376
 377     import re
 378     selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
 379     if not selectors:
 380         selectors = [ lambda x:True ]
 381
 382     test_suite = unittest.TestSuite()
 383
 384     if options.unittests:
 385         collect_unittests(os.getcwd(), test_suite, selectors)
 386
 387     if options.filetests:
 388         filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
 389                                 options.annotate_source, options.cleanup_workdir,
 390                                 options.cleanup_sharedlibs, options.pyregr)
 391         test_suite.addTests([filetests.build_suite()])
 392
 393     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 394
 395     if options.coverage:
 396         coverage.stop()
 397         ignored_modules = ('Options', 'Version', 'DebugFlags')
 398         modules = [ module for name, module in sys.modules.items()
 399                     if module is not None and
 400                     name.startswith('Cython.Compiler.') and
 401                     name[len('Cython.Compiler.'):] not in ignored_modules ]
 402         coverage.report(modules, show_missing=0)