runtests.py

   1 #!/usr/bin/python
   2
   3 import os, sys, re, shutil, unittest, doctest
   4
   5 WITH_CYTHON = True
   6
   7 from distutils.dist import Distribution
   8 from distutils.core import Extension
   9 from distutils.command.build_ext import build_ext
  10 distutils_distro = Distribution()
  11
  12 TEST_DIRS = ['compile', 'errors', 'run', 'pyregr']
  13 TEST_RUN_DIRS = ['run', 'pyregr']
  14
  15 INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
  16 CFLAGS = os.getenv('CFLAGS', '').split()
  17
  18
  19 class ErrorWriter(object):
  20     match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
  21     def __init__(self):
  22         self.output = []
  23         self.write = self.output.append
  24
  25     def _collect(self, collect_errors, collect_warnings):
  26         s = ''.join(self.output)
  27         result = []
  28         for line in s.split('\n'):
  29             match = self.match_error(line)
  30             if match:
  31                 is_warning, line, column, message = match.groups()
  32                 if (is_warning and collect_warnings) or \
  33                         (not is_warning and collect_errors):
  34                     result.append( (int(line), int(column), message.strip()) )
  35         result.sort()
  36         return [ "%d:%d: %s" % values for values in result ]
  37
  38     def geterrors(self):
  39         return self._collect(True, False)
  40
  41     def getwarnings(self):
  42         return self._collect(False, True)
  43
  44     def getall(self):
  45         return self._collect(True, True)
  46
  47 class TestBuilder(object):
  48     def __init__(self, rootdir, workdir, selectors, annotate,
  49                  cleanup_workdir, cleanup_sharedlibs, with_pyregr, cythononly):
  50         self.rootdir = rootdir
  51         self.workdir = workdir
  52         self.selectors = selectors
  53         self.annotate = annotate
  54         self.cleanup_workdir = cleanup_workdir
  55         self.cleanup_sharedlibs = cleanup_sharedlibs
  56         self.with_pyregr = with_pyregr
  57         self.cythononly = cythononly
  58
  59     def build_suite(self):
  60         suite = unittest.TestSuite()
  61         filenames = os.listdir(self.rootdir)
  62         filenames.sort()
  63         for filename in filenames:
  64             if not WITH_CYTHON and filename == "errors":
  65                 # we won't get any errors without running Cython
  66                 continue
  67             path = os.path.join(self.rootdir, filename)
  68             if os.path.isdir(path) and filename in TEST_DIRS:
  69                 if filename == 'pyregr' and not self.with_pyregr:
  70                     continue
  71                 suite.addTest(
  72                     self.handle_directory(path, filename))
  73         return suite
  74
  75     def handle_directory(self, path, context):
  76         workdir = os.path.join(self.workdir, context)
  77         if not os.path.exists(workdir):
  78             os.makedirs(workdir)
  79         if workdir not in sys.path:
  80             sys.path.insert(0, workdir)
  81
  82         expect_errors = (context == 'errors')
  83         suite = unittest.TestSuite()
  84         filenames = os.listdir(path)
  85         filenames.sort()
  86         for filename in filenames:
  87             if not (filename.endswith(".pyx") or filename.endswith(".py")):
  88                 continue
  89             if filename.startswith('.'): continue # certain emacs backup files
  90             if context == 'pyregr' and not filename.startswith('test_'):
  91                 continue
  92             module = os.path.splitext(filename)[0]
  93             fqmodule = "%s.%s" % (context, module)
  94             if not [ 1 for match in self.selectors
  95                      if match(fqmodule) ]:
  96                 continue
  97             if context in TEST_RUN_DIRS:
  98                 if module.startswith("test_"):
  99                     build_test = CythonUnitTestCase
 100                 else:
 101                     build_test = CythonRunTestCase
 102                 test = build_test(
 103                     path, workdir, module,
 104                     annotate=self.annotate,
 105                     cleanup_workdir=self.cleanup_workdir,
 106                     cleanup_sharedlibs=self.cleanup_sharedlibs,
 107                     cythononly=self.cythononly)
 108             else:
 109                 test = CythonCompileTestCase(
 110                     path, workdir, module,
 111                     expect_errors=expect_errors,
 112                     annotate=self.annotate,
 113                     cleanup_workdir=self.cleanup_workdir,
 114                     cleanup_sharedlibs=self.cleanup_sharedlibs,
 115                     cythononly=self.cythononly)
 116             suite.addTest(test)
 117         return suite
 118
 119 class CythonCompileTestCase(unittest.TestCase):
 120     def __init__(self, directory, workdir, module,
 121                  expect_errors=False, annotate=False, cleanup_workdir=True,
 122                  cleanup_sharedlibs=True, cythononly=False):
 123         self.directory = directory
 124         self.workdir = workdir
 125         self.module = module
 126         self.expect_errors = expect_errors
 127         self.annotate = annotate
 128         self.cleanup_workdir = cleanup_workdir
 129         self.cleanup_sharedlibs = cleanup_sharedlibs
 130         self.cythononly = cythononly
 131         unittest.TestCase.__init__(self)
 132
 133     def shortDescription(self):
 134         return "compiling " + self.module
 135
 136     def tearDown(self):
 137         cleanup_c_files = WITH_CYTHON and self.cleanup_workdir
 138         cleanup_lib_files = self.cleanup_sharedlibs
 139         if os.path.exists(self.workdir):
 140             for rmfile in os.listdir(self.workdir):
 141                 if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
 142                     continue
 143                 if not cleanup_lib_files and rmfile.endswith(".so") or rmfile.endswith(".dll"):
 144                     continue
 145                 if self.annotate and rmfile.endswith(".html"):
 146                     continue
 147                 try:
 148                     rmfile = os.path.join(self.workdir, rmfile)
 149                     if os.path.isdir(rmfile):
 150                         shutil.rmtree(rmfile, ignore_errors=True)
 151                     else:
 152                         os.remove(rmfile)
 153                 except IOError:
 154                     pass
 155         else:
 156             os.makedirs(self.workdir)
 157
 158     def runTest(self):
 159         self.runCompileTest()
 160
 161     def runCompileTest(self):
 162         self.compile(self.directory, self.module, self.workdir,
 163                      self.directory, self.expect_errors, self.annotate)
 164
 165     def find_module_source_file(self, source_file):
 166         if not os.path.exists(source_file):
 167             source_file = source_file[:-1]
 168         return source_file
 169
 170     def split_source_and_output(self, directory, module, workdir):
 171         source_file = os.path.join(directory, module) + '.pyx'
 172         source_and_output = open(
 173             self.find_module_source_file(source_file), 'rU')
 174         out = open(os.path.join(workdir, module + '.pyx'), 'w')
 175         for line in source_and_output:
 176             last_line = line
 177             if line.startswith("_ERRORS"):
 178                 out.close()
 179                 out = ErrorWriter()
 180             else:
 181                 out.write(line)
 182         try:
 183             geterrors = out.geterrors
 184         except AttributeError:
 185             return []
 186         else:
 187             return geterrors()
 188
 189     def run_cython(self, directory, module, targetdir, incdir, annotate):
 190         include_dirs = INCLUDE_DIRS[:]
 191         if incdir:
 192             include_dirs.append(incdir)
 193         source = self.find_module_source_file(
 194             os.path.join(directory, module + '.pyx'))
 195         target = os.path.join(targetdir, module + '.c')
 196         options = CompilationOptions(
 197             pyrex_default_options,
 198             include_path = include_dirs,
 199             output_file = target,
 200             annotate = annotate,
 201             use_listing_file = False, cplus = False, generate_pxi = False)
 202         cython_compile(source, options=options,
 203                        full_module_name=module)
 204
 205     def run_distutils(self, module, workdir, incdir):
 206         cwd = os.getcwd()
 207         os.chdir(workdir)
 208         try:
 209             build_extension = build_ext(distutils_distro)
 210             build_extension.include_dirs = INCLUDE_DIRS[:]
 211             if incdir:
 212                 build_extension.include_dirs.append(incdir)
 213             build_extension.finalize_options()
 214
 215             extension = Extension(
 216                 module,
 217                 sources = [module + '.c'],
 218                 extra_compile_args = CFLAGS,
 219                 )
 220             build_extension.extensions = [extension]
 221             build_extension.build_temp = workdir
 222             build_extension.build_lib  = workdir
 223             build_extension.run()
 224         finally:
 225             os.chdir(cwd)
 226
 227     def compile(self, directory, module, workdir, incdir,
 228                 expect_errors, annotate):
 229         expected_errors = errors = ()
 230         if expect_errors:
 231             expected_errors = self.split_source_and_output(
 232                 directory, module, workdir)
 233             directory = workdir
 234
 235         if WITH_CYTHON:
 236             old_stderr = sys.stderr
 237             try:
 238                 sys.stderr = ErrorWriter()
 239                 self.run_cython(directory, module, workdir, incdir, annotate)
 240                 errors = sys.stderr.geterrors()
 241             finally:
 242                 sys.stderr = old_stderr
 243
 244         if errors or expected_errors:
 245             for expected, error in zip(expected_errors, errors):
 246                 self.assertEquals(expected, error)
 247             if len(errors) < len(expected_errors):
 248                 expected_error = expected_errors[len(errors)]
 249                 self.assertEquals(expected_error, None)
 250             elif len(errors) > len(expected_errors):
 251                 unexpected_error = errors[len(expected_errors)]
 252                 self.assertEquals(None, unexpected_error)
 253         else:
 254             if not self.cythononly:
 255                 self.run_distutils(module, workdir, incdir)
 256
 257 class CythonRunTestCase(CythonCompileTestCase):
 258     def shortDescription(self):
 259         return "compiling and running " + self.module
 260
 261     def run(self, result=None):
 262         if result is None:
 263             result = self.defaultTestResult()
 264         result.startTest(self)
 265         try:
 266             self.runCompileTest()
 267             if not self.cythononly:
 268                 sys.stderr.write('running doctests in %s ...\n' % self.module)
 269                 doctest.DocTestSuite(self.module).run(result)
 270         except Exception:
 271             result.addError(self, sys.exc_info())
 272             result.stopTest(self)
 273         try:
 274             self.tearDown()
 275         except Exception:
 276             pass
 277
 278 class CythonUnitTestCase(CythonCompileTestCase):
 279     def shortDescription(self):
 280         return "compiling tests in " + self.module
 281
 282     def run(self, result=None):
 283         if result is None:
 284             result = self.defaultTestResult()
 285         result.startTest(self)
 286         try:
 287             self.runCompileTest()
 288             sys.stderr.write('running tests in %s ...\n' % self.module)
 289             unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
 290         except Exception:
 291             result.addError(self, sys.exc_info())
 292             result.stopTest(self)
 293         try:
 294             self.tearDown()
 295         except Exception:
 296             pass
 297
 298 def collect_unittests(path, module_prefix, suite, selectors):
 299     def file_matches(filename):
 300         return filename.startswith("Test") and filename.endswith(".py")
 301
 302     def package_matches(dirname):
 303         return dirname == "Tests"
 304
 305     loader = unittest.TestLoader()
 306
 307     skipped_dirs = []
 308
 309     for dirpath, dirnames, filenames in os.walk(path):
 310         if dirpath != path and "__init__.py" not in filenames:
 311             skipped_dirs.append(dirpath + os.path.sep)
 312             continue
 313         skip = False
 314         for dir in skipped_dirs:
 315             if dirpath.startswith(dir):
 316                 skip = True
 317         if skip:
 318             continue
 319         parentname = os.path.split(dirpath)[-1]
 320         if package_matches(parentname):
 321             for f in filenames:
 322                 if file_matches(f):
 323                     filepath = os.path.join(dirpath, f)[:-len(".py")]
 324                     modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
 325                     if not [ 1 for match in selectors if match(modulename) ]:
 326                         continue
 327                     module = __import__(modulename)
 328                     for x in modulename.split('.')[1:]:
 329                         module = getattr(module, x)
 330                     suite.addTests([loader.loadTestsFromModule(module)])
 331
 332 def collect_doctests(path, module_prefix, suite, selectors):
 333     def package_matches(dirname):
 334         return dirname not in ("Mac", "Distutils", "Plex")
 335     def file_matches(filename):
 336         return (filename.endswith(".py") and not ('~' in filename
 337                 or '#' in filename or filename.startswith('.')))
 338     import doctest, types
 339     for dirpath, dirnames, filenames in os.walk(path):
 340         parentname = os.path.split(dirpath)[-1]
 341         if package_matches(parentname):
 342             for f in filenames:
 343                 if file_matches(f):
 344                     if not f.endswith('.py'): continue
 345                     filepath = os.path.join(dirpath, f)[:-len(".py")]
 346                     modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
 347                     if not [ 1 for match in selectors if match(modulename) ]:
 348                         continue
 349                     module = __import__(modulename)
 350                     for x in modulename.split('.')[1:]:
 351                         module = getattr(module, x)
 352                     if hasattr(module, "__doc__") or hasattr(module, "__test__"):
 353                         try:
 354                             suite.addTest(doctest.DocTestSuite(module))
 355                         except ValueError: # no tests
 356                             pass
 357
 358 if __name__ == '__main__':
 359     from optparse import OptionParser
 360     parser = OptionParser()
 361     parser.add_option("--no-cleanup", dest="cleanup_workdir",
 362                       action="store_false", default=True,
 363                       help="do not delete the generated C files (allows passing --no-cython on next run)")
 364     parser.add_option("--no-cleanup-sharedlibs", dest="cleanup_sharedlibs",
 365                       action="store_false", default=True,
 366                       help="do not delete the generated shared libary files (allows manual module experimentation)")
 367     parser.add_option("--no-cython", dest="with_cython",
 368                       action="store_false", default=True,
 369                       help="do not run the Cython compiler, only the C compiler")
 370     parser.add_option("--no-unit", dest="unittests",
 371                       action="store_false", default=True,
 372                       help="do not run the unit tests")
 373     parser.add_option("--no-doctest", dest="doctests",
 374                       action="store_false", default=True,
 375                       help="do not run the doctests")
 376     parser.add_option("--no-file", dest="filetests",
 377                       action="store_false", default=True,
 378                       help="do not run the file based tests")
 379     parser.add_option("--no-pyregr", dest="pyregr",
 380                       action="store_false", default=True,
 381                       help="do not run the regression tests of CPython in tests/pyregr/")
 382     parser.add_option("--cython-only", dest="cythononly",
 383                       action="store_true", default=False,
 384                       help="only compile pyx to c, do not run C compiler or run the tests")
 385     parser.add_option("--sys-pyregr", dest="system_pyregr",
 386                       action="store_true", default=False,
 387                       help="run the regression tests of the CPython installation")
 388     parser.add_option("-C", "--coverage", dest="coverage",
 389                       action="store_true", default=False,
 390                       help="collect source coverage data for the Compiler")
 391     parser.add_option("-A", "--annotate", dest="annotate_source",
 392                       action="store_true", default=False,
 393                       help="generate annotated HTML versions of the test source files")
 394     parser.add_option("-v", "--verbose", dest="verbosity",
 395                       action="count", default=0,
 396                       help="display test progress, pass twice to print test names")
 397
 398     options, cmd_args = parser.parse_args()
 399
 400     if sys.version_info[0] >= 3:
 401         # make sure we do not import (or run) Cython itself
 402         options.doctests    = False
 403         options.with_cython = False
 404         options.unittests   = False
 405         options.pyregr      = False
 406
 407     if options.coverage:
 408         import coverage
 409         coverage.erase()
 410         coverage.start()
 411
 412     WITH_CYTHON = options.with_cython
 413
 414     if WITH_CYTHON:
 415         from Cython.Compiler.Main import \
 416             CompilationOptions, \
 417             default_options as pyrex_default_options, \
 418             compile as cython_compile
 419         from Cython.Compiler import Errors
 420         Errors.LEVEL = 0 # show all warnings
 421
 422     # RUN ALL TESTS!
 423     ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
 424     WORKDIR = os.path.join(os.getcwd(), 'BUILD')
 425     UNITTEST_MODULE = "Cython"
 426     UNITTEST_ROOT = os.path.join(os.getcwd(), UNITTEST_MODULE)
 427     if WITH_CYTHON:
 428         if os.path.exists(WORKDIR):
 429             shutil.rmtree(WORKDIR, ignore_errors=True)
 430     if not os.path.exists(WORKDIR):
 431         os.makedirs(WORKDIR)
 432
 433     if WITH_CYTHON:
 434         from Cython.Compiler.Version import version
 435         sys.stderr.write("Running tests against Cython %s\n" % version)
 436     else:
 437         sys.stderr.write("Running tests without Cython.\n")
 438     sys.stderr.write("Python %s\n" % sys.version)
 439     sys.stderr.write("\n")
 440
 441     import re
 442     selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
 443     if not selectors:
 444         selectors = [ lambda x:True ]
 445
 446     test_suite = unittest.TestSuite()
 447
 448     if options.unittests:
 449         collect_unittests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors)
 450
 451     if options.doctests:
 452         collect_doctests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors)
 453
 454     if options.filetests:
 455         filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
 456                                 options.annotate_source, options.cleanup_workdir,
 457                                 options.cleanup_sharedlibs, options.pyregr,
 458                                 options.cythononly)
 459         test_suite.addTest(filetests.build_suite())
 460
 461     if options.system_pyregr:
 462         filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
 463                                 options.annotate_source, options.cleanup_workdir,
 464                                 options.cleanup_sharedlibs, True,
 465                                 options.cythononly)
 466         test_suite.addTest(
 467             filetests.handle_directory(
 468                 os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test'),
 469                 'pyregr'))
 470
 471     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 472
 473     if options.coverage:
 474         coverage.stop()
 475         ignored_modules = ('Options', 'Version', 'DebugFlags')
 476         modules = [ module for name, module in sys.modules.items()
 477                     if module is not None and
 478                     name.startswith('Cython.Compiler.') and
 479                     name[len('Cython.Compiler.'):] not in ignored_modules ]
 480         coverage.report(modules, show_missing=0)