support HTML coverage reports
[cython.git] / runtests.py
index c68958f6eb707413d68f681df2ad5b2b98518fc7..eb2d340b85183778743fe15c9a184f0b78f05490 100644 (file)
@@ -94,16 +94,22 @@ VER_DEP_MODULES = {
     (2,5) : (operator.lt, lambda x: x in ['run.any',
                                           'run.all',
                                           'run.relativeimport_T542',
+                                          'run.relativeimport_star_T542',
                                           ]),
     (2,6) : (operator.lt, lambda x: x in ['run.print_function',
                                           'run.cython3',
+                                          'run.generators_py', # generators, with statement
                                           'run.pure_py', # decorators, with statement
                                           ]),
+    (2,7) : (operator.lt, lambda x: x in ['run.withstat_py', # multi context with statement
+                                          ]),
     # The next line should start (3,); but this is a dictionary, so
     # we can only have one (3,) key.  Since 2.7 is supposed to be the
     # last 2.x release, things would have to change drastically for this
     # to be unsafe...
-    (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3']),
+    (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3',
+                                           'run.test_raisefrom',
+                                           ]),
     (3,): (operator.ge, lambda x: x in ['run.non_future_division',
                                         'compile.extsetslice',
                                         'compile.extdelslice',
@@ -351,6 +357,8 @@ class CythonCompileTestCase(unittest.TestCase):
 
     def setUp(self):
         from Cython.Compiler import Options
+        self._saved_options = [ (name, getattr(Options, name))
+                                for name in ('warning_errors', 'error_on_unknown_names') ]
         Options.warning_errors = self.warning_errors
 
         if self.workdir not in sys.path:
@@ -358,7 +366,8 @@ class CythonCompileTestCase(unittest.TestCase):
 
     def tearDown(self):
         from Cython.Compiler import Options
-        Options.warning_errors = False
+        for name, value in self._saved_options:
+            setattr(Options, name, value)
 
         try:
             sys.path.remove(self.workdir)
@@ -581,64 +590,70 @@ class CythonRunTestCase(CythonCompileTestCase):
             self.run_doctests(self.module, result)
 
     def run_doctests(self, module_name, result):
-        if sys.version_info[0] >= 3 or not hasattr(os, 'fork') or not self.fork:
-            doctest.DocTestSuite(module_name).run(result)
-            gc.collect()
-            return
-
-        # fork to make sure we do not keep the tested module loaded
-        result_handle, result_file = tempfile.mkstemp()
-        os.close(result_handle)
-        child_id = os.fork()
-        if not child_id:
-            result_code = 0
-            try:
-                try:
-                    tests = None
-                    try:
-                        partial_result = PartialTestResult(result)
-                        tests = doctest.DocTestSuite(module_name)
-                        tests.run(partial_result)
-                        gc.collect()
-                    except Exception:
-                        if tests is None:
-                            # importing failed, try to fake a test class
-                            tests = _FakeClass(
-                                failureException=sys.exc_info()[1],
-                                _shortDescription=self.shortDescription(),
-                                module_name=None)
-                        partial_result.addError(tests, sys.exc_info())
-                        result_code = 1
-                    output = open(result_file, 'wb')
-                    pickle.dump(partial_result.data(), output)
-                except:
-                    traceback.print_exc()
-            finally:
-                try: output.close()
-                except: pass
-                os._exit(result_code)
+        def run_test(result):
+            tests = doctest.DocTestSuite(module_name)
+            tests.run(result)
+        run_forked_test(result, run_test, self.shortDescription(), self.fork)
+
 
+def run_forked_test(result, run_func, test_name, fork=True):
+    if not fork or sys.version_info[0] >= 3 or not hasattr(os, 'fork'):
+        run_func(result)
+        gc.collect()
+        return
+
+    # fork to make sure we do not keep the tested module loaded
+    result_handle, result_file = tempfile.mkstemp()
+    os.close(result_handle)
+    child_id = os.fork()
+    if not child_id:
+        result_code = 0
         try:
-            cid, result_code = os.waitpid(child_id, 0)
-            # os.waitpid returns the child's result code in the
-            # upper byte of result_code, and the signal it was
-            # killed by in the lower byte
-            if result_code & 255:
-                raise Exception("Tests in module '%s' were unexpectedly killed by signal %d"%
-                                (module_name, result_code & 255))
-            result_code = result_code >> 8
-            if result_code in (0,1):
-                input = open(result_file, 'rb')
+            try:
+                tests = None
                 try:
-                    PartialTestResult.join_results(result, pickle.load(input))
-                finally:
-                    input.close()
-            if result_code:
-                raise Exception("Tests in module '%s' exited with status %d" %
-                                (module_name, result_code))
+                    partial_result = PartialTestResult(result)
+                    run_func(partial_result)
+                    gc.collect()
+                except Exception:
+                    if tests is None:
+                        # importing failed, try to fake a test class
+                        tests = _FakeClass(
+                            failureException=sys.exc_info()[1],
+                            _shortDescription=test_name,
+                            module_name=None)
+                    partial_result.addError(tests, sys.exc_info())
+                    result_code = 1
+                output = open(result_file, 'wb')
+                pickle.dump(partial_result.data(), output)
+            except:
+                traceback.print_exc()
         finally:
-            try: os.unlink(result_file)
+            try: output.close()
             except: pass
+            os._exit(result_code)
+
+    try:
+        cid, result_code = os.waitpid(child_id, 0)
+        # os.waitpid returns the child's result code in the
+        # upper byte of result_code, and the signal it was
+        # killed by in the lower byte
+        if result_code & 255:
+            raise Exception("Tests in module '%s' were unexpectedly killed by signal %d"%
+                            (module_name, result_code & 255))
+        result_code = result_code >> 8
+        if result_code in (0,1):
+            input = open(result_file, 'rb')
+            try:
+                PartialTestResult.join_results(result, pickle.load(input))
+            finally:
+                input.close()
+        if result_code:
+            raise Exception("Tests in module '%s' exited with status %d" %
+                            (module_name, result_code))
+    finally:
+        try: os.unlink(result_file)
+        except: pass
 
 class PureDoctestTestCase(unittest.TestCase):
     def __init__(self, module_name, module_path):
@@ -736,6 +751,11 @@ class CythonUnitTestCase(CythonRunTestCase):
 
 
 class CythonPyregrTestCase(CythonRunTestCase):
+    def setUp(self):
+        CythonRunTestCase.setUp(self)
+        from Cython.Compiler import Options
+        Options.error_on_unknown_names = False
+
     def _run_unittest(self, result, *classes):
         """Run tests from unittest.TestCase-derived classes."""
         valid_types = (unittest.TestSuite, unittest.TestCase)
@@ -761,20 +781,23 @@ class CythonPyregrTestCase(CythonRunTestCase):
         except ImportError: # Py3k
             from test import support
 
-        def run_unittest(*classes):
-            return self._run_unittest(result, *classes)
-        def run_doctest(module, verbosity=None):
-            return self._run_doctest(result, module)
+        def run_test(result):
+            def run_unittest(*classes):
+                return self._run_unittest(result, *classes)
+            def run_doctest(module, verbosity=None):
+                return self._run_doctest(result, module)
 
-        support.run_unittest = run_unittest
-        support.run_doctest = run_doctest
+            support.run_unittest = run_unittest
+            support.run_doctest = run_doctest
 
-        try:
-            module = __import__(self.module)
-            if hasattr(module, 'test_main'):
-                module.test_main()
-        except (unittest.SkipTest, support.ResourceDenied):
-            result.addSkip(self, 'ok')
+            try:
+                module = __import__(self.module)
+                if hasattr(module, 'test_main'):
+                    module.test_main()
+            except (unittest.SkipTest, support.ResourceDenied):
+                result.addSkip(self, 'ok')
+
+        run_forked_test(result, run_test, self.shortDescription(), self.fork)
 
 include_debugger = sys.version_info[:2] > (2, 5)
 
@@ -1166,6 +1189,9 @@ def main():
     parser.add_option("--coverage-xml", dest="coverage_xml",
                       action="store_true", default=False,
                       help="collect source coverage data for the Compiler in XML format")
+    parser.add_option("--coverage-html", dest="coverage_html",
+                      action="store_true", default=False,
+                      help="collect source coverage data for the Compiler in HTML format")
     parser.add_option("-A", "--annotate", dest="annotate_source",
                       action="store_true", default=True,
                       help="generate annotated HTML versions of the test source files")
@@ -1221,9 +1247,9 @@ def main():
 
     WITH_CYTHON = options.with_cython
 
-    if options.coverage or options.coverage_xml:
+    if options.coverage or options.coverage_xml or options.coverage_html:
         if not WITH_CYTHON:
-            options.coverage = options.coverage_xml = False
+            options.coverage = options.coverage_xml = options.coverage_html = False
         else:
             from coverage import coverage as _coverage
             coverage = _coverage(branch=True)
@@ -1337,15 +1363,15 @@ def main():
         test_suite.addTest(filetests.build_suite())
 
     if options.system_pyregr and languages:
-        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors,
-                                options.annotate_source, options.cleanup_workdir,
-                                options.cleanup_sharedlibs, True,
-                                options.cython_only, languages, test_bugs,
-                                options.fork, options.language_level)
-        test_suite.addTest(
-            filetests.handle_directory(
-                os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test'),
-                'pyregr'))
+        sys_pyregr_dir = os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test')
+        if os.path.isdir(sys_pyregr_dir):
+            filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors,
+                                    options.annotate_source, options.cleanup_workdir,
+                                    options.cleanup_sharedlibs, True,
+                                    options.cython_only, languages, test_bugs,
+                                    options.fork, sys.version_info[0])
+            sys.stderr.write("Including CPython regression tests in %s\n" % sys_pyregr_dir)
+            test_suite.addTest(filetests.handle_directory(sys_pyregr_dir, 'pyregr'))
 
     if options.xml_output_dir:
         from Cython.Tests.xmlrunner import XMLTestRunner
@@ -1356,7 +1382,7 @@ def main():
 
     result = test_runner.run(test_suite)
 
-    if options.coverage or options.coverage_xml:
+    if options.coverage or options.coverage_xml or options.coverage_html:
         coverage.stop()
         ignored_modules = ('Options', 'Version', 'DebugFlags', 'CmdLine')
         modules = [ module for name, module in sys.modules.items()
@@ -1367,6 +1393,8 @@ def main():
             coverage.report(modules, show_missing=0)
         if options.coverage_xml:
             coverage.xml_report(modules, outfile="coverage-report.xml")
+        if options.coverage_html:
+            coverage.html_report(modules, directory="coverage-report-html")
 
     if missing_dep_excluder.tests_missing_deps:
         sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n")