import gc
import codecs
import shutil
+import time
import unittest
import doctest
import operator
import tempfile
+import traceback
try:
from StringIO import StringIO
except ImportError:
except ImportError:
import pickle
+try:
+ import threading
+except ImportError: # No threads, no problems
+ threading = None
WITH_CYTHON = True
+CY3_DIR = None
from distutils.dist import Distribution
from distutils.core import Extension
from distutils.command.build_ext import build_ext as _build_ext
distutils_distro = Distribution()
+if sys.platform == 'win32':
+ # TODO: Figure out why this hackery (see http://thread.gmane.org/gmane.comp.python.cython.devel/8280/).
+ config_files = distutils_distro.find_config_files()
+ try: config_files.remove('setup.cfg')
+ except ValueError: pass
+ distutils_distro.parse_config_files(config_files)
+
+ cfgfiles = distutils_distro.find_config_files()
+ try: cfgfiles.remove('setup.cfg')
+ except ValueError: pass
+ distutils_distro.parse_config_files(cfgfiles)
+
TEST_DIRS = ['compile', 'errors', 'run', 'wrappers', 'pyregr', 'build']
TEST_RUN_DIRS = ['run', 'wrappers', 'pyregr']
VER_DEP_MODULES = {
# tests are excluded if 'CurrentPythonVersion OP VersionTuple', i.e.
- # (2,4) : (operator.le, ...) excludes ... when PyVer <= 2.4.x
+ # (2,4) : (operator.lt, ...) excludes ... when PyVer < 2.4.x
+ (2,4) : (operator.lt, lambda x: x in ['run.extern_builtins_T258',
+ 'run.builtin_sorted'
+ ]),
(2,5) : (operator.lt, lambda x: x in ['run.any',
'run.all',
]),
- (2,4) : (operator.le, lambda x: x in ['run.extern_builtins_T258'
- ]),
(2,6) : (operator.lt, lambda x: x in ['run.print_function',
'run.cython3',
+ 'run.pure_py', # decorators, with statement
]),
# The next line should start (3,); but this is a dictionary, so
# we can only have one (3,) key. Since 2.7 is supposed to be the
'run.special_methods_T561_py2']),
}
+# files that should not be converted to Python 3 code with 2to3
+KEEP_2X_FILES = [
+ os.path.join('Cython', 'Debugger', 'Tests', 'test_libcython_in_gdb.py'),
+ os.path.join('Cython', 'Debugger', 'Tests', 'test_libpython_in_gdb.py'),
+ os.path.join('Cython', 'Debugger', 'libcython.py'),
+ os.path.join('Cython', 'Debugger', 'libpython.py'),
+]
+
+COMPILER = None
INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
CFLAGS = os.getenv('CFLAGS', '').split()
def build_extension(self, ext):
if ext.language == 'c++':
try:
- try: # Py2.7+ & Py3.2+
+ try: # Py2.7+ & Py3.2+
compiler_obj = self.compiler_obj
except AttributeError:
compiler_obj = self.compiler
continue
suite.addTest(
self.handle_directory(path, filename))
- if sys.platform not in ['win32'] and sys.version_info[0] < 3:
- # Non-Windows makefile, can't run Cython under Py3.
- if [1 for selector in self.selectors if selector("embedded")]:
+ if sys.platform not in ['win32']:
+ # Non-Windows makefile.
+ if [1 for selector in self.selectors if selector("embedded")] \
+ and not [1 for selector in self.exclude_selectors if selector("embedded")]:
suite.addTest(unittest.makeSuite(EmbedTest))
return suite
filenames = os.listdir(path)
filenames.sort()
for filename in filenames:
- if context == "build" and filename.endswith(".srctree"):
+ if filename.endswith(".srctree"):
if not [ 1 for match in self.selectors if match(filename) ]:
continue
- suite.addTest(EndToEndTest(filename, workdir, self.cleanup_workdir))
+ if self.exclude_selectors:
+ if [1 for match in self.exclude_selectors if match(filename)]:
+ continue
+ suite.addTest(EndToEndTest(os.path.join(path, filename), workdir, self.cleanup_workdir))
continue
if not (filename.endswith(".pyx") or filename.endswith(".py")):
continue
if self.exclude_selectors:
if [1 for match in self.exclude_selectors if match(fqmodule)]:
continue
- if context in TEST_RUN_DIRS:
+ if context == 'pyregr':
+ test_class = CythonPyregrTestCase
+ elif context in TEST_RUN_DIRS:
if module.startswith("test_"):
test_class = CythonUnitTestCase
else:
for test in self.build_tests(test_class, path, workdir,
module, expect_errors):
suite.addTest(test)
- if filename.endswith('.py'):
+ if context == 'run' and filename.endswith('.py'):
# additionally test file in real Python
suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename)))
return suite
if is_related(filename) and os.path.isfile(os.path.join(workdir, filename)) ]
def split_source_and_output(self, test_directory, module, workdir):
- source_file = os.path.join(test_directory, module) + '.pyx'
- source_and_output = codecs.open(
- self.find_module_source_file(source_file), 'rU', 'ISO-8859-1')
- out = codecs.open(os.path.join(workdir, module + '.pyx'),
- 'w', 'ISO-8859-1')
- for line in source_and_output:
- last_line = line
- if line.startswith("_ERRORS"):
- out.close()
- out = ErrorWriter()
- else:
- out.write(line)
+ source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx')
+ source_and_output = codecs.open(source_file, 'rU', 'ISO-8859-1')
+ try:
+ out = codecs.open(os.path.join(workdir, module + os.path.splitext(source_file)[1]),
+ 'w', 'ISO-8859-1')
+ for line in source_and_output:
+ last_line = line
+ if line.startswith("_ERRORS"):
+ out.close()
+ out = ErrorWriter()
+ else:
+ out.write(line)
+ finally:
+ source_and_output.close()
try:
geterrors = out.geterrors
except AttributeError:
+ out.close()
return []
else:
return geterrors()
- def run_cython(self, test_directory, module, targetdir, incdir, annotate):
+ def run_cython(self, test_directory, module, targetdir, incdir, annotate,
+ extra_compile_options=None):
include_dirs = INCLUDE_DIRS[:]
if incdir:
include_dirs.append(incdir)
source = self.find_module_source_file(
os.path.join(test_directory, module + '.pyx'))
target = os.path.join(targetdir, self.build_target_filename(module))
+
+ if extra_compile_options is None:
+ extra_compile_options = {}
+
+ try:
+ CompilationOptions
+ except NameError:
+ from Cython.Compiler.Main import CompilationOptions
+ from Cython.Compiler.Main import compile as cython_compile
+ from Cython.Compiler.Main import default_options
+
options = CompilationOptions(
- pyrex_default_options,
+ default_options,
include_path = include_dirs,
output_file = target,
annotate = annotate,
language_level = self.language_level,
generate_pxi = False,
evaluate_tree_assertions = True,
+ **extra_compile_options
)
cython_compile(source, options=options,
full_module_name=module)
- def run_distutils(self, test_directory, module, workdir, incdir):
+ def run_distutils(self, test_directory, module, workdir, incdir,
+ extra_extension_args=None):
cwd = os.getcwd()
os.chdir(workdir)
try:
if incdir:
build_extension.include_dirs.append(incdir)
build_extension.finalize_options()
+ if COMPILER:
+ build_extension.compiler = COMPILER
ext_include_dirs = []
for match, get_additional_include_dirs in EXT_DEP_INCLUDES:
if match(module):
ext_include_dirs += get_additional_include_dirs()
+ ext_compile_flags = CFLAGS[:]
+ if build_extension.compiler == 'mingw32':
+ ext_compile_flags.append('-Wno-format')
+ if extra_extension_args is None:
+ extra_extension_args = {}
+
self.copy_related_files(test_directory, workdir, module)
extension = Extension(
module,
sources = self.find_source_files(workdir, module),
include_dirs = ext_include_dirs,
- extra_compile_args = CFLAGS,
+ extra_compile_args = ext_compile_flags,
+ **extra_extension_args
)
if self.language == 'cpp':
extension.language = 'c++'
result.startTest(self)
try:
self.setUp()
- self.runCompileTest()
- if not self.cython_only:
- self.run_doctests(self.module, result)
+ try:
+ self.runCompileTest()
+ self.run_tests(result)
+ finally:
+ check_thread_termination()
except Exception:
result.addError(self, sys.exc_info())
result.stopTest(self)
except Exception:
pass
+ def run_tests(self, result):
+ if not self.cython_only:
+ self.run_doctests(self.module, result)
+
def run_doctests(self, module_name, result):
if sys.version_info[0] >= 3 or not hasattr(os, 'fork') or not self.fork:
doctest.DocTestSuite(module_name).run(result)
output = open(result_file, 'wb')
pickle.dump(partial_result.data(), output)
except:
- import traceback
traceback.print_exc()
finally:
try: output.close()
del m
if loaded_module_name in sys.modules:
del sys.modules[loaded_module_name]
+ check_thread_termination()
except Exception:
result.addError(self, sys.exc_info())
result.stopTest(self)
self.write("%s\n" % line)
-class CythonUnitTestCase(CythonCompileTestCase):
+class CythonUnitTestCase(CythonRunTestCase):
def shortDescription(self):
return "compiling (%s) tests in %s" % (self.language, self.module)
- def run(self, result=None):
- if result is None:
- result = self.defaultTestResult()
- result.startTest(self)
+ def run_tests(self, result):
+ unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
+
+
+class CythonPyregrTestCase(CythonRunTestCase):
+ def _run_unittest(self, result, *classes):
+ """Run tests from unittest.TestCase-derived classes."""
+ valid_types = (unittest.TestSuite, unittest.TestCase)
+ suite = unittest.TestSuite()
+ for cls in classes:
+ if isinstance(cls, str):
+ if cls in sys.modules:
+ suite.addTest(unittest.findTestCases(sys.modules[cls]))
+ else:
+ raise ValueError("str arguments must be keys in sys.modules")
+ elif isinstance(cls, valid_types):
+ suite.addTest(cls)
+ else:
+ suite.addTest(unittest.makeSuite(cls))
+ suite.run(result)
+
+ def _run_doctest(self, result, module):
+ self.run_doctests(module, result)
+
+ def run_tests(self, result):
try:
- self.setUp()
- self.runCompileTest()
- unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
- except Exception:
- result.addError(self, sys.exc_info())
- result.stopTest(self)
+ from test import test_support as support
+ except ImportError: # Py3k
+ from test import support
+
+ def run_unittest(*classes):
+ return self._run_unittest(result, *classes)
+ def run_doctest(module, verbosity=None):
+ return self._run_doctest(result, module)
+
+ support.run_unittest = run_unittest
+ support.run_doctest = run_doctest
+
try:
- self.tearDown()
- except Exception:
- pass
+ module = __import__(self.module)
+ if hasattr(module, 'test_main'):
+ module.test_main()
+ except (unittest.SkipTest, support.ResourceDenied):
+ result.addSkip(self, 'ok')
+
+include_debugger = sys.version_info[:2] > (2, 5)
def collect_unittests(path, module_prefix, suite, selectors):
def file_matches(filename):
loader = unittest.TestLoader()
- skipped_dirs = []
+ if include_debugger:
+ skipped_dirs = []
+ else:
+ skipped_dirs = ['Cython' + os.path.sep + 'Debugger' + os.path.sep]
for dirpath, dirnames, filenames in os.walk(path):
if dirpath != path and "__init__.py" not in filenames:
module = getattr(module, x)
suite.addTests([loader.loadTestsFromModule(module)])
+
+
def collect_doctests(path, module_prefix, suite, selectors):
def package_matches(dirname):
+ if dirname == 'Debugger' and not include_debugger:
+ return False
return dirname not in ("Mac", "Distutils", "Plex")
def file_matches(filename):
- return (filename.endswith(".py") and not ('~' in filename
- or '#' in filename or filename.startswith('.')))
+ filename, ext = os.path.splitext(filename)
+ blacklist = ['libcython', 'libpython', 'test_libcython_in_gdb',
+ 'TestLibCython']
+ return (ext == '.py' and not
+ '~' in filename and not
+ '#' in filename and not
+ filename.startswith('.') and not
+ filename in blacklist)
import doctest, types
for dirpath, dirnames, filenames in os.walk(path):
- parentname = os.path.split(dirpath)[-1]
- if package_matches(parentname):
- for f in filenames:
- if file_matches(f):
- if not f.endswith('.py'): continue
- filepath = os.path.join(dirpath, f)[:-len(".py")]
- modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
- if not [ 1 for match in selectors if match(modulename) ]:
- continue
- module = __import__(modulename)
- for x in modulename.split('.')[1:]:
- module = getattr(module, x)
- if hasattr(module, "__doc__") or hasattr(module, "__test__"):
- try:
- suite.addTest(doctest.DocTestSuite(module))
- except ValueError: # no tests
- pass
+ for dir in list(dirnames):
+ if not package_matches(dir):
+ dirnames.remove(dir)
+ for f in filenames:
+ if file_matches(f):
+ if not f.endswith('.py'): continue
+ filepath = os.path.join(dirpath, f)
+ if os.path.getsize(filepath) == 0: continue
+ filepath = filepath[:-len(".py")]
+ modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
+ if not [ 1 for match in selectors if match(modulename) ]:
+ continue
+ if 'in_gdb' in modulename:
+ # These should only be imported from gdb.
+ continue
+ module = __import__(modulename)
+ for x in modulename.split('.')[1:]:
+ module = getattr(module, x)
+ if hasattr(module, "__doc__") or hasattr(module, "__test__"):
+ try:
+ suite.addTest(doctest.DocTestSuite(module))
+ except ValueError: # no tests
+ pass
class EndToEndTest(unittest.TestCase):
directory structure and its header gives a list of commands to run.
"""
cython_root = os.path.dirname(os.path.abspath(__file__))
-
+
def __init__(self, treefile, workdir, cleanup_workdir=True):
+ self.name = os.path.splitext(os.path.basename(treefile))[0]
self.treefile = treefile
- self.workdir = os.path.join(workdir, os.path.splitext(treefile)[0])
+ self.workdir = os.path.join(workdir, self.name)
self.cleanup_workdir = cleanup_workdir
cython_syspath = self.cython_root
for path in sys.path[::-1]:
unittest.TestCase.__init__(self)
def shortDescription(self):
- return "End-to-end %s" % self.treefile
+ return "End-to-end %s" % self.name
def setUp(self):
from Cython.TestUtils import unpack_source_tree
- _, self.commands = unpack_source_tree(os.path.join('tests', 'build', self.treefile), self.workdir)
+ _, self.commands = unpack_source_tree(self.treefile, self.workdir)
self.old_dir = os.getcwd()
os.chdir(self.workdir)
if self.workdir not in sys.path:
def tearDown(self):
if self.cleanup_workdir:
- shutil.rmtree(self.workdir)
+ for trial in range(5):
+ try:
+ shutil.rmtree(self.workdir)
+ except OSError:
+ time.sleep(0.1)
+ else:
+ break
os.chdir(self.old_dir)
-
+
def runTest(self):
commands = (self.commands
.replace("CYTHON", "PYTHON %s" % os.path.join(self.cython_root, 'cython.py'))
.replace("PYTHON", sys.executable))
- old_path = os.environ.get('PYTHONPATH')
try:
- os.environ['PYTHONPATH'] = self.cython_syspath + os.pathsep + (old_path or '')
- print(os.environ['PYTHONPATH'])
- self.assertEqual(0, os.system(commands))
+ old_path = os.environ.get('PYTHONPATH')
+ os.environ['PYTHONPATH'] = self.cython_syspath + os.pathsep + os.path.join(self.cython_syspath, (old_path or ''))
+ for command in commands.split('\n'):
+ if sys.version_info[:2] >= (2,4):
+ import subprocess
+ p = subprocess.Popen(commands,
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ shell=True)
+ out, err = p.communicate()
+ res = p.returncode
+ if res != 0:
+ print(command)
+ print(out)
+ print(err)
+ else:
+ res = os.system(command)
+ self.assertEqual(0, res, "non-zero exit status")
finally:
if old_path:
os.environ['PYTHONPATH'] = old_path
# TODO: Windows support.
class EmbedTest(unittest.TestCase):
-
+
working_dir = "Demos/embed"
-
+
def setUp(self):
self.old_dir = os.getcwd()
os.chdir(self.working_dir)
os.system(
"make PYTHON='%s' clean > /dev/null" % sys.executable)
-
+
def tearDown(self):
try:
os.system(
except:
pass
os.chdir(self.old_dir)
-
+
def test_embed(self):
from distutils import sysconfig
libname = sysconfig.get_config_var('LIBRARY')
if not os.path.isdir(libdir) or libname not in os.listdir(libdir):
# report the error for the original directory
libdir = sysconfig.get_config_var('LIBDIR')
+ cython = 'cython.py'
+ if sys.version_info[0] >=3:
+ cython = os.path.join(CY3_DIR, cython)
+ cython = os.path.abspath(os.path.join('..', '..', cython))
self.assert_(os.system(
- "make PYTHON='%s' LIBDIR1='%s' test > make.output" % (sys.executable, libdir)) == 0)
+ "make PYTHON='%s' CYTHON='%s' LIBDIR1='%s' test > make.output" % (sys.executable, cython, libdir)) == 0)
try:
os.remove('make.output')
except OSError:
self.excludes[line.split()[0]] = True
finally:
f.close()
-
+
def __call__(self, testname):
return testname in self.excludes or testname.split('.')[-1] in self.excludes
graft Cython
recursive-exclude Cython *
recursive-include Cython *.py *.pyx *.pxd
+ recursive-include Cython/Debugger/Tests *
+ include runtests.py
+ include cython.py
''')
sys.path.insert(0, cy3_dir)
+ for keep_2x_file in KEEP_2X_FILES:
+ destfile = os.path.join(cy3_dir, keep_2x_file)
+ shutil.copy(keep_2x_file, destfile)
+
+class PendingThreadsError(RuntimeError):
+ pass
+
+threads_seen = []
+
+def check_thread_termination(ignore_seen=True):
+ if threading is None: # no threading enabled in CPython
+ return
+ current = threading.currentThread()
+ blocking_threads = []
+ for t in threading.enumerate():
+ if not t.isAlive() or t == current:
+ continue
+ t.join(timeout=2)
+ if t.isAlive():
+ if not ignore_seen:
+ blocking_threads.append(t)
+ continue
+ for seen in threads_seen:
+ if t is seen:
+ break
+ else:
+ threads_seen.append(t)
+ blocking_threads.append(t)
+ if not blocking_threads:
+ return
+ sys.stderr.write("warning: left-over threads found after running test:\n")
+ for t in blocking_threads:
+ sys.stderr.write('...%s\n' % repr(t))
+ raise PendingThreadsError("left-over threads found after running test")
+
+def main():
+
+ DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]))
-if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--no-cleanup", dest="cleanup_workdir",
parser.add_option("--no-cython", dest="with_cython",
action="store_false", default=True,
help="do not run the Cython compiler, only the C compiler")
+ parser.add_option("--compiler", dest="compiler", default=None,
+ help="C compiler type")
parser.add_option("--no-c", dest="use_c",
action="store_false", default=True,
help="do not test C compilation")
help="do not run the file based tests")
parser.add_option("--no-pyregr", dest="pyregr",
action="store_false", default=True,
- help="do not run the regression tests of CPython in tests/pyregr/")
+ help="do not run the regression tests of CPython in tests/pyregr/")
parser.add_option("--cython-only", dest="cython_only",
action="store_true", default=False,
help="only compile pyx to c, do not run C compiler or run the tests")
parser.add_option("--exit-ok", dest="exit_ok", default=False,
action="store_true",
help="exit without error code even on test failures")
+ parser.add_option("--root-dir", dest="root_dir", default=os.path.join(DISTDIR, 'tests'),
+ help="working directory")
+ parser.add_option("--work-dir", dest="work_dir", default=os.path.join(os.getcwd(), 'BUILD'),
+ help="working directory")
options, cmd_args = parser.parse_args()
- DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]))
- ROOTDIR = os.path.join(DISTDIR, 'tests')
- WORKDIR = os.path.join(os.getcwd(), 'BUILD')
+ ROOTDIR = os.path.abspath(options.root_dir)
+ WORKDIR = os.path.abspath(options.work_dir)
if sys.version_info[0] >= 3:
options.doctests = False
for name in cy_modules:
del sys.modules[name]
# hasn't been refactored yet - do it now
- cy3_dir = os.path.join(WORKDIR, 'Cy3')
+ global CY3_DIR
+ CY3_DIR = cy3_dir = os.path.join(WORKDIR, 'Cy3')
if sys.version_info >= (3,1):
refactor_for_py3(DISTDIR, cy3_dir)
elif os.path.isdir(cy3_dir):
coverage.start()
if WITH_CYTHON:
+ global CompilationOptions, pyrex_default_options, cython_compile
from Cython.Compiler.Main import \
CompilationOptions, \
default_options as pyrex_default_options, \
# RUN ALL TESTS!
UNITTEST_MODULE = "Cython"
- UNITTEST_ROOT = os.path.join(os.getcwd(), UNITTEST_MODULE)
+ UNITTEST_ROOT = os.path.join(os.path.dirname(__file__), UNITTEST_MODULE)
if WITH_CYTHON:
if os.path.exists(WORKDIR):
for path in os.listdir(WORKDIR):
# Chech which external modules are not present and exclude tests
# which depends on them (by prefix)
- missing_dep_excluder = MissingDependencyExcluder(EXT_DEP_MODULES)
- version_dep_excluder = VersionDependencyExcluder(VER_DEP_MODULES)
+ missing_dep_excluder = MissingDependencyExcluder(EXT_DEP_MODULES)
+ version_dep_excluder = VersionDependencyExcluder(VER_DEP_MODULES)
exclude_selectors = [missing_dep_excluder, version_dep_excluder] # want to pring msg at exit
if options.exclude:
exclude_selectors += [ re.compile(r, re.I|re.U).search for r in options.exclude ]
-
+
if not test_bugs:
- exclude_selectors += [ FileListExcluder("tests/bugs.txt") ]
-
+ exclude_selectors += [ FileListExcluder(os.path.join(ROOTDIR, "bugs.txt")) ]
+
if sys.platform in ['win32', 'cygwin'] and sys.version_info < (2,6):
exclude_selectors += [ lambda x: x == "run.specialfloat" ]
+ global COMPILER
+ if options.compiler:
+ COMPILER = options.compiler
languages = []
if options.use_c:
languages.append('c')
ignored_modules = ('Options', 'Version', 'DebugFlags', 'CmdLine')
modules = [ module for name, module in sys.modules.items()
if module is not None and
- name.startswith('Cython.Compiler.') and
+ name.startswith('Cython.Compiler.') and
name[len('Cython.Compiler.'):] not in ignored_modules ]
if options.coverage:
coverage.report(modules, show_missing=0)
import refnanny
sys.stderr.write("\n".join([repr(x) for x in refnanny.reflog]))
+ print("ALL DONE")
+
if options.exit_ok:
- sys.exit(0)
+ return_code = 0
else:
- sys.exit(not result.wasSuccessful())
+ return_code = not result.wasSuccessful()
+
+ try:
+ check_thread_termination(ignore_seen=False)
+ sys.exit(return_code)
+ except PendingThreadsError:
+ # normal program exit won't kill the threads, do it the hard way here
+ os._exit(return_code)
+
+if __name__ == '__main__':
+ try:
+ main()
+ except SystemExit: # <= Py2.4 ...
+ raise
+ except Exception:
+ traceback.print_exc()
+ try:
+ check_thread_termination(ignore_seen=False)
+ except PendingThreadsError:
+ # normal program exit won't kill the threads, do it the hard way here
+ os._exit(1)
projects / cython.git / blobdiff