import os
import sys
-import distutils
from distutils.dist import Distribution
from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
from distutils.extension import Extension
HAS_CYTHON = True
except ImportError:
HAS_CYTHON = False
-import shutil
DEBUG = 0
+
+_reloads={}
+
def pyx_to_dll(filename, ext = None, force_rebuild = 0,
- build_in_temp=False, pyxbuild_dir=None):
+ build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False):
"""Compile a PYX file to a DLL and return the name of the generated .so
or .dll ."""
assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename)
if not pyxbuild_dir:
pyxbuild_dir = os.path.join(path, "_pyxbld")
- if DEBUG:
+ script_args=setup_args.get("script_args",[])
+ if DEBUG or "--verbose" in script_args:
quiet = "--verbose"
else:
quiet = "--quiet"
args.append("--force")
if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp")
- dist = Distribution({"script_name": None, "script_args": args})
+ sargs = setup_args.copy()
+ sargs.update(
+ {"script_name": None,
+ "script_args": args + script_args} )
+ dist = Distribution(sargs)
if not dist.ext_modules:
dist.ext_modules = []
dist.ext_modules.append(ext)
except ValueError: pass
dist.parse_config_files(config_files)
+ cfgfiles = dist.find_config_files()
+ try: cfgfiles.remove('setup.cfg')
+ except ValueError: pass
+ dist.parse_config_files(cfgfiles)
try:
ok = dist.parse_command_line()
except DistutilsArgError:
try:
dist.run_commands()
- return dist.get_command_obj("build_ext").get_outputs()[0]
+ obj_build_ext = dist.get_command_obj("build_ext")
+ so_path = obj_build_ext.get_outputs()[0]
+ if obj_build_ext.inplace:
+ # Python distutils get_outputs()[ returns a wrong so_path
+ # when --inplace ; see http://bugs.python.org/issue5977
+ # workaround:
+ so_path = os.path.join(os.path.dirname(filename),
+ os.path.basename(so_path))
+ if reload_support:
+ org_path = so_path
+ timestamp = os.path.getmtime(org_path)
+ global _reloads
+ last_timestamp, last_path, count = _reloads.get(org_path, (None,None,0) )
+ if last_timestamp == timestamp:
+ so_path = last_path
+ else:
+ basename = os.path.basename(org_path)
+ while count < 100:
+ count += 1
+ r_path = os.path.join(obj_build_ext.build_lib,
+ basename + '.reload%s'%count)
+ try:
+ import shutil # late import / reload_support is: debugging
+ shutil.copy2(org_path, r_path)
+ so_path = r_path
+ except IOError:
+ continue
+ break
+ else:
+ # used up all 100 slots
+ raise ImportError("reload count for %s reached maximum"%org_path)
+ _reloads[org_path]=(timestamp, so_path, count)
+ return so_path
except KeyboardInterrupt:
sys.exit(1)
except (IOError, os.error):
if DEBUG:
sys.stderr.write(error + "\n")
- raise
- else:
- raise RuntimeError(error)
-
- except (DistutilsError, CCompilerError):
- if DEBUG:
- raise
- else:
- exc = sys.exc_info()[1]
- raise RuntimeError(repr(exc))
+ raise
if __name__=="__main__":
pyx_to_dll("dummy.pyx")
sitecustomize.py with only those two lines at
/usr/local/lib/python2.3/site-packages/sitecustomize.py .
+A custom distutils.core.Extension instance and setup() args
+(Distribution) for for the build can be defined by a <modulename>.pyxbld
+file like:
+
+# examplemod.pyxbdl
+def make_ext(modname, pyxfilename):
+ from distutils.extension import Extension
+ return Extension(name = modname,
+ sources=[pyxfilename, 'hello.c'],
+ include_dirs=['/myinclude'] )
+def make_setup_args():
+ return dict(script_args=["--compiler=mingw32"])
+
+Extra dependencies can be defined by a <modulename>.pyxdep .
+See README.
+
Since Cython 0.11, the :mod:`pyximport` module also has experimental
compilation support for normal Python modules. This allows you to
automatically run Cython on every .pyx and .py module that Python
This code is based on the Py2.3+ import protocol as described in PEP 302.
"""
+
import sys
import os
import glob
import imp
-import pyxbuild
-from distutils.dep_util import newer
-from distutils.extension import Extension
-
-try:
- import hashlib
-except ImportError:
- import md5 as hashlib
mod_name = "pyximport"
"Load a pyrex file given a name and filename."
def get_distutils_extension(modname, pyxfilename):
- extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
+# try:
+# import hashlib
+# except ImportError:
+# import md5 as hashlib
+# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
# modname = modname + extra
- extension_mod = handle_special_build(modname, pyxfilename)
+ extension_mod,setup_args = handle_special_build(modname, pyxfilename)
if not extension_mod:
+ from distutils.extension import Extension
extension_mod = Extension(name = modname, sources=[pyxfilename])
- return extension_mod
+ return extension_mod,setup_args
def handle_special_build(modname, pyxfilename):
special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
-
- if not os.path.exists(special_build):
- ext = None
- else:
- globls = {}
- locs = {}
+ ext = None
+ setup_args={}
+ if os.path.exists(special_build):
+ # globls = {}
+ # locs = {}
# execfile(special_build, globls, locs)
# ext = locs["make_ext"](modname, pyxfilename)
mod = imp.load_source("XXXX", special_build, open(special_build))
- ext = mod.make_ext(modname, pyxfilename)
- assert ext and ext.sources, ("make_ext in %s did not return Extension"
- % special_build)
+ make_ext = getattr(mod,'make_ext',None)
+ if make_ext:
+ ext = make_ext(modname, pyxfilename)
+ assert ext and ext.sources, ("make_ext in %s did not return Extension"
+ % special_build)
+ make_setup_args = getattr(mod,'make_setup_args',None)
+ if make_setup_args:
+ setup_args = make_setup_args()
+ assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
+ % special_build)
+ assert set or setup_args, ("neither make_ext nor make_setup_args %s"
+ % special_build)
ext.sources = [os.path.join(os.path.dirname(special_build), source)
for source in ext.sources]
- return ext
+ return ext, setup_args
def handle_dependencies(pyxfilename):
dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
files.extend(glob.glob(fullpath))
# only for unit testing to see we did the right thing
- _test_files[:] = []
+ _test_files[:] = [] #$pycheck_no
# if any file that the pyxfile depends upon is newer than
# the pyx file, 'touch' the pyx file so that distutils will
# be tricked into rebuilding it.
for file in files:
+ from distutils.dep_util import newer
if newer(file, pyxfilename):
print("Rebuilding because of ", file)
filetime = os.path.getmtime(file)
"Path does not exist: %s" % pyxfilename)
handle_dependencies(pyxfilename)
- extension_mod = get_distutils_extension(name, pyxfilename)
+ extension_mod,setup_args = get_distutils_extension(name, pyxfilename)
+ build_in_temp=pyxargs.build_in_temp
+ sargs=pyxargs.setup_args.copy()
+ sargs.update(setup_args)
+ build_in_temp=sargs.pop('build_in_temp',build_in_temp)
+ import pyxbuild
so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
- build_in_temp=True,
- pyxbuild_dir=pyxbuild_dir)
+ build_in_temp=build_in_temp,
+ pyxbuild_dir=pyxbuild_dir,
+ setup_args=sargs,
+ reload_support=pyxargs.reload_support)
assert os.path.exists(so_path), "Cannot find: %s" % so_path
-
- junkpath = os.path.join(os.path.dirname(so_path), name+"_*")
+
+ junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ?
junkstuff = glob.glob(junkpath)
for path in junkstuff:
if path!=so_path:
mod = imp.load_dynamic(name, so_path)
assert mod.__file__ == so_path, (mod.__file__, so_path)
except Exception, e:
- raise ImportError("Building module failed: %s" % e)
+ import traceback
+ raise ImportError("Building module failed: %s" %
+ traceback.format_exception_only(*sys.exc_info()[:2])),None,sys.exc_info()[2]
return mod
self.pyxbuild_dir = pyxbuild_dir
def find_module(self, fullname, package_path=None):
- if fullname in sys.modules:
- return None
- if DEBUG_IMPORT:
- print("SEARCHING", fullname, package_path)
- if '.' in fullname:
+ if fullname in sys.modules and not pyxargs.reload_support:
+ return None # only here when reload()
+ try:
+ fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
+ if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open!
+ if ty!=imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
+ return None
+
+ # find .pyx fast, when .so/.pyd exist --inplace
+ pyxpath = os.path.splitext(pathname)[0]+self.extension
+ if os.path.isfile(pyxpath):
+ return PyxLoader(fullname, pyxpath,
+ pyxbuild_dir=self.pyxbuild_dir)
+
+ # .so/.pyd's on PATH should not be remote from .pyx's
+ # think no need to implement PyxArgs.importer_search_remote here?
+
+ except ImportError:
+ pass
+
+ # searching sys.path ...
+
+ #if DEBUG_IMPORT: print "SEARCHING", fullname, package_path
+ if '.' in fullname: # only when package_path anyway?
mod_parts = fullname.split('.')
- package = '.'.join(mod_parts[:-1])
module_name = mod_parts[-1]
else:
- package = None
module_name = fullname
pyx_module_name = module_name + self.extension
# this may work, but it returns the file content, not its path
paths = sys.path
join_path = os.path.join
is_file = os.path.isfile
- is_dir = os.path.isdir
+ #is_dir = os.path.isdir
+ sep = os.path.sep
for path in paths:
- if not is_dir(path):
- if not path:
- path = os.getcwd()
- else:
- continue
- for filename in os.listdir(path):
- if filename == pyx_module_name:
- return PyxLoader(fullname, join_path(path, filename),
- pyxbuild_dir=self.pyxbuild_dir)
- elif filename == module_name:
- package_path = join_path(path, filename)
- init_path = join_path(package_path,
- '__init__' + self.extension)
- if is_file(init_path):
- return PyxLoader(fullname, package_path, init_path,
- pyxbuild_dir=self.pyxbuild_dir)
+ if not path:
+ path = os.getcwd()
+ if is_file(path+sep+pyx_module_name):
+ return PyxLoader(fullname, join_path(path, pyx_module_name),
+ pyxbuild_dir=self.pyxbuild_dir)
+
# not found, normal package, not a .pyx file, none of our business
return None
return module
-def install(pyximport=True, pyimport=False, build_dir=None):
+#install args
+class PyxArgs(object):
+ build_dir=True
+ build_in_temp=True
+ setup_args={} #None
+
+##pyxargs=None
+
+def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
+ setup_args={}, reload_support=False ):
"""Main entry point. Call this to install the .pyx import hook in
your meta-path for a single Python process. If you want it to be
installed whenever you use Python, add it to your sitecustomize
By default, compiled modules will end up in a ``.pyxbld``
directory in the user's home directory. Passing a different path
as ``build_dir`` will override this.
+
+ ``build_in_temp=False`` will produce the C files locally. Working
+ with complex dependencies and debugging becomes more easy. This
+ can principally interfere with existing files of the same name.
+ build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
+ by a dict item of 'build_in_temp'
+
+ ``setup_args``: dict of arguments for Distribution - see
+ distutils.core.setup() . They are extended/overriden by those of
+ <modulename>.pyxbld/make_setup_args()
+
+ ``reload_support``: Enables support for dynamic
+ reload(<pyxmodulename>), e.g. after a change in the Cython code.
+ Additional files <so_path>.reloadNN may arise on that account, when
+ the previously loaded module file cannot be overwritten.
"""
if not build_dir:
build_dir = os.path.expanduser('~/.pyxbld')
+
+ global pyxargs
+ pyxargs = PyxArgs() #$pycheck_no
+ pyxargs.build_dir = build_dir
+ pyxargs.build_in_temp = build_in_temp
+ pyxargs.setup_args = (setup_args or {}).copy()
+ pyxargs.reload_support = reload_support
has_py_importer = False
has_pyx_importer = False
projects / cython.git / commitdiff