From: PaulPrescod Date: Sat, 16 Aug 2008 09:21:40 +0000 (-0700) Subject: pyximport for compiling .pyx files on import X-Git-Tag: 0.9.8.1~21 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=53c3e2bfb16c58feda7069f7f5d05617fa6b1e3c;p=cython.git pyximport for compiling .pyx files on import --- diff --git a/pyximport/PKG-INFO b/pyximport/PKG-INFO new file mode 100644 index 00000000..6d1b811a --- /dev/null +++ b/pyximport/PKG-INFO @@ -0,0 +1,11 @@ +Metadata-Version: 1.0 +Name: pyximport +Version: 1.0 +Summary: Hooks to build and run Pyrex files as if they were simple Python files +Home-page: http://www.prescod.net/pyximport +Author: Paul Prescod +Author-email: paul@prescod.net +License: Python +Description: UNKNOWN +Keywords: pyrex import hook +Platform: UNKNOWN diff --git a/pyximport/README b/pyximport/README new file mode 100644 index 00000000..0fc3a90b --- /dev/null +++ b/pyximport/README @@ -0,0 +1,79 @@ + + == Pyximport == + +Download: pyx-import-1.0.tar.gz + + +Pyrex is a compiler. Therefore it is natural that people tend to go +through an edit/compile/test cycle with Pyrex modules. But my personal +opinion is that one of the deep insights in Python's implementation is +that a language can be compiled (Python modules are compiled to .pyc) +files and hide that compilation process from the end-user so that they +do not have to worry about it. Pyximport does this for Pyrex modules. +For instance if you write a Pyrex module called "foo.pyx", with +Pyximport you can import it in a regular Python module like this: + + +import pyximport; pyximport.install() +import foo + +Doing so will result in the compilation of foo.pyx (with appropriate +exceptions if it has an error in it). + +If you would always like to import pyrex files without building them +specially, you can also the first line above to your sitecustomize.py. +That will install the hook every time you run Python. Then you can use +Pyrex modules just with simple import statements. I like to test my +Pyrex modules like this: + + +python -c "import foo" + + == Dependency Handling == + +In Pyximport 1.1 it is possible to declare that your module depends on +multiple files, (likely ".h" and ".pxd" files). If your Pyrex module is +named "foo" and thus has the filename "foo.pyx" then you should make +another file in the same directory called "foo.pyxdep". The +"modname.pyxdep" file can be a list of filenames or "globs" (like +"*.pxd" or "include/*.h"). Each filename or glob must be on a separate +line. Pyximport will check the file date for each of those files before +deciding whether to rebuild the module. In order to keep track of the +fact that the dependency has been handled, Pyximport updates the +modification time of your ".pyx" source file. Future versions may do +something more sophisticated like informing distutils of the +dependencies directly. + + == Limitations == + +Pyximport does not give you any control over how your Pyrex file is +compiled. Usually the defaults are fine. You might run into problems if +you wanted to write your program in half-C, half-Pyrex and build them +into a single library. Pyximport 1.2 will probably do this. + +Pyximport does not hide the Distutils/GCC warnings and errors generated +by the import process. Arguably this will give you better feedback if +something went wrong and why. And if nothing went wrong it will give you +the warm fuzzy that pyximport really did rebuild your module as it was +supposed to. + + == For further thought and discussion == + +I don't think that Python's "reload" will do anything for changed .SOs +on some (all?) platforms. It would require some (easy) experimentation +that I haven't gotten around to. But reload is rarely used in +applications outside of the Python interactive interpreter and certainly +not used much for C extension modules. Info about Windows + + +"setup.py install" does not modify sitecustomize.py for you. Should it? +Modifying Python's "standard interpreter" behaviour may be more than +most people expect of a package they install.. + +Pyximport puts your ".c" file beside your ".pyx" file (analogous to +".pyc" beside ".py"). But it puts the platform-specific binary in a +build directory as per normal for Distutils. If I could wave a magic +wand and get Pyrex or distutils or whoever to put the build directory I +might do it but not necessarily: having it at the top level is VERY +HELPFUL for debugging Pyrex problems. + diff --git a/pyximport/Setup.py b/pyximport/Setup.py new file mode 100644 index 00000000..513449ec --- /dev/null +++ b/pyximport/Setup.py @@ -0,0 +1,35 @@ +from distutils.core import setup +import sys, os +from StringIO import StringIO + +if "sdist" in sys.argv: + try: + os.remove("MANIFEST") + except (IOError, OSError): + pass + + import html2text + out = StringIO() + html2text.convert_files(open("index.html"), out) + out.write("\n\n") + open("README", "w").write(out.getvalue()) + +setup( + name = "pyximport", + fullname = "Pyrex Import Hooks", + version = "1.0", + description = "Hooks to build and run Pyrex files as if they were simple Python files", + author = "Paul Prescod", + author_email = "paul@prescod.net", + url = "http://www.prescod.net/pyximport", + license = "Python", + keywords = "pyrex import hook", + scripts = ["pyxrun"], + data_files = [("examples/multi_file_extension", + ["README", "ccode.c", "test.pyx", "test.pyxbld"]), + ("examples/dependencies", + ["README", "test.pyx", "test.pyxdep", "header.h", + "header2.h", "header3.h", "header4.h"]) + ], + py_modules = ["pyximport", "pyxbuild"]) + diff --git a/pyximport/pyxbuild.py b/pyximport/pyxbuild.py new file mode 100644 index 00000000..3e987570 --- /dev/null +++ b/pyximport/pyxbuild.py @@ -0,0 +1,79 @@ +"""Build a Pyrex file from .pyx source to .so loadable module using +the installed distutils infrastructure. Call: + +out_fname = pyx_to_dll("foo.pyx") +""" +import os, md5 + +import distutils +from distutils.dist import Distribution +from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError +from distutils.extension import Extension +from distutils.util import grok_environment_error +from Pyrex.Distutils import build_ext +import shutil + +DEBUG = 0 +def pyx_to_dll(filename, ext = None, force_rebuild = 0): + """Compile a PYX file to a DLL and return the name of the generated .so + or .dll .""" + assert os.path.exists(filename) + + path, name = os.path.split(filename) + + if not ext: + modname, extension = os.path.splitext(name) + assert extension == ".pyx", extension + ext = Extension(name=modname, sources=[filename]) + + if DEBUG: + quiet = "--verbose" + else: + quiet = "--quiet" + args = [quiet, "build_ext"] + if force_rebuild: + args.append("--force") + dist = Distribution({"script_name": None, "script_args": args}) + if not dist.ext_modules: + dist.ext_modules = [] + dist.ext_modules.append(ext) + dist.cmdclass = {'build_ext': build_ext} + build = dist.get_command_obj('build') + build.build_base = os.path.join(path, "_pyxbld") + + try: + ok = dist.parse_command_line() + except DistutilsArgError, msg: + raise + + if DEBUG: + print "options (after parsing command line):" + dist.dump_option_dicts() + assert ok + + + try: + dist.run_commands() + return dist.get_command_obj("build_ext").get_outputs()[0] + except KeyboardInterrupt: + raise SystemExit, "interrupted" + except (IOError, os.error), exc: + error = grok_environment_error(exc) + + if DEBUG: + sys.stderr.write(error + "\n") + raise + else: + raise SystemExit, error + + except (DistutilsError, + CCompilerError), msg: + if DEBUG: + raise + else: + raise SystemExit, "error: " + str(msg) + +if __name__=="__main__": + pyx_to_dll("dummy.pyx") + import test + diff --git a/pyximport/pyximport.py b/pyximport/pyximport.py new file mode 100644 index 00000000..0f7c941c --- /dev/null +++ b/pyximport/pyximport.py @@ -0,0 +1,228 @@ +""" +Import hooks; when installed (with the install()) function, these hooks +allow importing .pyx files as if they were Python modules. + +If you want the hook installed every time you run Python +you can add it to your Python version by adding these lines to +sitecustomize.py (which you can create from scratch in site-packages +if it doesn't exist there are somewhere else on your python path) + +import pyximport +pyximport.install() + +For instance on the Mac with Python 2.3 built from CVS, you could +create sitecustomize.py with only those two lines at +/usr/local/lib/python2.3/site-packages/sitecustomize.py . + +Running this module as a top-level script will run a test and then print +the documentation. + +This code was modeled on Quixote's ptl_import. +""" +import sys, os, shutil +import imp, ihooks, glob, md5 +import __builtin__ +import pyxbuild +from distutils.dep_util import newer +from distutils.extension import Extension + +mod_name = "pyximport" + +assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later" + +PYX_FILE_TYPE = 1011 +PYX_EXT = ".pyx" +PYXDEP_EXT = ".pyxdep" +PYXBLD_EXT = ".pyxbld" +_test_files = [] + +class PyxHooks (ihooks.Hooks): + """Import hook that declares our suffixes. Let install() install it.""" + def get_suffixes (self): + # add our suffixes + return imp.get_suffixes() + [(PYX_EXT, "r", PYX_FILE_TYPE)] + +# Performance problem: for every PYX file that is imported, we will +# invoke the whole distutils infrastructure even if the module is +# already built. It might be more efficient to only do it when the +# mod time of the .pyx is newer than the mod time of the .so but +# the question is how to get distutils to tell me the name of the .so +# before it builds it. Maybe it is easy...but maybe the peformance +# issue isn't real. +def _load_pyrex(name, filename): + "Load a pyrex file given a name and filename." + +def get_distutils_extension(modname, pyxfilename): + + extra = "_" + md5.md5(open(pyxfilename).read()).hexdigest() + modname = modname + extra + + extension_mod = handle_special_build(modname, pyxfilename) + + if not extension_mod: + extension_mod = Extension(name = modname, sources=[pyxfilename]) + + return extension_mod + +def handle_special_build(modname, pyxfilename): + special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT + + if not os.path.exists(special_build): + ext = None + else: + globls = {} + locs = {} + # execfile(special_build, globls, locs) + # ext = locs["make_ext"](modname, pyxfilename) + mod = imp.load_source("XXXX", special_build, open(special_build)) + ext = mod.make_ext(modname, pyxfilename) + assert ext and ext.sources, ("make_ext in %s did not return Extension" + % special_build) + ext.sources = [os.path.join(os.path.dirname(special_build), source) + for source in ext.sources] + return ext + +def handle_dependencies(pyxfilename): + dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT + + # by default let distutils decide whether to rebuild on its own + # (it has a better idea of what the output file will be) + + # but we know more about dependencies so force a rebuild if + # some of the dependencies are newer than the pyxfile. + if os.path.exists(dependfile): + depends = open(dependfile).readlines() + depends = [depend.strip() for depend in depends] + + # gather dependencies in the "files" variable + # the dependency file is itself a dependency + files = [dependfile] + for depend in depends: + fullpath = os.path.join(os.path.dirname(dependfile), + depend) + files.extend(glob.glob(fullpath)) + + # only for unit testing to see we did the right thing + _test_files[:] = [] + + # if any file that the pyxfile depends upon is newer than + # the pyx file, 'touch' the pyx file so that distutils will + # be tricked into rebuilding it. + for file in files: + if newer(file, pyxfilename): + print "Rebuilding because of ", file + filetime = os.path.getmtime(file) + os.utime(pyxfilename, (filetime, filetime)) + _test_files.append(file) + +def build_module(name, pyxfilename): + assert os.path.exists(pyxfilename), ( + "Path does not exist: %s" % pyxfilename) + handle_dependencies(pyxfilename) + + extension_mod = get_distutils_extension(name, pyxfilename) + + so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod) + assert os.path.exists(so_path), "Cannot find: %s" % so_path + + junkpath = os.path.join(os.path.dirname(so_path), name+"_*") + junkstuff = glob.glob(junkpath) + for path in junkstuff: + if path!=so_path: + try: + os.remove(path) + except IOError: + print "Couldn't remove ", path + + return so_path + +def load_module(name, pyxfilename): + so_path = build_module(name, pyxfilename) + mod = imp.load_dynamic(name, so_path) + assert mod.__file__ == so_path, (mod.__file__, so_path) + return mod + +class PyxLoader (ihooks.ModuleLoader): + """Load a module. It checks whether a file is a .pyx and returns it. + Otherwise it lets the ihooks base class handle it. Let install() + install it.""" + + def load_module (self, name, stuff): + # If it's a Pyrex file, load it specially. + if stuff[2][2] == PYX_FILE_TYPE: + file, pyxfilename, info = stuff + (suff, mode, type) = info + if file: + file.close() + return load_module(name, pyxfilename) + else: + # Otherwise, use the default handler for loading + return ihooks.ModuleLoader.load_module( self, name, stuff) + +try: + import cimport +except ImportError: + cimport = None + +class cModuleImporter(ihooks.ModuleImporter): + """This was just left in from the Quixote implementation. I think + it allows a performance enhancement if you have the cimport module + from Quixote. Let install() install it.""" + def __init__(self, loader=None): + self.loader = loader or ihooks.ModuleLoader() + cimport.set_loader(self.find_import_module) + + def find_import_module(self, fullname, subname, path): + stuff = self.loader.find_module(subname, path) + if not stuff: + return None + return self.loader.load_module(fullname, stuff) + + def install(self): + self.save_import_module = __builtin__.__import__ + self.save_reload = __builtin__.reload + if not hasattr(__builtin__, 'unload'): + __builtin__.unload = None + self.save_unload = __builtin__.unload + __builtin__.__import__ = cimport.import_module + __builtin__.reload = cimport.reload_module + __builtin__.unload = self.unload + +_installed = 0 + +def install(): + """Main entry point. call this to install the import hook in your + for a single Python process. If you want it to be installed whenever + you use Python, add it to your sitecustomize (as described above). + + """ + global _installed + if not _installed: + hooks = PyxHooks() + loader = PyxLoader(hooks) + if cimport is not None: + importer = cModuleImporter(loader) + else: + importer = ihooks.ModuleImporter(loader) + ihooks.install(importer) + _installed = 1 + +def on_remove_file_error(func, path, excinfo): + print "Sorry! Could not remove a temp file:", path + print "Extra information." + print func, excinfo + print "You may want to delete this yourself when you get a chance." + +def show_docs(): + import __main__ + __main__.__name__ = mod_name + for name in dir(__main__): + item = getattr(__main__, name) + try: + setattr(item, "__module__", mod_name) + except (AttributeError, TypeError): + pass + help(__main__) + +if __name__ == '__main__': + show_docs() diff --git a/pyximport/test/test_pyximport.py b/pyximport/test/test_pyximport.py new file mode 100644 index 00000000..f149a9cc --- /dev/null +++ b/pyximport/test/test_pyximport.py @@ -0,0 +1,64 @@ +import pyximport; pyximport.install() +import os, sys +import time, shutil +import tempfile + +def make_tempdir(): + tempdir = os.path.join(tempfile.gettempdir(), "pyrex_temp") + if os.path.exists(tempdir): + remove_tempdir(tempdir) + + os.mkdir(tempdir) + return tempdir + +def remove_tempdir(tempdir): + shutil.rmtree(tempdir, 0, on_remove_file_error) + +def on_remove_file_error(func, path, excinfo): + print "Sorry! Could not remove a temp file:", path + print "Extra information." + print func, excinfo + print "You may want to delete this yourself when you get a chance." + +def test(): + tempdir = make_tempdir() + sys.path.append(tempdir) + filename = os.path.join(tempdir, "dummy.pyx") + open(filename, "w").write("print 'Hello world from the Pyrex install hook'") + import dummy + reload(dummy) + + depend_filename = os.path.join(tempdir, "dummy.pyxdep") + depend_file = open(depend_filename, "w") + depend_file.write("*.txt\nfoo.bar") + depend_file.close() + + build_filename = os.path.join(tempdir, "dummy.pyxbld") + build_file = open(build_filename, "w") + build_file.write(""" +from distutils.extension import Extension +def make_ext(name, filename): + return Extension(name=name, sources=[filename]) +""") + build_file.close() + + open(os.path.join(tempdir, "foo.bar"), "w").write(" ") + open(os.path.join(tempdir, "1.txt"), "w").write(" ") + open(os.path.join(tempdir, "abc.txt"), "w").write(" ") + reload(dummy) + assert len(pyximport._test_files)==1, pyximport._test_files + reload(dummy) + + time.sleep(1) # sleep a second to get safer mtimes + open(os.path.join(tempdir, "abc.txt"), "w").write(" ") + print "Here goes the reolad" + reload(dummy) + assert len(pyximport._test_files) == 1, pyximport._test_files + + reload(dummy) + assert len(pyximport._test_files) ==0, pyximport._test_files + remove_tempdir(tempdir) + +if __name__=="__main__": + test() + diff --git a/pyximport/test/test_reload.py b/pyximport/test/test_reload.py new file mode 100644 index 00000000..0e5b7058 --- /dev/null +++ b/pyximport/test/test_reload.py @@ -0,0 +1,33 @@ +# reload seems to work for Python 2.3 but not 2.2. +import time, os, sys +import test_pyximport + +# debugging the 2.2 problem +if 1: + from distutils import sysconfig + try: + sysconfig.set_python_build() + except AttributeError: + pass + import pyxbuild + print pyxbuild.distutils.sysconfig == sysconfig + +def test(): + tempdir = test_pyximport.make_tempdir() + sys.path.append(tempdir) + hello_file = os.path.join(tempdir, "hello.pyx") + open(hello_file, "w").write("x = 1; print x; before = 'before'\n") + import hello + assert hello.x == 1 + + time.sleep(1) # sleep to make sure that new "hello.pyx" has later + # timestamp than object file. + + open(hello_file, "w").write("x = 2; print x; after = 'after'\n") + reload(hello) + assert hello.x == 2, "Reload should work on Python 2.3 but not 2.2" + test_pyximport.remove_tempdir(tempdir) + +if __name__=="__main__": + test() +