6 if sys.version_info[:2] < (2, 3):
7 sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n")
14 from sets import Set as set
21 from Scanning import PyrexScanner, FileSourceDescriptor
22 from Errors import PyrexError, CompileError, InternalError, error
23 from Symtab import BuiltinScope, ModuleScope
24 from Cython import Utils
25 from Cython.Utils import open_new_file, replace_suffix
29 module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
34 # For quick debugging in pipelines
38 def abort_on_errors(node):
39 # Stop the pipeline if there are any errors.
40 if Errors.num_errors != 0:
41 raise InternalError, "abort"
44 class CompilationData(object):
45 # Bundles the information that is passed from transform to transform.
46 # (For now, this is only)
48 # While Context contains every pxd ever loaded, path information etc.,
49 # this only contains the data related to a single compilation pass
51 # pyx ModuleNode Main code tree of this compilation.
52 # pxds {string : ModuleNode} Trees for the pxds used in the pyx.
53 # codewriter CCodeWriter Where to output final code.
54 # options CompilationOptions
55 # result CompilationResult
58 class Context(object):
59 # This class encapsulates the context needed for compiling
60 # one or more Cython implementation files along with their
61 # associated and imported declaration files. It includes
62 # the root of the module import namespace and the list
63 # of directories to search for include files.
65 # modules {string : ModuleScope}
66 # include_directories [string]
67 # future_directives [object]
69 def __init__(self, include_directories, compiler_directives, cpp=False):
70 #self.modules = {"__builtin__" : BuiltinScope()}
71 import Builtin, CythonScope
72 self.modules = {"__builtin__" : Builtin.builtin_scope}
73 self.modules["cython"] = CythonScope.create_cython_scope(self)
74 self.include_directories = include_directories
75 self.future_directives = set()
76 self.compiler_directives = compiler_directives
79 self.pxds = {} # full name -> node tree
81 standard_include_path = os.path.abspath(os.path.normpath(
82 os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
83 self.include_directories = include_directories + [standard_include_path]
85 def create_pipeline(self, pxd, py=False):
86 from Visitor import PrintTree
87 from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
88 from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
89 from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
90 from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
91 from TypeInference import MarkAssignments, MarkOverflowingArithmetic
92 from ParseTreeTransforms import AlignFunctionDefinitions, GilCheck
93 from AnalysedTreeTransforms import AutoTestDictTransform
94 from AutoDocTransforms import EmbedSignature
95 from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
96 from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
97 from Optimize import ConstantFolding, FinalOptimizePhase
98 from Optimize import DropRefcountingTransform
99 from Buffer import IntroduceBufferAuxiliaryVars
100 from ModuleNode import check_c_declarations, check_c_declarations_pxd
102 # Temporary hack that can be used to ensure that all result_code's
103 # are generated at code generation time.
105 class ClearResultCodes(Visitor.CythonTransform):
106 def visit_ExprNode(self, node):
107 self.visitchildren(node)
108 node.result_code = "<cleared>"
112 _check_c_declarations = check_c_declarations_pxd
113 _specific_post_parse = PxdPostParse(self)
115 _check_c_declarations = check_c_declarations
116 _specific_post_parse = None
119 _align_function_definitions = AlignFunctionDefinitions(self)
121 _align_function_definitions = None
126 _specific_post_parse,
127 InterpretCompilerDirectives(self, self.compiler_directives),
128 _align_function_definitions,
129 MarkClosureVisitor(self),
131 FlattenInListTransform(),
133 DecoratorTransform(self),
134 AnalyseDeclarationsTransform(self),
135 CreateClosureClasses(self),
136 AutoTestDictTransform(self),
137 EmbedSignature(self),
138 EarlyReplaceBuiltinCalls(self),
139 MarkAssignments(self),
140 MarkOverflowingArithmetic(self),
141 TransformBuiltinMethods(self),
142 IntroduceBufferAuxiliaryVars(self),
143 _check_c_declarations,
144 AnalyseExpressionsTransform(self),
145 OptimizeBuiltinCalls(self),
146 IterationTransform(),
148 DropRefcountingTransform(),
149 FinalOptimizePhase(self),
151 #ClearResultCodes(self),
152 #SpecialFunctions(self),
153 #CreateClosureClasses(context),
156 def create_pyx_pipeline(self, options, result, py=False):
157 def generate_pyx_code(module_node):
158 module_node.process_implementation(options, result)
159 result.compilation_source = module_node.compilation_source
162 def inject_pxd_code(module_node):
163 from textwrap import dedent
164 stats = module_node.body.stats
165 for name, (statlistnode, scope) in self.pxds.iteritems():
166 # Copy over function nodes to the module
167 # (this seems strange -- I believe the right concept is to split
168 # ModuleNode into a ModuleNode and a CodeGenerator, and tell that
169 # CodeGenerator to generate code both from the pyx and pxd ModuleNodes.
170 stats.append(statlistnode)
171 # Until utility code is moved to code generation phase everywhere,
172 # we need to copy it over to the main scope
173 module_node.scope.utility_code_list.extend(scope.utility_code_list)
177 if options.evaluate_tree_assertions:
178 from Cython.TestUtils import TreeAssertVisitor
179 test_support.append(TreeAssertVisitor())
183 ] + self.create_pipeline(pxd=False, py=py) + test_support + [
189 def create_pxd_pipeline(self, scope, module_name):
190 def parse_pxd(source_desc):
191 tree = self.parse(source_desc, scope, pxd=True,
192 full_module_name=module_name)
197 from CodeGeneration import ExtractPxdCode
199 # The pxd pipeline ends up with a CCodeWriter containing the
200 # code of the pxd, as well as a pxd scope.
201 return [parse_pxd] + self.create_pipeline(pxd=True) + [
202 ExtractPxdCode(self),
205 def create_py_pipeline(self, options, result):
206 return self.create_pyx_pipeline(options, result, py=True)
209 def process_pxd(self, source_desc, scope, module_name):
210 pipeline = self.create_pxd_pipeline(scope, module_name)
211 result = self.run_pipeline(pipeline, source_desc)
214 def nonfatal_error(self, exc):
215 return Errors.report_error(exc)
217 def run_pipeline(self, pipeline, source):
221 for phase in pipeline:
222 if phase is not None:
223 if DebugFlags.debug_verbose_pipeline:
225 print "Entering pipeline phase %r" % phase
227 if DebugFlags.debug_verbose_pipeline:
228 print " %.3f seconds" % (time() - t)
229 except CompileError, err:
231 Errors.report_error(err)
233 except InternalError, err:
234 # Only raise if there was not an earlier error
235 if Errors.num_errors == 0:
240 def find_module(self, module_name,
241 relative_to = None, pos = None, need_pxd = 1):
242 # Finds and returns the module scope corresponding to
243 # the given relative or absolute module name. If this
244 # is the first time the module has been requested, finds
245 # the corresponding .pxd file and process it.
246 # If relative_to is not None, it must be a module scope,
247 # and the module will first be searched for relative to
248 # that module, provided its name is not a dotted name.
249 debug_find_module = 0
250 if debug_find_module:
251 print("Context.find_module: module_name = %s, relative_to = %s, pos = %s, need_pxd = %s" % (
252 module_name, relative_to, pos, need_pxd))
256 if not module_name_pattern.match(module_name):
258 pos = (module_name, 0, 0)
259 raise CompileError(pos,
260 "'%s' is not a valid module name" % module_name)
261 if "." not in module_name and relative_to:
262 if debug_find_module:
263 print("...trying relative import")
264 scope = relative_to.lookup_submodule(module_name)
266 qualified_name = relative_to.qualify_name(module_name)
267 pxd_pathname = self.find_pxd_file(qualified_name, pos)
269 scope = relative_to.find_submodule(module_name)
271 if debug_find_module:
272 print("...trying absolute import")
274 for name in module_name.split("."):
275 scope = scope.find_submodule(name)
276 if debug_find_module:
277 print("...scope =", scope)
278 if not scope.pxd_file_loaded:
279 if debug_find_module:
280 print("...pxd not loaded")
281 scope.pxd_file_loaded = 1
283 if debug_find_module:
284 print("...looking for pxd file")
285 pxd_pathname = self.find_pxd_file(module_name, pos)
286 if debug_find_module:
287 print("......found ", pxd_pathname)
288 if not pxd_pathname and need_pxd:
289 package_pathname = self.search_include_directories(module_name, ".py", pos)
290 if package_pathname and package_pathname.endswith('__init__.py'):
293 error(pos, "'%s.pxd' not found" % module_name)
296 if debug_find_module:
297 print("Context.find_module: Parsing %s" % pxd_pathname)
298 source_desc = FileSourceDescriptor(pxd_pathname)
299 err, result = self.process_pxd(source_desc, scope, module_name)
302 (pxd_codenodes, pxd_scope) = result
303 self.pxds[module_name] = (pxd_codenodes, pxd_scope)
308 def find_pxd_file(self, qualified_name, pos):
309 # Search include path for the .pxd file corresponding to the
310 # given fully-qualified module name.
311 # Will find either a dotted filename or a file in a
312 # package directory. If a source file position is given,
313 # the directory containing the source file is searched first
314 # for a dotted filename, and its containing package root
315 # directory is searched first for a non-dotted filename.
316 return self.search_include_directories(qualified_name, ".pxd", pos)
318 def find_pyx_file(self, qualified_name, pos):
319 # Search include path for the .pyx file corresponding to the
320 # given fully-qualified module name, as for find_pxd_file().
321 return self.search_include_directories(qualified_name, ".pyx", pos)
323 def find_include_file(self, filename, pos):
324 # Search list of include directories for filename.
325 # Reports an error and returns None if not found.
326 path = self.search_include_directories(filename, "", pos,
329 error(pos, "'%s' not found" % filename)
332 def search_include_directories(self, qualified_name, suffix, pos,
334 # Search the list of include directories for the given
335 # file name. If a source file position is given, first
336 # searches the directory containing that file. Returns
337 # None if not found, but does not report an error.
338 # The 'include' option will disable package dereferencing.
339 dirs = self.include_directories
342 if not isinstance(file_desc, FileSourceDescriptor):
343 raise RuntimeError("Only file sources for code supported")
345 dirs = [os.path.dirname(file_desc.filename)] + dirs
347 dirs = [self.find_root_package_dir(file_desc.filename)] + dirs
349 dotted_filename = qualified_name
351 dotted_filename += suffix
353 names = qualified_name.split('.')
354 package_names = names[:-1]
355 module_name = names[-1]
356 module_filename = module_name + suffix
357 package_filename = "__init__" + suffix
360 path = os.path.join(dir, dotted_filename)
361 if Utils.path_exists(path):
364 package_dir = self.check_package_dir(dir, package_names)
365 if package_dir is not None:
366 path = os.path.join(package_dir, module_filename)
367 if Utils.path_exists(path):
369 path = os.path.join(dir, package_dir, module_name,
371 if Utils.path_exists(path):
375 def find_root_package_dir(self, file_path):
376 dir = os.path.dirname(file_path)
377 while self.is_package_dir(dir):
378 parent = os.path.dirname(dir)
384 def check_package_dir(self, dir, package_names):
385 for dirname in package_names:
386 dir = os.path.join(dir, dirname)
387 if not self.is_package_dir(dir):
391 def c_file_out_of_date(self, source_path):
392 c_path = Utils.replace_suffix(source_path, ".c")
393 if not os.path.exists(c_path):
395 c_time = Utils.modification_time(c_path)
396 if Utils.file_newer_than(source_path, c_time):
399 pxd_path = Utils.replace_suffix(source_path, ".pxd")
400 if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time):
402 for kind, name in self.read_dependency_file(source_path):
403 if kind == "cimport":
404 dep_path = self.find_pxd_file(name, pos)
405 elif kind == "include":
406 dep_path = self.search_include_directories(name, pos)
409 if dep_path and Utils.file_newer_than(dep_path, c_time):
413 def find_cimported_module_names(self, source_path):
414 return [ name for kind, name in self.read_dependency_file(source_path)
415 if kind == "cimport" ]
417 def is_package_dir(self, dir_path):
418 # Return true if the given directory is a package directory.
419 for filename in ("__init__.py",
422 path = os.path.join(dir_path, filename)
423 if Utils.path_exists(path):
426 def read_dependency_file(self, source_path):
427 dep_path = Utils.replace_suffix(source_path, ".dep")
428 if os.path.exists(dep_path):
429 f = open(dep_path, "rU")
430 chunks = [ line.strip().split(" ", 1)
431 for line in f.readlines()
432 if " " in line.strip() ]
438 def lookup_submodule(self, name):
439 # Look up a top-level module. Returns None if not found.
440 return self.modules.get(name, None)
442 def find_submodule(self, name):
443 # Find a top-level module, creating a new one if needed.
444 scope = self.lookup_submodule(name)
446 scope = ModuleScope(name,
447 parent_module = None, context = self)
448 self.modules[name] = scope
451 def parse(self, source_desc, scope, pxd, full_module_name):
452 if not isinstance(source_desc, FileSourceDescriptor):
453 raise RuntimeError("Only file sources for code supported")
454 source_filename = Utils.encode_filename(source_desc.filename)
456 # Parse the given source file and return a parse tree.
458 f = Utils.open_source_file(source_filename, "rU")
460 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
461 scope = scope, context = self)
462 tree = Parsing.p_module(s, pxd, full_module_name)
465 except UnicodeDecodeError, msg:
467 #traceback.print_exc()
468 error((source_desc, 0, 0), "Decoding error, missing or incorrect coding=<encoding-name> at top of source (%s)" % msg)
469 if Errors.num_errors > 0:
473 def extract_module_name(self, path, options):
474 # Find fully_qualified module name from the full pathname
476 dir, filename = os.path.split(path)
477 module_name, _ = os.path.splitext(filename)
478 if "." in module_name:
480 if module_name == "__init__":
481 dir, module_name = os.path.split(dir)
482 names = [module_name]
483 while self.is_package_dir(dir):
484 parent, package_name = os.path.split(dir)
487 names.append(package_name)
490 return ".".join(names)
492 def setup_errors(self, options, result):
493 if options.use_listing_file:
494 result.listing_file = Utils.replace_suffix(source, ".lis")
495 path = result.listing_file
498 Errors.open_listing_file(path=path,
499 echo_to_stderr=options.errors_to_stderr)
501 def teardown_errors(self, err, options, result):
502 source_desc = result.compilation_source.source_desc
503 if not isinstance(source_desc, FileSourceDescriptor):
504 raise RuntimeError("Only file sources for code supported")
505 Errors.close_listing_file()
506 result.num_errors = Errors.num_errors
507 if result.num_errors > 0:
509 if err and result.c_file:
511 Utils.castrate_file(result.c_file, os.stat(source_desc.filename))
512 except EnvironmentError:
516 def create_parse(context):
518 source_desc = compsrc.source_desc
519 full_module_name = compsrc.full_module_name
520 initial_pos = (source_desc, 1, 0)
521 scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
522 tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
523 tree.compilation_source = compsrc
529 def create_default_resultobj(compilation_source, options):
530 result = CompilationResult()
531 result.main_source_file = compilation_source.source_desc.filename
532 result.compilation_source = compilation_source
533 source_desc = compilation_source.source_desc
534 if options.output_file:
535 result.c_file = os.path.join(compilation_source.cwd, options.output_file)
541 result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
544 def run_pipeline(source, options, full_module_name = None):
546 context = Context(options.include_path, options.compiler_directives, options.cplus)
548 # Set up source object
550 source_desc = FileSourceDescriptor(os.path.join(cwd, source))
551 full_module_name = full_module_name or context.extract_module_name(source, options)
552 source = CompilationSource(source_desc, full_module_name, cwd)
554 # Set up result object
555 result = create_default_resultobj(source, options)
558 if source_desc.filename.endswith(".py"):
559 pipeline = context.create_py_pipeline(options, result)
561 pipeline = context.create_pyx_pipeline(options, result)
563 context.setup_errors(options, result)
564 err, enddata = context.run_pipeline(pipeline, source)
565 context.teardown_errors(err, options, result)
568 #------------------------------------------------------------------------
570 # Main Python entry points
572 #------------------------------------------------------------------------
574 class CompilationSource(object):
576 Contains the data necesarry to start up a compilation pipeline for
577 a single compilation unit.
579 def __init__(self, source_desc, full_module_name, cwd):
580 self.source_desc = source_desc
581 self.full_module_name = full_module_name
584 class CompilationOptions(object):
586 Options to the Cython compiler:
588 show_version boolean Display version number
589 use_listing_file boolean Generate a .lis file
590 errors_to_stderr boolean Echo errors to stderr when using .lis
591 include_path [string] Directories to search for include files
592 output_file string Name of generated .c file
593 generate_pxi boolean Generate .pxi file for public declarations
594 recursive boolean Recursively find and compile dependencies
595 timestamps boolean Only compile changed source files. If None,
596 defaults to true when recursive is true.
597 verbose boolean Always print source names being compiled
598 quiet boolean Don't print source names in recursive mode
599 compiler_directives dict Overrides for pragma options (see Options.py)
600 evaluate_tree_assertions boolean Test support: evaluate parse tree assertions
602 cplus boolean Compile as c++ code
605 def __init__(self, defaults = None, **kw):
606 self.include_path = []
608 if isinstance(defaults, CompilationOptions):
609 defaults = defaults.__dict__
611 defaults = default_options
612 self.__dict__.update(defaults)
613 self.__dict__.update(kw)
616 class CompilationResult(object):
618 Results from the Cython compiler:
620 c_file string or None The generated C source file
621 h_file string or None The generated C header file
622 i_file string or None The generated .pxi file
623 api_file string or None The generated C API .h file
624 listing_file string or None File of error messages
625 object_file string or None Result of compiling the C file
626 extension_file string or None Result of linking the object file
627 num_errors integer Number of compilation errors
628 compilation_source CompilationSource
636 self.listing_file = None
637 self.object_file = None
638 self.extension_file = None
639 self.main_source_file = None
642 class CompilationResultSet(dict):
644 Results from compiling multiple Pyrex source files. A mapping
645 from source file paths to CompilationResult instances. Also
646 has the following attributes:
648 num_errors integer Total number of compilation errors
653 def add(self, source, result):
654 self[source] = result
655 self.num_errors += result.num_errors
658 def compile_single(source, options, full_module_name = None):
660 compile_single(source, options, full_module_name)
662 Compile the given Pyrex implementation file and return a CompilationResult.
663 Always compiles a single file; does not perform timestamp checking or
666 return run_pipeline(source, options, full_module_name)
669 def compile_multiple(sources, options):
671 compile_multiple(sources, options)
673 Compiles the given sequence of Pyrex implementation files and returns
674 a CompilationResultSet. Performs timestamp checking and/or recursion
675 if these are specified in the options.
677 sources = [os.path.abspath(source) for source in sources]
679 results = CompilationResultSet()
680 recursive = options.recursive
681 timestamps = options.timestamps
682 if timestamps is None:
683 timestamps = recursive
684 verbose = options.verbose or ((recursive or timestamps) and not options.quiet)
685 for source in sources:
686 if source not in processed:
687 # Compiling multiple sources in one context doesn't quite
689 if not timestamps or context.c_file_out_of_date(source):
691 sys.stderr.write("Compiling %s\n" % source)
693 result = run_pipeline(source, options)
694 results.add(source, result)
695 processed.add(source)
697 for module_name in context.find_cimported_module_names(source):
698 path = context.find_pyx_file(module_name, [source])
703 "Cannot find .pyx file for cimported module '%s'\n" % module_name)
706 def compile(source, options = None, full_module_name = None, **kwds):
708 compile(source [, options], [, <option> = <value>]...)
710 Compile one or more Pyrex implementation files, with optional timestamp
711 checking and recursing on dependecies. The source argument may be a string
712 or a sequence of strings If it is a string and no recursion or timestamp
713 checking is requested, a CompilationResult is returned, otherwise a
714 CompilationResultSet is returned.
716 options = CompilationOptions(defaults = options, **kwds)
717 if isinstance(source, basestring) and not options.timestamps \
718 and not options.recursive:
719 return compile_single(source, options, full_module_name)
721 return compile_multiple(source, options)
723 #------------------------------------------------------------------------
725 # Main command-line entry point
727 #------------------------------------------------------------------------
728 def setuptools_main():
729 return main(command_line = 1)
731 def main(command_line = 0):
735 from CmdLine import parse_command_line
736 options, sources = parse_command_line(args)
738 options = CompilationOptions(default_options)
741 if options.show_version:
742 sys.stderr.write("Cython version %s\n" % Version.version)
743 if options.working_path!="":
744 os.chdir(options.working_path)
746 result = compile(sources, options)
747 if result.num_errors > 0:
749 except (EnvironmentError, PyrexError), e:
750 sys.stderr.write(str(e) + '\n')
757 #------------------------------------------------------------------------
759 # Set the default options depending on the platform
761 #------------------------------------------------------------------------
763 default_options = dict(
765 use_listing_file = 0,
766 errors_to_stderr = 1,
776 compiler_directives = {},
777 evaluate_tree_assertions = False,
778 emit_linenums = False,