From f6a3f4c64402069a48608ac94f78955ef8901751 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 4 Jun 2008 18:55:23 +0200 Subject: [PATCH] merged (and partially rewrote) dependency tracking and package resolution changes from Pyrex 0.9.8 --- Cython/Compiler/CmdLine.py | 10 ++ Cython/Compiler/Main.py | 247 ++++++++++++++++++++++++++-------- Cython/Compiler/ModuleNode.py | 15 +++ Cython/Compiler/Parsing.py | 4 +- Cython/Compiler/Scanning.py | 8 +- Cython/Compiler/Symtab.py | 6 +- Cython/Utils.py | 9 +- 7 files changed, 235 insertions(+), 64 deletions(-) diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index 11caafc8..16315d0b 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -18,6 +18,10 @@ Options: -I, --include-dir Search for include files in named directory (multiply include directories are allowed). -o, --output-file Specify name of generated C file + -r, --recursive Recursively find and compile dependencies + -t, --timestamps Only compile newer source files (implied with -r) + -f, --force Compile all source files (overrides implied -t) + -q, --quiet Don't print module names in recursive mode -p, --embed-positions If specified, the positions in Cython files of each function definition is embedded in its docstring. -z, --pre-import If specified, assume undeclared names in this @@ -111,6 +115,12 @@ def parse_command_line(args): options.working_path = pop_arg() elif option in ("-o", "--output-file"): options.output_file = pop_arg() + elif option in ("-r", "--recursive"): + options.recursive = 1 + elif option in ("-t", "--timestamps"): + options.timestamps = 1 + elif option in ("-f", "--force"): + options.timestamps = 0 elif option in ("-p", "--embed-positions"): Options.embed_pos_in_docstring = 1 elif option in ("-z", "--pre-import"): diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 0782f52f..d7a3d6c6 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -3,8 +3,8 @@ # import os, sys, re, codecs -if sys.version_info[:2] < (2, 2): - sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n") +if sys.version_info[:2] < (2, 3): + sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n") sys.exit(1) try: @@ -14,14 +14,13 @@ except NameError: from sets import Set as set from time import time -import Version -from Scanning import PyrexScanner +import Code import Errors -from Errors import PyrexError, CompileError, error import Parsing +import Version +from Errors import PyrexError, CompileError, error +from Scanning import PyrexScanner from Symtab import BuiltinScope, ModuleScope -import Code -from Cython.Utils import replace_suffix from Cython import Utils import Transform @@ -93,31 +92,34 @@ class Context: try: if debug_find_module: print("Context.find_module: Parsing %s" % pxd_pathname) - pxd_tree = self.parse(pxd_pathname, scope.type_names, pxd = 1, + pxd_tree = self.parse(pxd_pathname, scope, pxd = 1, full_module_name = module_name) pxd_tree.analyse_declarations(scope) except CompileError: pass return scope - def find_pxd_file(self, module_name, pos): - # Search include directories for the .pxd file - # corresponding to the given (full) module name. - if "." in module_name: - pxd_filename = "%s.pxd" % os.path.join(*module_name.split('.')) - else: - pxd_filename = "%s.pxd" % module_name - return self.search_include_directories(pxd_filename, pos) + def find_pxd_file(self, qualified_name, pos): + # Search include path for the .pxd file corresponding to the + # given fully-qualified module name. + return self.search_include_directories(qualified_name, ".pxd", pos) + + def find_pyx_file(self, qualified_name, pos): + # Search include path for the .pyx file corresponding to the + # given fully-qualified module name, as for find_pxd_file(). + return self.search_include_directories(qualified_name, ".pyx", pos) def find_include_file(self, filename, pos): # Search list of include directories for filename. # Reports an error and returns None if not found. - path = self.search_include_directories(filename, pos) + path = self.search_include_directories(filename, "", pos, + split_package=False) if not path: error(pos, "'%s' not found" % filename) return path - def search_include_directories(self, filename, pos): + def search_include_directories(self, qualified_name, suffix, pos, + split_package=True): # Search the list of include directories for the given # file name. If a source file position is given, first # searches the directory containing that file. Returns @@ -126,12 +128,81 @@ class Context: if pos: here_dir = os.path.dirname(pos[0]) dirs = [here_dir] + dirs + + dotted_filename = qualified_name + suffix + if split_package: + names = qualified_name.split('.') + package_names = names[:-1] + module_name = names[-1] + module_filename = module_name + suffix + package_filename = "__init__" + suffix + for dir in dirs: - path = os.path.join(dir, filename) + path = os.path.join(dir, dotted_filename) if os.path.exists(path): return path + if split_package: + package_dir = self.check_package_dir(dir, package_names) + if package_dir is not None: + path = os.path.join(package_dir, module_filename) + if os.path.exists(path): + return path + path = os.path.join(dir, package_dir, module_name, + package_filename) + if os.path.exists(path): + return path return None + def check_package_dir(self, dir, package_names): + package_dir = os.path.join(dir, *package_names) + if not os.path.exists(package_dir): + return None + for dirname in package_names: + dir = os.path.join(dir, dirname) + package_init = os.path.join(dir, "__init__.py") + if not os.path.exists(package_init) and \ + not os.path.exists(package_init + "x"): # same with .pyx ? + return None + return package_dir + + def c_file_out_of_date(self, source_path): + c_path = Utils.replace_suffix(source_path, ".c") + if not os.path.exists(c_path): + return 1 + c_time = Utils.modification_time(c_path) + if Utils.file_newer_than(source_path, c_time): + return 1 + pos = [source_path] + pxd_path = Utils.replace_suffix(source_path, ".pxd") + if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): + return 1 + for kind, name in self.read_dependency_file(source_path): + if kind == "cimport": + dep_path = self.find_pxd_file(name, pos) + elif kind == "include": + dep_path = self.search_include_directories(name, pos) + else: + continue + if dep_path and Utils.file_newer_than(dep_path, c_time): + return 1 + return 0 + + def find_cimported_module_names(self, source_path): + return [ name for kind, name in self.read_dependency_file(source_path) + if kind == "cimport" ] + + def read_dependency_file(self, source_path): + dep_path = replace_suffix(source_path, ".dep") + if os.path.exists(dep_path): + f = open(dep_path, "rU") + chunks = [ line.strip().split(" ", 1) + for line in f.readlines() + if " " in line.strip() ] + f.close() + return chunks + else: + return () + def lookup_submodule(self, name): # Look up a top-level module. Returns None if not found. return self.modules.get(name, None) @@ -145,14 +216,14 @@ class Context: self.modules[name] = scope return scope - def parse(self, source_filename, type_names, pxd, full_module_name): + def parse(self, source_filename, scope, pxd, full_module_name): name = Utils.encode_filename(source_filename) # Parse the given source file and return a parse tree. try: f = Utils.open_source_file(source_filename, "rU") try: s = PyrexScanner(f, name, source_encoding = f.encoding, - type_names = type_names, context = self) + scope = scope, context = self) tree = Parsing.p_module(s, pxd, full_module_name) finally: f.close() @@ -185,7 +256,7 @@ class Context: result.main_source_file = source if options.use_listing_file: - result.listing_file = replace_suffix(source, ".lis") + result.listing_file = Utils.replace_suffix(source, ".lis") Errors.open_listing_file(result.listing_file, echo_to_stderr = options.errors_to_stderr) else: @@ -197,19 +268,14 @@ class Context: c_suffix = ".cpp" else: c_suffix = ".c" - result.c_file = replace_suffix(source, c_suffix) - c_stat = None - if result.c_file: - try: - c_stat = os.stat(result.c_file) - except EnvironmentError: - pass + result.c_file = Utils.replace_suffix(source, c_suffix) module_name = full_module_name # self.extract_module_name(source, options) initial_pos = (source, 1, 0) scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0) errors_occurred = False try: - tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name) + tree = self.parse(source, scope, pxd = 0, + full_module_name = full_module_name) tree.process_implementation(scope, options, result) except CompileError: errors_occurred = True @@ -219,8 +285,7 @@ class Context: errors_occurred = True if errors_occurred and result.c_file: try: - #os.unlink(result.c_file) - Utils.castrate_file(result.c_file, c_stat) + Utils.castrate_file(result.c_file, os.stat(source)) except EnvironmentError: pass result.c_file = None @@ -237,7 +302,7 @@ class Context: #------------------------------------------------------------------------ # -# Main Python entry point +# Main Python entry points # #------------------------------------------------------------------------ @@ -251,6 +316,10 @@ class CompilationOptions: include_path [string] Directories to search for include files output_file string Name of generated .c file generate_pxi boolean Generate .pxi file for public declarations + recursive boolean Recursively find and compile dependencies + timestamps boolean Only compile changed source files. If None, + defaults to true when recursive is true. + quiet boolean Don't print source names in recursive mode transforms Transform.TransformSet Transforms to use on the parse tree Following options are experimental and only used on MacOSX: @@ -261,7 +330,7 @@ class CompilationOptions: cplus boolean Compile as c++ code """ - def __init__(self, defaults = None, **kw): + def __init__(self, defaults = None, c_compile = 0, c_link = 0, **kw): self.include_path = [] self.objects = [] if defaults: @@ -271,6 +340,10 @@ class CompilationOptions: defaults = default_options self.__dict__.update(defaults) self.__dict__.update(kw) + if c_compile: + self.c_only = 0 + if c_link: + self.obj_only = 0 class CompilationResult: @@ -298,24 +371,87 @@ class CompilationResult: self.main_source_file = None -def compile(source, options = None, c_compile = 0, c_link = 0, - full_module_name = None): +class CompilationResultSet(dict): """ - compile(source, options = default_options) + Results from compiling multiple Pyrex source files. A mapping + from source file paths to CompilationResult instances. Also + has the following attributes: - Compile the given Cython implementation file and return - a CompilationResult object describing what was produced. + num_errors integer Total number of compilation errors + """ + + num_errors = 0 + + def add(self, source, result): + self[source] = result + self.num_errors += result.num_errors + + +def compile_single(source, options, full_module_name = None): + """ + compile_single(source, options, full_module_name) + + Compile the given Pyrex implementation file and return a CompilationResult. + Always compiles a single file; does not perform timestamp checking or + recursion. """ - if not options: - options = default_options - options = CompilationOptions(defaults = options) - if c_compile: - options.c_only = 0 - if c_link: - options.obj_only = 0 context = Context(options.include_path) return context.compile(source, options, full_module_name) +def compile_multiple(sources, options): + """ + compile_multiple(sources, options) + + Compiles the given sequence of Pyrex implementation files and returns + a CompilationResultSet. Performs timestamp checking and/or recursion + if these are specified in the options. + """ + sources = [os.path.abspath(source) for source in sources] + processed = set() + results = CompilationResultSet() + context = Context(options.include_path) + recursive = options.recursive + timestamps = options.timestamps + if timestamps is None: + timestamps = recursive + verbose = recursive and not options.quiet + for source in sources: + if source not in processed: + if not timestamps or context.c_file_out_of_date(source): + if verbose: + sys.stderr.write("Compiling %s\n" % source) + result = context.compile(source, options) + results.add(source, result) + processed.add(source) + if recursive: + for module_name in context.find_cimported_module_names(source): + path = context.find_pyx_file(module_name, [source]) + if path: + sources.append(path) + else: + sys.stderr.write( + "Cannot find .pyx file for cimported module '%s'\n" % module_name) + return results + +def compile(source, options = None, c_compile = 0, c_link = 0, + full_module_name = None, **kwds): + """ + compile(source [, options], [,