From b73ebf6329e34349d633a5233e46de715a81c067 Mon Sep 17 00:00:00 2001 From: Robert Bradshaw Date: Thu, 28 Apr 2011 12:25:36 -0700 Subject: [PATCH] Reduce testrunner disk io. --- runtests.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/runtests.py b/runtests.py index eb2d340b..28ff15f4 100644 --- a/runtests.py +++ b/runtests.py @@ -135,6 +135,8 @@ def memoize(f): res = f._cache.get(args, uncomputed) if res is uncomputed: res = f._cache[args] = f(*args) + else: + print "dup", args return res return func @@ -159,6 +161,8 @@ def parse_tags(filepath): parse_tags = memoize(parse_tags) +list_unchanging_dir = memoize(lambda x: os.listdir(x)) + class build_ext(_build_ext): def build_extension(self, ext): @@ -245,7 +249,7 @@ class TestBuilder(object): os.makedirs(workdir) suite = unittest.TestSuite() - filenames = os.listdir(path) + filenames = list_unchanging_dir(path) filenames.sort() for filename in filenames: filepath = os.path.join(path, filename) @@ -414,19 +418,21 @@ class CythonCompileTestCase(unittest.TestCase): def build_target_filename(self, module_name): target = '%s.%s' % (module_name, self.language) return target - - def copy_related_files(self, test_directory, target_directory, module_name): + + def related_files(self, test_directory, module_name): is_related = re.compile('%s_.*[.].*' % module_name).match - for filename in os.listdir(test_directory): - if is_related(filename): - shutil.copy(os.path.join(test_directory, filename), - target_directory) + return [filename for filename in list_unchanging_dir(test_directory) + if is_related(filename)] + + def copy_files(self, test_directory, target_directory, file_list): + for filename in file_list: + shutil.copy(os.path.join(test_directory, filename), + target_directory) - def find_source_files(self, workdir, module_name): - is_related = re.compile('%s_.*[.]%s' % (module_name, self.language)).match - return [self.build_target_filename(module_name)] + [ - filename for filename in os.listdir(workdir) - if is_related(filename) and os.path.isfile(os.path.join(workdir, filename)) ] + def source_files(self, workdir, module_name, file_list): + return ([self.build_target_filename(module_name)] + + [filename for filename in file_list + if not os.path.isfile(os.path.join(workdir, filename))]) def split_source_and_output(self, test_directory, module, workdir): source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx') @@ -507,10 +513,11 @@ class CythonCompileTestCase(unittest.TestCase): if extra_extension_args is None: extra_extension_args = {} - self.copy_related_files(test_directory, workdir, module) + related_files = self.related_files(test_directory, module) + self.copy_files(test_directory, workdir, related_files) extension = Extension( module, - sources = self.find_source_files(workdir, module), + sources = self.source_files(workdir, module, related_files), include_dirs = ext_include_dirs, extra_compile_args = ext_compile_flags, **extra_extension_args -- 2.26.2