From 38a9a438266fbbdc7c924617c5885961ecbb2328 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 9 Jul 2008 17:05:16 +0000 Subject: [PATCH] Merged from trunk 10988:10996 | 10989 | * Call self._wait() when unregistering output handlers, in | | zmedico | order to avoid triggering a tight loop. Also fix | | | Subprocess._wait() to only schedule when self.registered is | | | True. * Add an assertion inside Scheduler._schedule_main() | | | to try and detect tight loops like the one above. * Fix typo | | | in Scheduler._merge_exit(). | | 10990 | Remove redundant call to _schedule_main(). | | zmedico | | | 10991 | Fix Scheduler._dependent_on_scheduled_merges() to properly | | zmedico | distinguish "nomerge" nodes from "uninstall" nodes. | | 10992 | In scheduler._schedule_main(), only consider a poll() event | | zmedico | to be a state change if it results in a handler being | | | unregistered. | | 10993 | Make the "tight loop" assertion trigger whenever there is no | | zmedico | state change detected for a given _schedule_main() call. | | 10994 | * Fix _choose_pkg() to pop the first node if there are no | | zmedico | other jobs running. * Allow _choose_pkg() to choose an | | | uninstall node when it reaches the front of the queue. TODO: | | | fix order for uninstall operations. * Fix | | | _dependent_on_scheduled_merges() to handle direct circular | | | deps correctly. | | 10995 | Fix _prune_digraph() to correctly distinguish "nomerge" | | zmedico | nodes from "uninstall" nodes. | | 10996 | Fix Scheduler._choose_pkg() to perform uninstallation | | zmedico | actions (due to blockers) in the correct order when building | | | in parallel. | svn path=/main/branches/prefix/; revision=11000 --- pym/_emerge/__init__.py | 71 +++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 518b260f9..3cd49cf01 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1751,7 +1751,8 @@ class SubProcess(AsynchronousTask): def _wait(self): if self.returncode is not None: return self.returncode - self.scheduler.schedule(self._reg_id) + if self.registered: + self.scheduler.schedule(self._reg_id) self._set_returncode(os.waitpid(self.pid, 0)) return self.returncode @@ -1873,6 +1874,7 @@ class SpawnProcess(SubProcess): f.flush() f.close() self.registered = False + self._wait() return self.registered def _dummy_handler(self, fd, event): @@ -1894,6 +1896,7 @@ class SpawnProcess(SubProcess): for f in files.values(): f.close() self.registered = False + self._wait() return self.registered class EbuildFetcher(SpawnProcess): @@ -2358,6 +2361,7 @@ class EbuildPhase(SubProcess): for f in files.values(): f.close() self.registered = False + self._wait() return self.registered def _dummy_handler(self, fd, event): @@ -2379,6 +2383,7 @@ class EbuildPhase(SubProcess): for f in files.values(): f.close() self.registered = False + self._wait() return self.registered def _set_returncode(self, wait_retval): @@ -7746,8 +7751,31 @@ class Scheduler(object): return self._digraph = digraph + self._reverse_uninstall_edges() self._prune_digraph() + def _reverse_uninstall_edges(self): + """ + The uninstall is performed only after blocking packages have been + merged on top of it (similar to how a normal upgrade is performed + by first merging the new version on top of the onld version). This + is implemented by reversing the the parent -> uninstall edges in + the graph. + """ + + graph = self._digraph + + for node in self._mergelist: + if not isinstance(node, Package) or \ + node.operation != "uninstall": + continue + + parent_nodes = graph.parent_nodes(node) + graph.remove(node) + for blocked_pkg in parent_nodes: + graph.add(blocked_pkg, node, + priority=BlockerDepPriority.instance) + def _prune_digraph(self): """ Prune any root nodes that are irrelevant. @@ -7759,7 +7787,8 @@ class Scheduler(object): while True: for node in graph.root_nodes(): if not isinstance(node, Package) or \ - node.installed or node.onlydeps or \ + (node.installed and node.operation == "nomerge") or \ + node.onlydeps or \ node in completed_tasks: removed_nodes.add(node) if removed_nodes: @@ -8075,7 +8104,7 @@ class Scheduler(object): self._job_exit(merge.merge) pkg = merge.merge.pkg if merge.returncode != os.EX_OK: - self._failed_pkgs.append((pkg, retval)) + self._failed_pkgs.append((pkg, merge.returncode)) return self._completed_tasks.add(pkg) @@ -8139,15 +8168,16 @@ class Scheduler(object): return rval def _choose_pkg(self): - if self._max_jobs < 2: + """ + Choose a task that has all it's dependencies satisfied. + """ + if self._max_jobs < 2 or self._jobs == 0: return self._pkg_queue.pop(0) self._prune_digraph() chosen_pkg = None for pkg in self._pkg_queue: - if pkg.operation == "uninstall": - continue if not self._dependent_on_scheduled_merges(pkg): chosen_pkg = pkg break @@ -8168,14 +8198,14 @@ class Scheduler(object): completed_tasks = self._completed_tasks dependent = False - traversed_nodes = set() + traversed_nodes = set([pkg]) node_stack = graph.child_nodes(pkg) while node_stack: node = node_stack.pop() if node in traversed_nodes: continue traversed_nodes.add(node) - if not node.installed and \ + if not (node.installed and node.operation == "nomerge") and \ node not in completed_tasks: dependent = True break @@ -8254,8 +8284,6 @@ class Scheduler(object): task.addExitListener(self._build_exit) task_queues.build.add(task) - self._schedule_main() - while self._jobs: self._schedule_main(wait=True) @@ -8265,7 +8293,10 @@ class Scheduler(object): poll = self._poll.poll max_jobs = self._max_jobs - self._schedule_tasks() + state_change = 0 + + if self._schedule_tasks(): + state_change += 1 while event_handlers: jobs = self._jobs @@ -8273,16 +8304,28 @@ class Scheduler(object): for f, event in poll(): handler, reg_id = event_handlers[f] if not handler(f, event): + state_change += 1 self._unregister(reg_id) if jobs == self._jobs: continue - self._schedule_tasks() + if self._schedule_tasks(): + state_change += 1 if not wait and self._jobs < max_jobs: break + if not state_change: + raise AssertionError("tight loop") + + def _schedule_tasks(self): + state_change = 0 + for x in self._task_queues.values(): + if x.schedule(): + state_change += 1 + return bool(state_change) + def _task(self, pkg, background): task = MergeListItem(args_set=self._args_set, @@ -8389,10 +8432,6 @@ class Scheduler(object): del self._poll_event_handler_ids[reg_id] self._schedule_tasks() - def _schedule_tasks(self): - for x in self._task_queues.values(): - x.schedule() - def _schedule(self, wait_id): """ Schedule until wait_id is not longer registered -- 2.26.2