Merged from trunk 10816:10824
authorFabian Groffen <grobian@gentoo.org>
Tue, 1 Jul 2008 16:48:46 +0000 (16:48 -0000)
committerFabian Groffen <grobian@gentoo.org>
Tue, 1 Jul 2008 16:48:46 +0000 (16:48 -0000)
   | 10817   | Generate and log eerror messages for any packages that need  |
   | zmedico | to be dropped due to unsatisfied dependencies when           |
   |         | --keep-going is enabled.                                     |

   | 10818   | Add documentation for the new --keep-going option.           |
   | zmedico |                                                              |

   | 10819   | Fix typo.                                                    |
   | zmedico |                                                              |

   | 10820   | Pass the digraph into the MergeTask constructor. It's        |
   | zmedico | currently unused but it will be useful for parallel          |
   |         | scheduling.                                                  |

   | 10821   | Break references from Package instances in the digraph       |
   | zmedico | before passing it into the MergeTask constructor.            |

   | 10822   | Remove the MergeTask._blocker_db attribute since it can just |
   | zmedico | be a local variable.                                         |

   | 10823   | Fix depgraph.break_refs() to work with DependencyArg         |
   | zmedico | instances (or anything else that may have a "root_config"    |
   |         | attribute).                                                  |

   | 10824   | Bug #226307 - Copy come code from                            |
   | zmedico | depgraph._iter_atoms_for_pkg() that was used to solve bug    |
   |         | #218854, and use it inside unmerge() when matching sets to   |
   |         | packages.                                                    |

svn path=/main/branches/prefix/; revision=10876

NEWS
man/emerge.1
pym/_emerge/__init__.py
pym/_emerge/help.py

diff --git a/NEWS b/NEWS
index 45dc3798ecae3933c324a2d915cd9f1684ec67f0..7463b463c9cd15c653111d54139c626909f1dfc6 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,11 @@ News (mainly features/major bug fixes)
 portage-2.2
 -------------
 
+* Add emerge --keep-going option to continue as much as possible after
+  an error. When an error occurs, dependencies are recalculated for
+  remaining packages and any with unsatisfied dependencies are
+  automatically dropped. The --skipfirst option automatically drops
+  packages in the same way, and also drops any masked packages.
 * Add subversion support for repoman.
 * It is now possible to use `emerge <file>` to reinstall the package that
   installed a particular file. Package contents entries are indexed to
index 4f163701cdc7490475720a30e4b5c35f991e2505..2bbcc0850bf466cfc18f45450b2a02c2c50989ae 100644 (file)
@@ -321,6 +321,12 @@ directory.
 .BR "\-\-ignore-default-opts"
 Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored.
 .TP
+.BR "\-\-keep\-going"
+Continue as much as possible after an error. When an error occurs,
+dependencies are recalculated for remaining packages and any with
+unsatisfied dependencies are automatically dropped. Also see
+the related \fB\-\-skipfirst\fR option.
+.TP
 .BR "\-\-newuse " (\fB\-N\fR)
 Tells emerge to include installed packages where USE flags have changed since 
 compilation.  USE flag changes include:
@@ -399,9 +405,10 @@ enabled are added or removed.
 .TP
 .BR "\-\-skipfirst"
 This option is only valid when used with \fB\-\-resume\fR.  It removes the 
-first package in the resume list so that a merge may continue in the presence 
-of an uncorrectable or inconsequential error.  This should only be used in 
-cases where skipping the package will not result in failed dependencies.
+first package in the resume list. Dependencies are recalculated for
+remaining packages and any that have unsatisfied dependencies or are
+masked will be automatically dropped. Also see the related
+\fB\-\-keep\-going\fR option.
 .TP
 .BR "\-\-tree " (\fB\-t\fR)
 Shows the dependency tree for the given target by indenting dependencies.
index e6876205b9efc5752e9d06492e256f6b34782efb..4e7fe38ff3888f26228a6539eba11275f496b20b 100644 (file)
@@ -67,6 +67,7 @@ import portage.locks
 import portage.exception
 from portage.const import EPREFIX, BPREFIX
 from portage.data import secpass
+from portage.elog.messages import eerror
 from portage.util import normalize_path as normpath
 from portage.util import writemsg
 from portage.sets import load_default_config, SETPREFIX
@@ -3892,25 +3893,23 @@ class depgraph(object):
                        retlist.reverse()
                return retlist
 
-       def break_refs(self, mergelist):
+       def break_refs(self, nodes):
                """
                Take a mergelist like that returned from self.altlist() and
                break any references that lead back to the depgraph. This is
                useful if you want to hold references to packages without
                also holding the depgraph on the heap.
                """
-               for node in mergelist:
-                       if not isinstance(node, Package):
-                               continue
-
-                       # The FakeVartree references the _package_cache which
-                       # references the depgraph. So that Package instances don't
-                       # hold the depgraph and FakeVartree on the heap, replace
-                       # the RootConfig that references the FakeVartree with the
-                       # original RootConfig instance which references the actual
-                       # vartree.
-                       node.root_config = \
-                               self._trees_orig[node.root]["root_config"]
+               for node in nodes:
+                       if hasattr(node, "root_config"):
+                               # The FakeVartree references the _package_cache which
+                               # references the depgraph. So that Package instances don't
+                               # hold the depgraph and FakeVartree on the heap, replace
+                               # the RootConfig that references the FakeVartree with the
+                               # original RootConfig instance which references the actual
+                               # vartree.
+                               node.root_config = \
+                                       self._trees_orig[node.root_config.root]["root_config"]
 
        def _resolve_conflicts(self):
                if not self._complete_graph():
@@ -5914,7 +5913,7 @@ class MergeTask(object):
                "--nodeps", "--pretend"])
 
        def __init__(self, settings, trees, mtimedb, myopts,
-               spinner, mergelist, favorites):
+               spinner, mergelist, favorites, digraph):
                self.settings = settings
                self.target_root = settings["ROOT"]
                self.trees = trees
@@ -5927,11 +5926,9 @@ class MergeTask(object):
                if settings.get("PORTAGE_DEBUG", "") == "1":
                        self.edebug = 1
                self.pkgsettings = {}
-               self._blocker_db = {}
                for root in trees:
                        self.pkgsettings[root] = portage.config(
                                clone=trees[root]["vartree"].settings)
-                       self._blocker_db[root] = BlockerDB(trees[root]["root_config"])
                self.curval = 0
                self._spawned_pids = []
 
@@ -5954,10 +5951,11 @@ class MergeTask(object):
                import gc
                gc.collect()
 
+               blocker_db = BlockerDB(self.trees[new_pkg.root]["root_config"])
+
                blocker_dblinks = []
-               for blocking_pkg in self._blocker_db[
-                       new_pkg.root].findInstalledBlockers(new_pkg,
-                       acquire_lock=acquire_lock):
+               for blocking_pkg in blocker_db.findInstalledBlockers(
+                       new_pkg, acquire_lock=acquire_lock):
                        if new_pkg.slot_atom == blocking_pkg.slot_atom:
                                continue
                        if new_pkg.cpv == blocking_pkg.cpv:
@@ -5999,15 +5997,40 @@ class MergeTask(object):
                                break
                        if mergelist[0][-1] != "merge":
                                break
+
                        # Skip the first one because it failed to build or install.
+                       pkg_key = tuple(mergelist[0])
                        del mergelist[0]
+                       failed_pkg = None
+                       for task in self._mergelist:
+                               if task == pkg_key:
+                                       failed_pkg = task
+                                       break
+                       if failed_pkg is None:
+                               break
                        if not mergelist:
                                break
-                       mylist = self._calc_resume_list()
+
+                       mylist, dropped_tasks = self._calc_resume_list()
                        clear_caches(self.trees)
                        if not mylist:
                                break
-                       self.curval += 1
+
+                       if dropped_tasks:
+
+                               def _eerror(lines):
+                                       for l in lines:
+                                               eerror(l, phase="other", key=failed_pkg.cpv)
+
+                               msg = []
+                               msg.append("One or more packages have been " + \
+                                       "dropped due to unsatisfied dependencies:")
+                               msg.append("")
+                               msg.extend("  " + str(task) for task in dropped_tasks)
+                               msg.append("")
+                               _eerror(msg)
+                               del _eerror, msg
+                       del dropped_tasks
                        self._mergelist = mylist
 
                return rval
@@ -6035,20 +6058,12 @@ class MergeTask(object):
 
                if not success:
                        mydepgraph.display_problems()
-                       return None
-
-               if dropped_tasks:
-                       portage.writemsg("!!! One or more packages have been " + \
-                               "dropped due to\n" + \
-                               "!!! masking or unsatisfied dependencies:\n\n",
-                               noiselevel=-1)
-                       for task in dropped_tasks:
-                               portage.writemsg("  " + str(task) + "\n", noiselevel=-1)
-                       portage.writemsg("\n", noiselevel=-1)
+                       return (None, None)
 
                mylist = mydepgraph.altlist()
                mydepgraph.break_refs(mylist)
-               return mylist
+               mydepgraph.break_refs(dropped_tasks)
+               return (mylist, dropped_tasks)
 
        def _poll_child_processes(self):
                """
@@ -6562,6 +6577,19 @@ def unmerge(root_config, myopts, unmerge_action,
        global_unmerge=0
        xterm_titles = "notitles" not in settings.features
 
+       pkg_cache = {}
+
+       def _pkg(cpv):
+               pkg = pkg_cache.get(cpv)
+               if pkg is None:
+                       pkg = Package(cpv=cpv, installed=True,
+                               metadata=izip(Package.metadata_keys,
+                                       vartree.dbapi.aux_get(cpv, Package.metadata_keys)),
+                               root_config=root_config,
+                               type_name="installed")
+                       pkg_cache[cpv] = pkg
+               return pkg
+
        vdb_path = os.path.join(settings["ROOT"], portage.VDB_PATH)
        try:
                # At least the parent needs to exist for the lock file.
@@ -6804,6 +6832,12 @@ def unmerge(root_config, myopts, unmerge_action,
        # relevant package sets.
        for cp in xrange(len(pkgmap)):
                for cpv in pkgmap[cp]["selected"].copy():
+                       try:
+                               pkg = _pkg(cpv)
+                       except KeyError:
+                               # It could have been uninstalled
+                               # by a concurrent process.
+                               continue
                        parents = []
                        for s in installed_sets:
                                # skip sets that the user requested to unmerge, and skip world 
@@ -6814,9 +6848,34 @@ def unmerge(root_config, myopts, unmerge_action,
                                # only check instances of EditablePackageSet as other classes are generally used for
                                # special purposes and can be ignored here (and are usually generated dynamically, so the
                                # user can't do much about them anyway)
-                               elif sets[s].containsCPV(cpv) \
-                                       and isinstance(sets[s], EditablePackageSet):
-                                       parents.append(s)
+                               if isinstance(sets[s], EditablePackageSet):
+
+                                       # This is derived from a snippet of code in the
+                                       # depgraph._iter_atoms_for_pkg() method.
+                                       for atom in sets[s].iterAtomsForPackage(pkg):
+                                               inst_matches = vartree.dbapi.match(atom)
+                                               inst_matches.reverse() # descending order
+                                               higher_slot = None
+                                               for inst_cpv in inst_matches:
+                                                       try:
+                                                               inst_pkg = _pkg(inst_cpv)
+                                                       except KeyError:
+                                                               # It could have been uninstalled
+                                                               # by a concurrent process.
+                                                               continue
+
+                                                       if inst_pkg.cp != atom.cp:
+                                                               continue
+                                                       if pkg >= inst_pkg:
+                                                               # This is descending order, and we're not
+                                                               # interested in any versions <= pkg given.
+                                                               break
+                                                       if pkg.slot_atom != inst_pkg.slot_atom:
+                                                               higher_slot = inst_pkg
+                                                               break
+                                               if higher_slot is None:
+                                                       parents.append(s)
+                                                       break
                        if parents:
                                #print colorize("WARN", "Package %s is going to be unmerged," % cpv)
                                #print colorize("WARN", "but still listed in the following package sets:")
@@ -8981,11 +9040,12 @@ def action_build(settings, trees, mtimedb,
                                time.sleep(3) # allow the parent to have first fetch
                        mymergelist = mydepgraph.altlist()
                        mydepgraph.break_refs(mymergelist)
+                       mydepgraph.break_refs(mydepgraph.digraph.order)
+                       mergetask = MergeTask(settings, trees, mtimedb, myopts,
+                               spinner, mymergelist, favorites, mydepgraph.digraph)
                        del mydepgraph
                        clear_caches(trees)
 
-                       mergetask = MergeTask(settings, trees, mtimedb, myopts,
-                               spinner, mymergelist, favorites)
                        retval = mergetask.merge()
                        merge_count = mergetask.curval
                else:
@@ -9027,11 +9087,12 @@ def action_build(settings, trees, mtimedb,
                        pkglist = mydepgraph.altlist()
                        mydepgraph.saveNomergeFavorites()
                        mydepgraph.break_refs(pkglist)
+                       mydepgraph.break_refs(mydepgraph.digraph.order)
+                       mergetask = MergeTask(settings, trees, mtimedb, myopts,
+                               spinner, pkglist, favorites, mydepgraph.digraph)
                        del mydepgraph
                        clear_caches(trees)
 
-                       mergetask = MergeTask(settings, trees, mtimedb, myopts,
-                               spinner, pkglist, favorites)
                        retval = mergetask.merge()
                        merge_count = mergetask.curval
 
index 1ec7e96bccfc3828ee8b95d0af52bf5770764085..27e150d882699c7ca4d4fbe0822d6b4baa2e56d9 100644 (file)
@@ -15,8 +15,8 @@ def shorthelp():
        print "   "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]"
        print "   "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] "
        print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]"
-       print "          [ "+green("--columns")+"   ] [ "+green("--deep")+"      ] [ "+green("--newuse")+"    ]"
-       print "          [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+"   ]"
+       print "          [ "+green("--columns")+"   ] [ "+green("--deep")+"      ] [ "+green("--keep-going")+" ] [ "+green("--newuse")+"   ]"
+       print "          [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+"    ]"
        print "          [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" >           ] [ "+green("--complete-graph")+"            ]"
        print "          [ "+green("--reinstall ")+turquoise("changed-use")+"     ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" >      ]"
        print bold("Actions:")+"  [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]"
@@ -292,6 +292,17 @@ def help(myaction,myopts,havecolor=1):
                print "              downloaded from the remote server without consulting packages"
                print "              existing in the packages directory."
                print
+               print "       "+green("--keep-going")
+               desc = "Continue as much as possible after " + \
+                       "an error. When an error occurs, " + \
+                       "dependencies are recalculated for " + \
+                       "remaining packages and any with " + \
+                       "unsatisfied dependencies are " + \
+                       "automatically dropped. Also see " + \
+                       "the related --skipfirst option."
+               for line in wrap(desc, desc_width):
+                       print desc_indent + line
+               print
                print "       "+green("--newuse")+" ("+green("-N")+" short option)"
                print "              Tells emerge to include installed packages where USE flags have "
                print "              changed since installation."
@@ -349,11 +360,17 @@ def help(myaction,myopts,havecolor=1):
                print "              enabled are added or removed."
                print
                print "       "+green("--skipfirst")
-               print "              This option is only valid in a resume situation. It removes the"
-               print "              first package in the resume list so that a merge may continue in"
-               print "              the presence of an uncorrectable or inconsequential error. This"
-               print "              should only be used in cases where skipping the package will not"
-               print "              result in failed dependencies."
+               desc = "This option is only valid when " + \
+                       "used with --resume.  It removes the " + \
+                       "first package in the resume list. " + \
+                       "Dependencies are recalculated for " + \
+                       "remaining packages and any that " + \
+                       "have unsatisfied dependencies or are " + \
+                       "masked will be automatically dropped. " + \
+                       "Also see the related " + \
+                       "--keep-going option."
+               for line in wrap(desc, desc_width):
+                       print desc_indent + line
                print
                print "       "+green("--tree")+" ("+green("-t")+" short option)"
                print "              Shows the dependency tree using indentation for dependencies."