From 2c966f8d2f58c5b569eb5cf0f3d4c9754a9a5fe1 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 1 Jul 2008 16:48:46 +0000 Subject: [PATCH] Merged from trunk 10816:10824 | 10817 | Generate and log eerror messages for any packages that need | | zmedico | to be dropped due to unsatisfied dependencies when | | | --keep-going is enabled. | | 10818 | Add documentation for the new --keep-going option. | | zmedico | | | 10819 | Fix typo. | | zmedico | | | 10820 | Pass the digraph into the MergeTask constructor. It's | | zmedico | currently unused but it will be useful for parallel | | | scheduling. | | 10821 | Break references from Package instances in the digraph | | zmedico | before passing it into the MergeTask constructor. | | 10822 | Remove the MergeTask._blocker_db attribute since it can just | | zmedico | be a local variable. | | 10823 | Fix depgraph.break_refs() to work with DependencyArg | | zmedico | instances (or anything else that may have a "root_config" | | | attribute). | | 10824 | Bug #226307 - Copy come code from | | zmedico | depgraph._iter_atoms_for_pkg() that was used to solve bug | | | #218854, and use it inside unmerge() when matching sets to | | | packages. | svn path=/main/branches/prefix/; revision=10876 --- NEWS | 5 ++ man/emerge.1 | 13 +++- pym/_emerge/__init__.py | 139 +++++++++++++++++++++++++++++----------- pym/_emerge/help.py | 31 +++++++-- 4 files changed, 139 insertions(+), 49 deletions(-) diff --git a/NEWS b/NEWS index 45dc3798e..7463b463c 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,11 @@ News (mainly features/major bug fixes) portage-2.2 ------------- +* Add emerge --keep-going option to continue as much as possible after + an error. When an error occurs, dependencies are recalculated for + remaining packages and any with unsatisfied dependencies are + automatically dropped. The --skipfirst option automatically drops + packages in the same way, and also drops any masked packages. * Add subversion support for repoman. * It is now possible to use `emerge ` to reinstall the package that installed a particular file. Package contents entries are indexed to diff --git a/man/emerge.1 b/man/emerge.1 index 4f163701c..2bbcc0850 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -321,6 +321,12 @@ directory. .BR "\-\-ignore-default-opts" Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored. .TP +.BR "\-\-keep\-going" +Continue as much as possible after an error. When an error occurs, +dependencies are recalculated for remaining packages and any with +unsatisfied dependencies are automatically dropped. Also see +the related \fB\-\-skipfirst\fR option. +.TP .BR "\-\-newuse " (\fB\-N\fR) Tells emerge to include installed packages where USE flags have changed since compilation. USE flag changes include: @@ -399,9 +405,10 @@ enabled are added or removed. .TP .BR "\-\-skipfirst" This option is only valid when used with \fB\-\-resume\fR. It removes the -first package in the resume list so that a merge may continue in the presence -of an uncorrectable or inconsequential error. This should only be used in -cases where skipping the package will not result in failed dependencies. +first package in the resume list. Dependencies are recalculated for +remaining packages and any that have unsatisfied dependencies or are +masked will be automatically dropped. Also see the related +\fB\-\-keep\-going\fR option. .TP .BR "\-\-tree " (\fB\-t\fR) Shows the dependency tree for the given target by indenting dependencies. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index e6876205b..4e7fe38ff 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -67,6 +67,7 @@ import portage.locks import portage.exception from portage.const import EPREFIX, BPREFIX from portage.data import secpass +from portage.elog.messages import eerror from portage.util import normalize_path as normpath from portage.util import writemsg from portage.sets import load_default_config, SETPREFIX @@ -3892,25 +3893,23 @@ class depgraph(object): retlist.reverse() return retlist - def break_refs(self, mergelist): + def break_refs(self, nodes): """ Take a mergelist like that returned from self.altlist() and break any references that lead back to the depgraph. This is useful if you want to hold references to packages without also holding the depgraph on the heap. """ - for node in mergelist: - if not isinstance(node, Package): - continue - - # The FakeVartree references the _package_cache which - # references the depgraph. So that Package instances don't - # hold the depgraph and FakeVartree on the heap, replace - # the RootConfig that references the FakeVartree with the - # original RootConfig instance which references the actual - # vartree. - node.root_config = \ - self._trees_orig[node.root]["root_config"] + for node in nodes: + if hasattr(node, "root_config"): + # The FakeVartree references the _package_cache which + # references the depgraph. So that Package instances don't + # hold the depgraph and FakeVartree on the heap, replace + # the RootConfig that references the FakeVartree with the + # original RootConfig instance which references the actual + # vartree. + node.root_config = \ + self._trees_orig[node.root_config.root]["root_config"] def _resolve_conflicts(self): if not self._complete_graph(): @@ -5914,7 +5913,7 @@ class MergeTask(object): "--nodeps", "--pretend"]) def __init__(self, settings, trees, mtimedb, myopts, - spinner, mergelist, favorites): + spinner, mergelist, favorites, digraph): self.settings = settings self.target_root = settings["ROOT"] self.trees = trees @@ -5927,11 +5926,9 @@ class MergeTask(object): if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 self.pkgsettings = {} - self._blocker_db = {} for root in trees: self.pkgsettings[root] = portage.config( clone=trees[root]["vartree"].settings) - self._blocker_db[root] = BlockerDB(trees[root]["root_config"]) self.curval = 0 self._spawned_pids = [] @@ -5954,10 +5951,11 @@ class MergeTask(object): import gc gc.collect() + blocker_db = BlockerDB(self.trees[new_pkg.root]["root_config"]) + blocker_dblinks = [] - for blocking_pkg in self._blocker_db[ - new_pkg.root].findInstalledBlockers(new_pkg, - acquire_lock=acquire_lock): + for blocking_pkg in blocker_db.findInstalledBlockers( + new_pkg, acquire_lock=acquire_lock): if new_pkg.slot_atom == blocking_pkg.slot_atom: continue if new_pkg.cpv == blocking_pkg.cpv: @@ -5999,15 +5997,40 @@ class MergeTask(object): break if mergelist[0][-1] != "merge": break + # Skip the first one because it failed to build or install. + pkg_key = tuple(mergelist[0]) del mergelist[0] + failed_pkg = None + for task in self._mergelist: + if task == pkg_key: + failed_pkg = task + break + if failed_pkg is None: + break if not mergelist: break - mylist = self._calc_resume_list() + + mylist, dropped_tasks = self._calc_resume_list() clear_caches(self.trees) if not mylist: break - self.curval += 1 + + if dropped_tasks: + + def _eerror(lines): + for l in lines: + eerror(l, phase="other", key=failed_pkg.cpv) + + msg = [] + msg.append("One or more packages have been " + \ + "dropped due to unsatisfied dependencies:") + msg.append("") + msg.extend(" " + str(task) for task in dropped_tasks) + msg.append("") + _eerror(msg) + del _eerror, msg + del dropped_tasks self._mergelist = mylist return rval @@ -6035,20 +6058,12 @@ class MergeTask(object): if not success: mydepgraph.display_problems() - return None - - if dropped_tasks: - portage.writemsg("!!! One or more packages have been " + \ - "dropped due to\n" + \ - "!!! masking or unsatisfied dependencies:\n\n", - noiselevel=-1) - for task in dropped_tasks: - portage.writemsg(" " + str(task) + "\n", noiselevel=-1) - portage.writemsg("\n", noiselevel=-1) + return (None, None) mylist = mydepgraph.altlist() mydepgraph.break_refs(mylist) - return mylist + mydepgraph.break_refs(dropped_tasks) + return (mylist, dropped_tasks) def _poll_child_processes(self): """ @@ -6562,6 +6577,19 @@ def unmerge(root_config, myopts, unmerge_action, global_unmerge=0 xterm_titles = "notitles" not in settings.features + pkg_cache = {} + + def _pkg(cpv): + pkg = pkg_cache.get(cpv) + if pkg is None: + pkg = Package(cpv=cpv, installed=True, + metadata=izip(Package.metadata_keys, + vartree.dbapi.aux_get(cpv, Package.metadata_keys)), + root_config=root_config, + type_name="installed") + pkg_cache[cpv] = pkg + return pkg + vdb_path = os.path.join(settings["ROOT"], portage.VDB_PATH) try: # At least the parent needs to exist for the lock file. @@ -6804,6 +6832,12 @@ def unmerge(root_config, myopts, unmerge_action, # relevant package sets. for cp in xrange(len(pkgmap)): for cpv in pkgmap[cp]["selected"].copy(): + try: + pkg = _pkg(cpv) + except KeyError: + # It could have been uninstalled + # by a concurrent process. + continue parents = [] for s in installed_sets: # skip sets that the user requested to unmerge, and skip world @@ -6814,9 +6848,34 @@ def unmerge(root_config, myopts, unmerge_action, # only check instances of EditablePackageSet as other classes are generally used for # special purposes and can be ignored here (and are usually generated dynamically, so the # user can't do much about them anyway) - elif sets[s].containsCPV(cpv) \ - and isinstance(sets[s], EditablePackageSet): - parents.append(s) + if isinstance(sets[s], EditablePackageSet): + + # This is derived from a snippet of code in the + # depgraph._iter_atoms_for_pkg() method. + for atom in sets[s].iterAtomsForPackage(pkg): + inst_matches = vartree.dbapi.match(atom) + inst_matches.reverse() # descending order + higher_slot = None + for inst_cpv in inst_matches: + try: + inst_pkg = _pkg(inst_cpv) + except KeyError: + # It could have been uninstalled + # by a concurrent process. + continue + + if inst_pkg.cp != atom.cp: + continue + if pkg >= inst_pkg: + # This is descending order, and we're not + # interested in any versions <= pkg given. + break + if pkg.slot_atom != inst_pkg.slot_atom: + higher_slot = inst_pkg + break + if higher_slot is None: + parents.append(s) + break if parents: #print colorize("WARN", "Package %s is going to be unmerged," % cpv) #print colorize("WARN", "but still listed in the following package sets:") @@ -8981,11 +9040,12 @@ def action_build(settings, trees, mtimedb, time.sleep(3) # allow the parent to have first fetch mymergelist = mydepgraph.altlist() mydepgraph.break_refs(mymergelist) + mydepgraph.break_refs(mydepgraph.digraph.order) + mergetask = MergeTask(settings, trees, mtimedb, myopts, + spinner, mymergelist, favorites, mydepgraph.digraph) del mydepgraph clear_caches(trees) - mergetask = MergeTask(settings, trees, mtimedb, myopts, - spinner, mymergelist, favorites) retval = mergetask.merge() merge_count = mergetask.curval else: @@ -9027,11 +9087,12 @@ def action_build(settings, trees, mtimedb, pkglist = mydepgraph.altlist() mydepgraph.saveNomergeFavorites() mydepgraph.break_refs(pkglist) + mydepgraph.break_refs(mydepgraph.digraph.order) + mergetask = MergeTask(settings, trees, mtimedb, myopts, + spinner, pkglist, favorites, mydepgraph.digraph) del mydepgraph clear_caches(trees) - mergetask = MergeTask(settings, trees, mtimedb, myopts, - spinner, pkglist, favorites) retval = mergetask.merge() merge_count = mergetask.curval diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 1ec7e96bc..27e150d88 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -15,8 +15,8 @@ def shorthelp(): print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]" print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] " print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]" - print " [ "+green("--columns")+" ] [ "+green("--deep")+" ] [ "+green("--newuse")+" ]" - print " [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+" ]" + print " [ "+green("--columns")+" ] [ "+green("--deep")+" ] [ "+green("--keep-going")+" ] [ "+green("--newuse")+" ]" + print " [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+" ]" print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--complete-graph")+" ]" print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]" print bold("Actions:")+" [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]" @@ -292,6 +292,17 @@ def help(myaction,myopts,havecolor=1): print " downloaded from the remote server without consulting packages" print " existing in the packages directory." print + print " "+green("--keep-going") + desc = "Continue as much as possible after " + \ + "an error. When an error occurs, " + \ + "dependencies are recalculated for " + \ + "remaining packages and any with " + \ + "unsatisfied dependencies are " + \ + "automatically dropped. Also see " + \ + "the related --skipfirst option." + for line in wrap(desc, desc_width): + print desc_indent + line + print print " "+green("--newuse")+" ("+green("-N")+" short option)" print " Tells emerge to include installed packages where USE flags have " print " changed since installation." @@ -349,11 +360,17 @@ def help(myaction,myopts,havecolor=1): print " enabled are added or removed." print print " "+green("--skipfirst") - print " This option is only valid in a resume situation. It removes the" - print " first package in the resume list so that a merge may continue in" - print " the presence of an uncorrectable or inconsequential error. This" - print " should only be used in cases where skipping the package will not" - print " result in failed dependencies." + desc = "This option is only valid when " + \ + "used with --resume. It removes the " + \ + "first package in the resume list. " + \ + "Dependencies are recalculated for " + \ + "remaining packages and any that " + \ + "have unsatisfied dependencies or are " + \ + "masked will be automatically dropped. " + \ + "Also see the related " + \ + "--keep-going option." + for line in wrap(desc, desc_width): + print desc_indent + line print print " "+green("--tree")+" ("+green("-t")+" short option)" print " Shows the dependency tree using indentation for dependencies." -- 2.26.2