From: Fabian Groffen Date: Fri, 27 Jun 2008 14:53:20 +0000 (-0000) Subject: Merged from trunk 10801:10812 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=12d43c4a415029d5572925073cf603daaebdaec1;p=portage.git Merged from trunk 10801:10812 | 10802 | Handle KeyError from aux_get() inside | | zmedico | dbapi._iter_match_slot() and _iter_match_use(). Thanks to | | | grobian for reporting. | | 10803 | Use settings["ROOT"] at the beginning of env_update() to | | zmedico | avoid breakage due to "root" being an ObjectProxy instance. | | 10804 | * Remove PORTAGE_LEGACY_GLOBALS hack for portage import | | zmedico | since late initialization of portage.settings (via | | | ObjectProxy) allows us to rely on being able to access the | | | portage.exception namespace before portage.settings is | | | initialized. * Use portage.settings["ROOT"] instead of | | | portage.root to avoid potential ObjectProxy compatibility | | | issues. | | 10805 | Fix the PermissionDenied message for color.map. | | zmedico | | | 10806 | Add "PATH" to the set of allowed pacakge metadata keys for | | zmedico | the PackageIndex. | | 10807 | Call use_reduce() with matchall=1 when checking depstrings. | | zmedico | Otherwise it's possible for invalid deps to be ignored. | | | Thanks to grobian for reporting. | | 10808 | For brevity, and consistency with pkgcore, rename | | zmedico | "all-installed" to just "installed". Thanks to ferringb for | | | the suggestion. | | 10809 | Split the --skipfirst code into a resume_depgraph() function | | zmedico | that will be useful for implementing a --keep-going option | | | (bug #12768). | | 10810 | Use bool() to simplify ObjectProxy.__nonzero__(). | | zmedico | | | 10811 | Bug #12768 - Add a --keep-going option. This uses the same | | zmedico | resume_depgraph() function that's called for emerge --resume | | | operations. Given the remaining tasks, depedencies are | | | recalculated and any tasks with unsatisfied dependencies are | | | automatically dropped. | | 10812 | Call clear_caches() after the depgraph is out of scope, in | | zmedico | MergeTask.merge(). | svn path=/main/branches/prefix/; revision=10815 --- diff --git a/bin/portageq b/bin/portageq index 5f346f21d..b4ba70fde 100755 --- a/bin/portageq +++ b/bin/portageq @@ -462,25 +462,19 @@ def main(): global portage - # First import the main portage module without legacy globals since it - # is almost certain to succeed in that case. This provides access to - # the portage.exception namespace which is needed for later exception - # handling, like if portage.exception.PermissionDenied is raised when - # constructing the legacy global config instance. - os.environ["PORTAGE_LEGACY_GLOBALS"] = "false" - - # for an explanation on this logic, see pym/_emerge/__init__.py - if os.environ.__contains__("PORTAGE_PYTHONPATH"): - sys.path.insert(0, os.environ["PORTAGE_PYTHONPATH"]) - else: - sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym")) - import portage - - del os.environ["PORTAGE_LEGACY_GLOBALS"] try: - reload(portage) + try: + import portage + except ImportError: + from os import path as osp + # for an explanation on this logic, see pym/_emerge/__init__.py + if os.environ.__contains__("PORTAGE_PYTHONPATH"): + sys.path.insert(0, os.environ["PORTAGE_PYTHONPATH"]) + else: + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym")) + import portage if uses_root: - sys.argv[2] = portage.root + sys.argv[2] = portage.settings["ROOT"] retval = function(sys.argv[2:]) if retval: sys.exit(retval) diff --git a/bin/repoman b/bin/repoman index 93c01f677..0003acda4 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1247,7 +1247,7 @@ for x in scanlist: mydeplist = [] try: - portage.dep.use_reduce(mydeplist, excludeall=myiuse) + portage.dep.use_reduce(mydeplist, matchall=1) except portage.exception.InvalidDependString, e: badsyntax.append(str(e)) diff --git a/cnf/sets.conf b/cnf/sets.conf index 997e13d69..241fe63c5 100644 --- a/cnf/sets.conf +++ b/cnf/sets.conf @@ -25,7 +25,7 @@ class = portage.sets.security.NewAffectedSet world-candidate = False # Again, not much to change here, though people might prefer a different name -[all-installed] +[installed] class = portage.sets.dbapi.EverythingSet world-candidate = False diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index a3d7bdb69..e6876205b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -212,6 +212,7 @@ options=[ "--fetchonly", "--fetch-all-uri", "--getbinpkg", "--getbinpkgonly", "--help", "--ignore-default-opts", +"--keep-going", "--noconfmem", "--newuse", "--nocolor", "--nodeps", "--noreplace", @@ -5428,7 +5429,7 @@ class depgraph(object): world_set.update(all_added) world_set.unlock() - def loadResumeCommand(self, resume_data): + def loadResumeCommand(self, resume_data, skip_masked=False): """ Add a resume command to the graph and validate it in the process. This will raise a PackageNotFound exception if a package is not available. @@ -5441,7 +5442,6 @@ class depgraph(object): if not isinstance(mergelist, list): mergelist = [] - skipfirst = "--skipfirst" in self.myopts fakedb = self.mydbapi trees = self.trees serialized_tasks = [] @@ -5479,7 +5479,7 @@ class depgraph(object): root_config = self.roots[pkg.root] if "merge" == pkg.operation and \ not visible(root_config.settings, pkg): - if skipfirst: + if skip_masked: masked_tasks.append(Dependency(root=pkg.root, parent=pkg)) else: self._unsatisfied_deps_for_display.append( @@ -5533,7 +5533,7 @@ class depgraph(object): # was dropped via --skipfirst. It makes the # resume list invalid, so convert it to a # UnsatisfiedResumeDep exception. - raise self.UnsatisfiedResumeDep( + raise self.UnsatisfiedResumeDep(self, masked_tasks + self._unsatisfied_deps) self._serialized_tasks_cache = None try: @@ -5610,6 +5610,9 @@ class depgraph(object): can occur when a required package is dropped from the merge list via --skipfirst. """ + def __init__(self, depgraph, value): + portage.exception.PortageException.__init__(self, value) + self.depgraph = depgraph class _internal_exception(portage.exception.PortageException): def __init__(self, value=""): @@ -5910,11 +5913,16 @@ class MergeTask(object): "--fetchonly", "--fetch-all-uri", "--nodeps", "--pretend"]) - def __init__(self, settings, trees, myopts): + def __init__(self, settings, trees, mtimedb, myopts, + spinner, mergelist, favorites): self.settings = settings self.target_root = settings["ROOT"] self.trees = trees self.myopts = myopts + self._spinner = spinner + self._mtimedb = mtimedb + self._mergelist = mergelist + self._favorites = favorites self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 @@ -5963,14 +5971,84 @@ class MergeTask(object): return blocker_dblinks - def merge(self, mylist, favorites, mtimedb): - try: - return self._merge(mylist, favorites, mtimedb) - finally: - if self._spawned_pids: - from portage import process - process.spawned_pids.extend(self._spawned_pids) - self._spawned_pids = [] + def merge(self): + + keep_going = "--keep-going" in self.myopts + + while True: + try: + rval = self._merge() + finally: + spawned_pids = self._spawned_pids + while spawned_pids: + pid = spawned_pids.pop() + try: + if os.waitpid(pid, os.WNOHANG) == (0, 0): + os.kill(pid, signal.SIGTERM) + os.waitpid(pid, 0) + except OSError: + pass # cleaned up elsewhere. + + if rval == os.EX_OK or not keep_going: + break + mtimedb = self._mtimedb + if "resume" not in mtimedb: + break + mergelist = self._mtimedb["resume"].get("mergelist") + if not mergelist: + break + if mergelist[0][-1] != "merge": + break + # Skip the first one because it failed to build or install. + del mergelist[0] + if not mergelist: + break + mylist = self._calc_resume_list() + clear_caches(self.trees) + if not mylist: + break + self.curval += 1 + self._mergelist = mylist + + return rval + + def _calc_resume_list(self): + """ + Use the current resume list to calculate a new one, + dropping any packages with unsatisfied deps. + """ + print colorize("GOOD", "*** Resuming merge...") + + show_spinner = "--quiet" not in self.myopts and \ + "--nodeps" not in self.myopts + + if show_spinner: + print "Calculating dependencies ", + + myparams = create_depgraph_params(self.myopts, None) + success, mydepgraph, dropped_tasks = resume_depgraph( + self.settings, self.trees, self._mtimedb, self.myopts, + myparams, self._spinner, skip_unsatisfied=True) + + if show_spinner: + print "\b\b... done!" + + if not success: + mydepgraph.display_problems() + return None + + if dropped_tasks: + portage.writemsg("!!! One or more packages have been " + \ + "dropped due to\n" + \ + "!!! masking or unsatisfied dependencies:\n\n", + noiselevel=-1) + for task in dropped_tasks: + portage.writemsg(" " + str(task) + "\n", noiselevel=-1) + portage.writemsg("\n", noiselevel=-1) + + mylist = mydepgraph.altlist() + mydepgraph.break_refs(mylist) + return mylist def _poll_child_processes(self): """ @@ -5991,7 +6069,10 @@ class MergeTask(object): pass spawned_pids.remove(pid) - def _merge(self, mylist, favorites, mtimedb): + def _merge(self): + mylist = self._mergelist + favorites = self._favorites + mtimedb = self._mtimedb from portage.elog import elog_process from portage.elog.filtering import filter_mergephases buildpkgonly = "--buildpkgonly" in self.myopts @@ -8483,6 +8564,73 @@ def action_depclean(settings, trees, ldpath_mtimes, else: print "Number removed: "+str(len(cleanlist)) +def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner, + skip_masked=False, skip_unsatisfied=False): + """ + Construct a depgraph for the given resume list. This will raise + PackageNotFound or depgraph.UnsatisfiedResumeDep when necessary. + @rtype: tuple + @returns: (success, depgraph, dropped_tasks) + """ + mergelist = mtimedb["resume"]["mergelist"] + dropped_tasks = set() + while True: + mydepgraph = depgraph(settings, trees, + myopts, myparams, spinner) + try: + success = mydepgraph.loadResumeCommand(mtimedb["resume"], + skip_masked=skip_masked) + except depgraph.UnsatisfiedResumeDep, e: + if not skip_unsatisfied: + raise + + graph = mydepgraph.digraph + unsatisfied_parents = dict((dep.parent, dep.parent) \ + for dep in e.value) + traversed_nodes = set() + unsatisfied_stack = list(unsatisfied_parents) + while unsatisfied_stack: + pkg = unsatisfied_stack.pop() + if pkg in traversed_nodes: + continue + traversed_nodes.add(pkg) + + # If this package was pulled in by a parent + # package scheduled for merge, removing this + # package may cause the the parent package's + # dependency to become unsatisfied. + for parent_node in graph.parent_nodes(pkg): + if not isinstance(parent_node, Package) \ + or parent_node.operation != "merge": + continue + unsatisfied = \ + graph.child_nodes(parent_node, + ignore_priority=DepPriority.SOFT) + if pkg in unsatisfied: + unsatisfied_parents[parent_node] = parent_node + unsatisfied_stack.append(parent_node) + + pruned_mergelist = [x for x in mergelist \ + if isinstance(x, list) and \ + tuple(x) not in unsatisfied_parents] + + # It shouldn't happen, but if the size of mergelist + # does not decrease for some reason then the loop + # will be infinite. Therefore, if that case ever + # occurs for some reason, raise the exception to + # break out of the loop. + if not pruned_mergelist or \ + len(pruned_mergelist) == len(mergelist): + raise + mergelist[:] = pruned_mergelist + dropped_tasks.update(unsatisfied_parents) + del e, graph, traversed_nodes, \ + unsatisfied_parents, unsatisfied_stack + continue + else: + break + return (success, mydepgraph, dropped_tasks) + def action_build(settings, trees, mtimedb, myopts, myaction, myfiles, spinner): @@ -8606,66 +8754,18 @@ def action_build(settings, trees, mtimedb, del mergelist[i] break - dropped_tasks = set() - + skip_masked = "--skipfirst" in myopts + skip_unsatisfied = "--skipfirst" in myopts success = False + mydepgraph = None try: - while True: - mydepgraph = depgraph(settings, trees, - myopts, myparams, spinner) - try: - success = mydepgraph.loadResumeCommand(mtimedb["resume"]) - except depgraph.UnsatisfiedResumeDep, e: - if "--skipfirst" not in myopts: - raise - - graph = mydepgraph.digraph - unsatisfied_parents = dict((dep.parent, dep.parent) \ - for dep in e.value) - traversed_nodes = set() - unsatisfied_stack = list(unsatisfied_parents) - while unsatisfied_stack: - pkg = unsatisfied_stack.pop() - if pkg in traversed_nodes: - continue - traversed_nodes.add(pkg) - - # If this package was pulled in by a parent - # package scheduled for merge, removing this - # package may cause the the parent package's - # dependency to become unsatisfied. - for parent_node in graph.parent_nodes(pkg): - if not isinstance(parent_node, Package) \ - or parent_node.operation != "merge": - continue - unsatisfied = \ - graph.child_nodes(parent_node, - ignore_priority=DepPriority.SOFT) - if pkg in unsatisfied: - unsatisfied_parents[parent_node] = parent_node - unsatisfied_stack.append(parent_node) - - pruned_mergelist = [x for x in mergelist \ - if isinstance(x, list) and \ - tuple(x) not in unsatisfied_parents] - - # It shouldn't happen, but if the size of mergelist - # does not decrease for some reason then the loop - # will be infinite. Therefore, if that case ever - # occurs for some reason, raise the exception to - # break out of the loop. - if not pruned_mergelist or \ - len(pruned_mergelist) == len(mergelist): - raise - mergelist[:] = pruned_mergelist - dropped_tasks.update(unsatisfied_parents) - del e, graph, traversed_nodes, \ - unsatisfied_parents, unsatisfied_stack - continue - else: - break + success, mydepgraph, dropped_tasks = resume_depgraph( + settings, trees, mtimedb, myopts, myparams, spinner, + skip_masked=skip_masked, skip_unsatisfied=skip_unsatisfied) except (portage.exception.PackageNotFound, depgraph.UnsatisfiedResumeDep), e: + if isinstance(e, depgraph.UnsatisfiedResumeDep): + mydepgraph = e.depgraph if show_spinner: print from textwrap import wrap @@ -8734,7 +8834,8 @@ def action_build(settings, trees, mtimedb, portage.writemsg("\n", noiselevel=-1) del dropped_tasks else: - mydepgraph.display_problems() + if mydepgraph is not None: + mydepgraph.display_problems() if not (ask or pretend): # delete the current list and also the backup # since it's probably stale too. @@ -8873,7 +8974,6 @@ def action_build(settings, trees, mtimedb, if ("--resume" in myopts): favorites=mtimedb["resume"]["favorites"] - mergetask = MergeTask(settings, trees, myopts) if "PORTAGE_PARALLEL_FETCHONLY" in settings: """ parallel-fetch uses --resume --fetchonly and we don't want it to write the mtimedb""" @@ -8884,7 +8984,9 @@ def action_build(settings, trees, mtimedb, del mydepgraph clear_caches(trees) - retval = mergetask.merge(mymergelist, favorites, mtimedb) + mergetask = MergeTask(settings, trees, mtimedb, myopts, + spinner, mymergelist, favorites) + retval = mergetask.merge() merge_count = mergetask.curval else: if "resume" in mtimedb and \ @@ -8928,8 +9030,9 @@ def action_build(settings, trees, mtimedb, del mydepgraph clear_caches(trees) - mergetask = MergeTask(settings, trees, myopts) - retval = mergetask.merge(pkglist, favorites, mtimedb) + mergetask = MergeTask(settings, trees, mtimedb, myopts, + spinner, pkglist, favorites) + retval = mergetask.merge() merge_count = mergetask.curval if retval == os.EX_OK and not (buildpkgonly or fetchonly or pretend): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e6ce9b591..a8c120e2b 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -542,8 +542,8 @@ class digraph(object): def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, env=None): if target_root is None: - global root - target_root = root + global settings + target_root = settings["ROOT"] if prev_mtimes is None: global mtimedb prev_mtimes = mtimedb["ldpath"] diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index ba37c867e..54ce7afc3 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -135,8 +135,11 @@ class dbapi(object): def _iter_match_slot(self, atom, cpv_iter): for cpv in cpv_iter: - if self.aux_get(cpv, ["SLOT"])[0] == atom.slot: - yield cpv + try: + if self.aux_get(cpv, ["SLOT"])[0] == atom.slot: + yield cpv + except KeyError: + continue def _iter_match_use(self, atom, cpv_iter): """ @@ -146,7 +149,10 @@ class dbapi(object): if self._iuse_implicit is None: self._iuse_implicit = self.settings._get_implicit_iuse() for cpv in cpv_iter: - iuse, use = self.aux_get(cpv, ["IUSE", "USE"]) + try: + iuse, use = self.aux_get(cpv, ["IUSE", "USE"]) + except KeyError: + continue use = use.split() iuse = self._iuse_implicit.union( re.escape(x.lstrip("+-")) for x in iuse.split()) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 2fe21b4f2..05ff9cd97 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -173,6 +173,7 @@ class binarytree(object): "IUSE" : "", "KEYWORDS": "", "LICENSE" : "", + "PATH" : "", "PDEPEND" : "", "PROVIDE" : "", "RDEPEND" : "", diff --git a/pym/portage/output.py b/pym/portage/output.py index a7a500cd4..45def2b38 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -231,8 +231,11 @@ try: parse_color_map(onerror=lambda e: writemsg("%s\n" % str(e), noiselevel=-1)) except FileNotFound: pass +except PermissionDenied, e: + writemsg("Permission denied: '%s'\n" % str(e), noiselevel=-1) + del e except PortageException, e: - writemsg("%s\n" % str(e)) + writemsg("%s\n" % str(e), noiselevel=-1) del e def nc_len(mystr): diff --git a/pym/portage/util.py b/pym/portage/util.py index 6012b0f26..bb3b42e0d 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -970,9 +970,7 @@ class ObjectProxy(object): return object.__getattribute__(self, '_get_target')() != other def __nonzero__(self): - if object.__getattribute__(self, '_get_target')(): - return True - return False + return bool(object.__getattribute__(self, '_get_target')()) class LazyItemsDict(dict): """A mapping object that behaves like a standard dict except that it allows