From 7fa7aba94c9e2f5b35a05bde00d355aba42171d0 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 13 Dec 2006 16:22:34 +0000 Subject: [PATCH] Merged from trunk 5271:5283 For bug #157823, ignore blocker collisions between two packages that are already installed if no merge is currently scheduled for either of those package slots. These blockers will be handled as necessary the next time that a merge of either package is triggered. Make the fetch phase bail out if FETCHCOMMAND or RESUMECOMMAND are missing. Thanks to Flameeyes for reporting. All portage.fetch() to succeed even if FETCHCOMMAND and RESUMECOMMAND aren't defined. Centralize all of the vardb metadata mangling in FakeVartree. Store dependency metadata in depgraph.mydbapi so that it can be used in place of the portdb for optimized dep_check calls in depgraph.validate_blockers(). Optimize dep_check calls in depgraph.validate_blockers() by using a fakedbapi instance instead of the real pordbapi (minimizes available matches). For higher efficiency, call self.__int__() instead of int(self). For bug #157897, only let depgraph.create() inject "merge" nodes into the final state dbapi. For bug #157897, ensure that only "merge" nodes are stored in digraph.pkg_node_map. Move the biolerplate cpv_all() implementation from portdbapi to dbapi. Track slot atom to digraph node mappings as the depgraph is built. This mapping will be used to detect node collisions for bug #157897. Remove redundant digraph.pkg_node_map node insertions. Once per digraph.create() call will cover all nodes. svn path=/main/branches/prefix/; revision=5284 --- bin/emerge | 145 +++++++++++++++++++++++++++---------------------- pym/portage.py | 21 +++++-- 2 files changed, 96 insertions(+), 70 deletions(-) diff --git a/bin/emerge b/bin/emerge index af26bbb01..50be592eb 100755 --- a/bin/emerge +++ b/bin/emerge @@ -679,22 +679,22 @@ class DepPriority(object): return -3 return -4 def __lt__(self, other): - return int(self) < other + return self.__int__() < other def __le__(self, other): - return int(self) <= other + return self.__int__() <= other def __eq__(self, other): - return int(self) == other + return self.__int__() == other def __ne__(self, other): - return int(self) != other + return self.__int__() != other def __gt__(self, other): - return int(self) > other + return self.__int__() > other def __ge__(self, other): - return int(self) >= other + return self.__int__() >= other def copy(self): import copy return copy.copy(self) def __str__(self): - myvalue = int(self) + myvalue = self.__int__() if myvalue > self.MEDIUM: return "hard" if myvalue > self.SOFT: @@ -711,10 +711,11 @@ class FakeVartree(portage.vartree): user doesn't necessarily need write access to the vardb in cases where global updates are necessary (updates are performed when necessary if there is not a matching ebuild in the tree).""" - def __init__(self, real_vartree): + def __init__(self, real_vartree, portdb): self.root = real_vartree.root self.settings = real_vartree.settings self.dbapi = portage.fakedbapi(settings=real_vartree.settings) + global_updates = None vdb_path = os.path.join(self.root, portage.VDB_PATH) try: # At least the parent needs to exist for the lock file. @@ -727,6 +728,7 @@ class FakeVartree(portage.vartree): vdb_lock = portage_locks.lockdir(vdb_path) mykeys = ["SLOT", "COUNTER", "PROVIDE", "USE", "IUSE", "DEPEND", "RDEPEND", "PDEPEND"] + portdb_keys = ["DEPEND", "RDEPEND", "PDEPEND"] real_dbapi = real_vartree.dbapi slot_counters = {} for cpv in real_dbapi.cpv_all(): @@ -744,6 +746,17 @@ class FakeVartree(portage.vartree): continue slot_counters[myslot_atom] = mycounter self.dbapi.cpv_inject(cpv, metadata=metadata) + try: + # Use the live ebuild metadata if possible. + live_metadata = dict(izip(portdb_keys, + portdb.aux_get(cpv, portdb_keys))) + self.dbapi.aux_update(cpv, live_metadata) + except KeyError: + if global_updates is None: + global_updates = \ + grab_global_updates(portdb.porttree_root) + perform_global_updates( + cpv, self.dbapi, global_updates) finally: if vdb_lock: portage_locks.unlockdir(vdb_lock) @@ -790,8 +803,12 @@ class depgraph: self.edebug = 1 self.spinner = spinner self.pkgsettings = {} + # Maps cpv to digraph node for "merge" nodes only. self.pkg_node_map = {} + # Maps slot atom to digraph node for all nodes added to the graph. + self._slot_node_map = {} self.mydbapi = {} + self._mydbapi_keys = ["SLOT", "DEPEND", "RDEPEND", "PDEPEND"] self.useFlags = {} self.trees = {} for myroot in trees: @@ -799,18 +816,21 @@ class depgraph: for tree in ("porttree", "bintree"): self.trees[myroot][tree] = trees[myroot][tree] self.trees[myroot]["vartree"] = \ - FakeVartree(trees[myroot]["vartree"]) + FakeVartree(trees[myroot]["vartree"], + trees[myroot]["porttree"].dbapi) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) self.pkg_node_map[myroot] = {} + self._slot_node_map[myroot] = {} vardb = self.trees[myroot]["vartree"].dbapi # This fakedbapi instance will model the state that the vdb will # have after new packages have been installed. fakedb = portage.fakedbapi(settings=self.pkgsettings[myroot]) self.mydbapi[myroot] = fakedb for pkg in vardb.cpv_all(): - myslot = vardb.aux_get(pkg, ["SLOT"])[0] - fakedb.cpv_inject(pkg, metadata={"SLOT":myslot}) + fakedb.cpv_inject(pkg, + metadata=dict(izip(self._mydbapi_keys, + vardb.aux_get(pkg, self._mydbapi_keys)))) del vardb, fakedb self.useFlags[myroot] = {} if "--usepkg" in self.myopts: @@ -824,7 +844,6 @@ class depgraph: self.orderedkeys=[] self.outdatedpackages=[] self.args_keys = [] - self.global_updates = {} self.blocker_digraph = digraph() self.blocker_parents = {} self._altlist_cache = {} @@ -849,20 +868,16 @@ class depgraph: if addme and jbigkey != myparent: # Refuse to make a node depend on itself so that the we don't # don't create a bogus circular dependency in self.altlist(). - self.pkg_node_map[myroot][mykey] = jbigkey if rev_dep and myparent: ptype, proot, pkey, pstatus = myparent - self.pkg_node_map[proot][pkey] = myparent self.digraph.addnode(myparent, jbigkey, priority=priority) else: self.digraph.addnode(jbigkey, myparent, priority=priority) return 1 jbigkey = tuple(chain(mybigkey, ["nomerge"])) if self.digraph.hasnode(jbigkey): - self.pkg_node_map[myroot][mykey] = jbigkey if rev_dep and myparent: ptype, proot, pkey, pstatus = myparent - self.pkg_node_map[proot][pkey] = myparent self.digraph.addnode(myparent, jbigkey, priority=priority) else: self.digraph.addnode(jbigkey, myparent, priority=priority) @@ -937,19 +952,21 @@ class depgraph: depgraph already and returned, or we are here. Whether we are merging or not; we must add the package to the depgraph; so we do that here. """ jbigkey = tuple(mybigkey) - self.pkg_node_map[myroot][mykey] = jbigkey if rev_dep and myparent: ptype, proot, pkey, pstatus = myparent - self.pkg_node_map[proot][pkey] = myparent self.digraph.addnode(myparent, jbigkey, priority=priority) else: self.digraph.addnode(jbigkey, myparent, priority=priority) - if addme: - myslot = mydbapi.aux_get(mykey, ["SLOT"])[0] - self.mydbapi[myroot].cpv_inject(mykey, metadata={"SLOT":myslot}) + metadata = dict(izip(self._mydbapi_keys, + mydbapi.aux_get(mykey, self._mydbapi_keys))) + slot_atom = "%s:%s" % (portage.dep_getkey(mykey), metadata["SLOT"]) + self._slot_node_map[myroot][slot_atom] = jbigkey + if mybigkey[-1] == "merge": + self.pkg_node_map[myroot][mykey] = jbigkey + self.mydbapi[myroot].cpv_inject(mykey, metadata=metadata) """ This section determines whether we go deeper into dependencies or not. We want to go deeper on a few occasions: @@ -1368,15 +1385,6 @@ class depgraph: myeb_inst, ["USE"])[0].split() matched_packages.append( ["installed", myroot, myeb_inst, binpkguseflags]) - """Now make sure that *DEPEND atoms are up to date. - This is done on the fly for single packages only when - necessary, since it can be time consuming to run this - on all installed packages.""" - if myroot not in self.global_updates: - self.global_updates[myroot] = \ - grab_global_updates(pkgsettings["PORTDIR"]) - perform_global_updates( - myeb_inst, vardb, self.global_updates[myroot]) if not matched_packages: if raise_on_missing: @@ -1466,6 +1474,19 @@ class depgraph: # been pulled into the depgraph. This is not enabled by default # due to the performance penalty that is incurred by all the # additional dep_check calls that are required. + + # Optimization hack for dep_check calls that minimizes the + # available matches by replacing the portdb with a fakedbapi + # instance. + class FakePortageTree(object): + def __init__(self, mydb): + self.dbapi = mydb + dep_check_trees = {} + for myroot in self.trees: + dep_check_trees[myroot] = self.trees[myroot].copy() + dep_check_trees[myroot]["porttree"] = \ + FakePortageTree(self.mydbapi[myroot]) + dep_keys = ["DEPEND","RDEPEND","PDEPEND"] for myroot in self.trees: pkg_node_map = self.pkg_node_map[myroot] @@ -1481,24 +1502,14 @@ class depgraph: # If this node has any blockers, create a "nomerge" # node for it so that they can be enforced. self.spinner.update() - try: - dep_vals = portdb.aux_get(pkg, dep_keys) - ptype = "ebuild" - except KeyError: - if myroot not in self.global_updates: - self.global_updates[myroot] = \ - grab_global_updates(pkgsettings["PORTDIR"]) - perform_global_updates( - pkg, vardb, self.global_updates[myroot]) - dep_vals = vardb.aux_get(pkg, dep_keys) - ptype = "installed" + dep_vals = vardb.aux_get(pkg, dep_keys) myuse = vardb.aux_get(pkg, ["USE"])[0].split() depstr = " ".join(dep_vals) # It is crucial to pass in final_db here in order to # optimize dep_check calls by eliminating atoms via # dep_wordreduce and dep_eval calls. success, atoms = portage.dep_check(depstr, final_db, - pkgsettings, myuse=myuse, trees=self.trees, + pkgsettings, myuse=myuse, trees=dep_check_trees, myroot=myroot) if not success: print "\n\n" @@ -1511,7 +1522,7 @@ class depgraph: # Don't store this parent in pkg_node_map, because it's # not needed there and it might overwrite a "merge" # node with the same cpv. - myparent = (ptype, myroot, pkg, "nomerge") + myparent = ("installed", myroot, pkg, "nomerge") for myatom in blocker_atoms: blocker = ("blocks", myroot, myatom[1:]) myparents = \ @@ -1521,6 +1532,18 @@ class depgraph: self.blocker_parents[blocker] = myparents myparents.add(myparent) + modified_slots = {} + if self.blocker_parents: + for myroot in self.trees: + myslots = {} + modified_slots[myroot] = myslots + final_db = self.mydbapi[myroot] + slot_node_map = self._slot_node_map[myroot] + for slot_atom, mynode in slot_node_map.iteritems(): + mytype, myroot, mycpv, mystatus = mynode + if mystatus == "merge": + modified_slots[myroot][slot_atom] = mycpv + for blocker in self.blocker_parents.keys(): mytype, myroot, mydep = blocker initial_db = self.trees[myroot]["vartree"].dbapi @@ -1545,6 +1568,7 @@ class depgraph: pdbapi = self.trees[proot][self.pkg_tree_map[ptype]].dbapi pslot = pdbapi.aux_get(pcpv, ["SLOT"])[0] pslot_atom = "%s:%s" % (portage.dep_getkey(pcpv), pslot) + parent_static = pslot_atom not in modified_slots[proot] unresolved_blocks = False depends_on_order = set() for cpv in blocked_initial: @@ -1555,6 +1579,11 @@ class depgraph: # node matters. In any case, this particular block is # automatically resolved. continue + if parent_static and \ + slot_atom not in modified_slots[myroot]: + # This blocker will be handled the next time that a + # merge of either package is triggered. + continue if pstatus == "merge" and \ slot_atom not in blocked_slots_final.values(): upgrade_matches = final_db.match(slot_atom) @@ -1574,6 +1603,11 @@ class depgraph: # The parent blocks itself, so the merge order does not # need to be enforced. continue + if parent_static and \ + slot_atom not in modified_slots[myroot]: + # This blocker will be handled the next time that a + # merge of either package is triggered. + continue # None of the above blocker resolutions techniques apply, # so apparently this one is unresolvable. unresolved_blocks = True @@ -3984,7 +4018,7 @@ def action_depclean(settings, trees, ldpath_mtimes, xterm_titles = "notitles" not in settings.features portdb = trees[settings["ROOT"]]["porttree"].dbapi real_vartree = trees[settings["ROOT"]]["vartree"] - vartree = FakeVartree(real_vartree) + vartree = FakeVartree(real_vartree, portdb) trees[settings["ROOT"]]["vartree"] = vartree vardb = vartree.dbapi syslist = getlist(settings, "system") @@ -4018,26 +4052,9 @@ def action_depclean(settings, trees, ldpath_mtimes, unresolveable = {} aux_keys = ["DEPEND", "RDEPEND", "PDEPEND"] - fake_vardb = portage.fakedbapi(settings=settings) - fakedb_auxkeys = aux_keys[:] - fakedb_auxkeys.append("SLOT") - global_updates = None - for cpv in myvarlist: - try: - # Prefer live ebuild metadata when available. - aux_vals = portdb.aux_get(cpv, fakedb_auxkeys) - # SLOT always comes from the vardb, for multislot. - aux_vals[-1] = vardb.aux_get(cpv, ["SLOT"])[0] - live_ebuild = True - except KeyError: - aux_vals = vardb.aux_get(cpv, fakedb_auxkeys) - live_ebuild = False - fake_vardb.cpv_inject( - cpv, metadata=dict(izip(fakedb_auxkeys, aux_vals))) - if not live_ebuild: - if global_updates is None: - global_updates = grab_global_updates(settings["PORTDIR"]) - perform_global_updates(cpv, fake_vardb, global_updates) + # Now that FakeVartree pulls metadata from the portdb and performs global + # updates when necessary, fake_vardb and vardb are equivalent. + fake_vardb = vardb # HACK: Ensure that installed packages are preferenced by dep_check(). trees[settings["ROOT"]]["porttree"].dbapi = fake_vardb diff --git a/pym/portage.py b/pym/portage.py index 36c7e3f53..171073281 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -2170,6 +2170,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", can_fetch=True + for var_name in ("FETCHCOMMAND", "RESUMECOMMAND"): + if not mysettings.get(var_name, None): + can_fetch = False + if not listonly: dirmode = 02070 filemode = 060 @@ -2316,6 +2320,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", else: writemsg("!!! File %s isn't fully fetched, but unable to complete it\n" % myfile, noiselevel=-1) + for var_name in ("FETCHCOMMAND", "RESUMECOMMAND"): + if not mysettings.get(var_name, None): + writemsg(("!!! %s is unset. It should " + \ + "have been defined in /etc/make.globals.\n") \ + % var_name, noiselevel=-1) return 0 else: continue @@ -4333,6 +4342,12 @@ class dbapi: def cp_list(self,cp,use_cache=1): return + def cpv_all(self): + cpv_list = [] + for cp in self.cp_all(): + cpv_list.extend(self.cp_list(cp)) + return cpv_list + def aux_get(self,mycpv,mylist): "stub code for returning auxiliary db information, such as SLOT, DEPEND, etc." 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' @@ -5524,12 +5539,6 @@ class portdbapi(dbapi): l.sort() return l - def cpv_all(self): - cpv_list = [] - for cp in self.cp_all(): - cpv_list.extend(self.cp_list(cp)) - return cpv_list - def p_list(self,mycp): d={} for oroot in self.porttrees: -- 2.26.2