Merged from trunk 5271:5283
authorFabian Groffen <grobian@gentoo.org>
Wed, 13 Dec 2006 16:22:34 +0000 (16:22 -0000)
committerFabian Groffen <grobian@gentoo.org>
Wed, 13 Dec 2006 16:22:34 +0000 (16:22 -0000)
For bug #157823, ignore blocker collisions between two packages that are
already installed if no merge is currently scheduled for either of those
package slots.  These blockers will be handled as necessary the next
time that a merge of either package is triggered.

Make the fetch phase bail out if FETCHCOMMAND or RESUMECOMMAND are
missing.  Thanks to Flameeyes for reporting.

All portage.fetch() to succeed even if FETCHCOMMAND and RESUMECOMMAND
aren't defined.

Centralize all of the vardb metadata mangling in FakeVartree.

Store dependency metadata in depgraph.mydbapi so that it can be used in
place of the portdb for optimized dep_check calls in
depgraph.validate_blockers().

Optimize dep_check calls in depgraph.validate_blockers() by using a
fakedbapi instance instead of the real pordbapi (minimizes available
matches).

For higher efficiency, call self.__int__() instead of int(self).

For bug #157897, only let depgraph.create() inject "merge" nodes into
the final state dbapi.

For bug #157897, ensure that only "merge" nodes are stored in
digraph.pkg_node_map.

Move the biolerplate cpv_all() implementation from portdbapi to dbapi.

Track slot atom to digraph node mappings as the depgraph is built.  This
mapping will be used to detect node collisions for bug #157897.

Remove redundant digraph.pkg_node_map node insertions.  Once per
digraph.create() call will cover all nodes.

svn path=/main/branches/prefix/; revision=5284

bin/emerge
pym/portage.py

index af26bbb0150c865de5c1555a617394dcb4d8f3e4..50be592eb4dcc6ca358dbd96552efbcb26d54b40 100755 (executable)
@@ -679,22 +679,22 @@ class DepPriority(object):
                        return -3
                return -4
        def __lt__(self, other):
-               return int(self) < other
+               return self.__int__() < other
        def __le__(self, other):
-               return int(self) <= other
+               return self.__int__() <= other
        def __eq__(self, other):
-               return int(self) == other
+               return self.__int__() == other
        def __ne__(self, other):
-               return int(self) != other
+               return self.__int__() != other
        def __gt__(self, other):
-               return int(self) > other
+               return self.__int__() > other
        def __ge__(self, other):
-               return int(self) >= other
+               return self.__int__() >= other
        def copy(self):
                import copy
                return copy.copy(self)
        def __str__(self):
-               myvalue = int(self)
+               myvalue = self.__int__()
                if myvalue > self.MEDIUM:
                        return "hard"
                if myvalue > self.SOFT:
@@ -711,10 +711,11 @@ class FakeVartree(portage.vartree):
        user doesn't necessarily need write access to the vardb in cases where
        global updates are necessary (updates are performed when necessary if there
        is not a matching ebuild in the tree)."""
-       def __init__(self, real_vartree):
+       def __init__(self, real_vartree, portdb):
                self.root = real_vartree.root
                self.settings = real_vartree.settings
                self.dbapi = portage.fakedbapi(settings=real_vartree.settings)
+               global_updates = None
                vdb_path = os.path.join(self.root, portage.VDB_PATH)
                try:
                        # At least the parent needs to exist for the lock file.
@@ -727,6 +728,7 @@ class FakeVartree(portage.vartree):
                                vdb_lock = portage_locks.lockdir(vdb_path)
                        mykeys = ["SLOT", "COUNTER", "PROVIDE", "USE", "IUSE",
                                "DEPEND", "RDEPEND", "PDEPEND"]
+                       portdb_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
                        real_dbapi = real_vartree.dbapi
                        slot_counters = {}
                        for cpv in real_dbapi.cpv_all():
@@ -744,6 +746,17 @@ class FakeVartree(portage.vartree):
                                                continue
                                slot_counters[myslot_atom] = mycounter
                                self.dbapi.cpv_inject(cpv, metadata=metadata)
+                               try:
+                                       # Use the live ebuild metadata if possible.
+                                       live_metadata = dict(izip(portdb_keys,
+                                               portdb.aux_get(cpv, portdb_keys)))
+                                       self.dbapi.aux_update(cpv, live_metadata)
+                               except KeyError:
+                                       if global_updates is None:
+                                               global_updates = \
+                                                       grab_global_updates(portdb.porttree_root)
+                                       perform_global_updates(
+                                               cpv, self.dbapi, global_updates)
                finally:
                        if vdb_lock:
                                portage_locks.unlockdir(vdb_lock)
@@ -790,8 +803,12 @@ class depgraph:
                        self.edebug = 1
                self.spinner = spinner
                self.pkgsettings = {}
+               # Maps cpv to digraph node for "merge" nodes only.
                self.pkg_node_map = {}
+               # Maps slot atom to digraph node for all nodes added to the graph.
+               self._slot_node_map = {}
                self.mydbapi = {}
+               self._mydbapi_keys = ["SLOT", "DEPEND", "RDEPEND", "PDEPEND"]
                self.useFlags = {}
                self.trees = {}
                for myroot in trees:
@@ -799,18 +816,21 @@ class depgraph:
                        for tree in ("porttree", "bintree"):
                                self.trees[myroot][tree] = trees[myroot][tree]
                        self.trees[myroot]["vartree"] = \
-                               FakeVartree(trees[myroot]["vartree"])
+                               FakeVartree(trees[myroot]["vartree"],
+                                       trees[myroot]["porttree"].dbapi)
                        self.pkgsettings[myroot] = portage.config(
                                clone=self.trees[myroot]["vartree"].settings)
                        self.pkg_node_map[myroot] = {}
+                       self._slot_node_map[myroot] = {}
                        vardb = self.trees[myroot]["vartree"].dbapi
                        # This fakedbapi instance will model the state that the vdb will
                        # have after new packages have been installed.
                        fakedb = portage.fakedbapi(settings=self.pkgsettings[myroot])
                        self.mydbapi[myroot] = fakedb
                        for pkg in vardb.cpv_all():
-                               myslot = vardb.aux_get(pkg, ["SLOT"])[0]
-                               fakedb.cpv_inject(pkg, metadata={"SLOT":myslot})
+                               fakedb.cpv_inject(pkg,
+                                       metadata=dict(izip(self._mydbapi_keys,
+                                       vardb.aux_get(pkg, self._mydbapi_keys))))
                        del vardb, fakedb
                        self.useFlags[myroot] = {}
                        if "--usepkg" in self.myopts:
@@ -824,7 +844,6 @@ class depgraph:
                self.orderedkeys=[]
                self.outdatedpackages=[]
                self.args_keys = []
-               self.global_updates = {}
                self.blocker_digraph = digraph()
                self.blocker_parents = {}
                self._altlist_cache = {}
@@ -849,20 +868,16 @@ class depgraph:
                        if addme and jbigkey != myparent:
                                # Refuse to make a node depend on itself so that the we don't
                                # don't create a bogus circular dependency in self.altlist().
-                               self.pkg_node_map[myroot][mykey] = jbigkey
                                if rev_dep and myparent:
                                        ptype, proot, pkey, pstatus = myparent
-                                       self.pkg_node_map[proot][pkey] = myparent
                                        self.digraph.addnode(myparent, jbigkey, priority=priority)
                                else:
                                        self.digraph.addnode(jbigkey, myparent, priority=priority)
                        return 1
                jbigkey = tuple(chain(mybigkey, ["nomerge"]))
                if self.digraph.hasnode(jbigkey):
-                       self.pkg_node_map[myroot][mykey] = jbigkey
                        if rev_dep and myparent:
                                ptype, proot, pkey, pstatus = myparent
-                               self.pkg_node_map[proot][pkey] = myparent
                                self.digraph.addnode(myparent, jbigkey, priority=priority)
                        else:
                                self.digraph.addnode(jbigkey, myparent, priority=priority)
@@ -937,19 +952,21 @@ class depgraph:
                    depgraph already and returned, or we are here.  Whether we are merging or not; we must
                    add the package to the depgraph; so we do that here. """
                jbigkey = tuple(mybigkey)
-               self.pkg_node_map[myroot][mykey] = jbigkey
                if rev_dep and myparent:
                        ptype, proot, pkey, pstatus = myparent
-                       self.pkg_node_map[proot][pkey] = myparent
                        self.digraph.addnode(myparent, jbigkey,
                                priority=priority)
                else:
                        self.digraph.addnode(jbigkey, myparent,
                                priority=priority)
 
-               if addme:
-                       myslot = mydbapi.aux_get(mykey, ["SLOT"])[0]
-                       self.mydbapi[myroot].cpv_inject(mykey, metadata={"SLOT":myslot})
+               metadata = dict(izip(self._mydbapi_keys,
+                       mydbapi.aux_get(mykey, self._mydbapi_keys)))
+               slot_atom = "%s:%s" % (portage.dep_getkey(mykey), metadata["SLOT"])
+               self._slot_node_map[myroot][slot_atom] = jbigkey
+               if mybigkey[-1] == "merge":
+                       self.pkg_node_map[myroot][mykey] = jbigkey
+                       self.mydbapi[myroot].cpv_inject(mykey, metadata=metadata)
 
                """ This section determines whether we go deeper into dependencies or not.
                    We want to go deeper on a few occasions:
@@ -1368,15 +1385,6 @@ class depgraph:
                                                        myeb_inst, ["USE"])[0].split()
                                                matched_packages.append(
                                                        ["installed", myroot, myeb_inst, binpkguseflags])
-                                               """Now make sure that *DEPEND atoms are up to date.
-                                               This is done on the fly for single packages only when
-                                               necessary, since it can be time consuming to run this
-                                               on all installed packages."""
-                                               if myroot not in self.global_updates:
-                                                       self.global_updates[myroot] = \
-                                                               grab_global_updates(pkgsettings["PORTDIR"])
-                                               perform_global_updates(
-                                                       myeb_inst, vardb, self.global_updates[myroot])
 
                                if not matched_packages:
                                        if raise_on_missing:
@@ -1466,6 +1474,19 @@ class depgraph:
                        # been pulled into the depgraph.  This is not enabled by default
                        # due to the performance penalty that is incurred by all the
                        # additional dep_check calls that are required.
+
+                       # Optimization hack for dep_check calls that minimizes the
+                       # available matches by replacing the portdb with a fakedbapi
+                       # instance.
+                       class FakePortageTree(object):
+                               def __init__(self, mydb):
+                                       self.dbapi = mydb
+                       dep_check_trees = {}
+                       for myroot in self.trees:
+                               dep_check_trees[myroot] = self.trees[myroot].copy()
+                               dep_check_trees[myroot]["porttree"] = \
+                                       FakePortageTree(self.mydbapi[myroot])
+
                        dep_keys = ["DEPEND","RDEPEND","PDEPEND"]
                        for myroot in self.trees:
                                pkg_node_map = self.pkg_node_map[myroot]
@@ -1481,24 +1502,14 @@ class depgraph:
                                                # If this node has any blockers, create a "nomerge"
                                                # node for it so that they can be enforced.
                                                self.spinner.update()
-                                               try:
-                                                       dep_vals = portdb.aux_get(pkg, dep_keys)
-                                                       ptype = "ebuild"
-                                               except KeyError:
-                                                       if myroot not in self.global_updates:
-                                                               self.global_updates[myroot] = \
-                                                                       grab_global_updates(pkgsettings["PORTDIR"])
-                                                       perform_global_updates(
-                                                               pkg, vardb, self.global_updates[myroot])
-                                                       dep_vals = vardb.aux_get(pkg, dep_keys)
-                                                       ptype = "installed"
+                                               dep_vals = vardb.aux_get(pkg, dep_keys)
                                                myuse = vardb.aux_get(pkg, ["USE"])[0].split()
                                                depstr = " ".join(dep_vals)
                                                # It is crucial to pass in final_db here in order to
                                                # optimize dep_check calls by eliminating atoms via
                                                # dep_wordreduce and dep_eval calls.
                                                success, atoms = portage.dep_check(depstr, final_db,
-                                                       pkgsettings, myuse=myuse, trees=self.trees,
+                                                       pkgsettings, myuse=myuse, trees=dep_check_trees,
                                                        myroot=myroot)
                                                if not success:
                                                        print "\n\n"
@@ -1511,7 +1522,7 @@ class depgraph:
                                                # Don't store this parent in pkg_node_map, because it's
                                                # not needed there and it might overwrite a "merge"
                                                # node with the same cpv.
-                                               myparent = (ptype, myroot, pkg, "nomerge")
+                                               myparent = ("installed", myroot, pkg, "nomerge")
                                                for myatom in blocker_atoms:
                                                        blocker = ("blocks", myroot, myatom[1:])
                                                        myparents = \
@@ -1521,6 +1532,18 @@ class depgraph:
                                                                self.blocker_parents[blocker] = myparents
                                                        myparents.add(myparent)
 
+               modified_slots = {}
+               if self.blocker_parents:
+                       for myroot in self.trees:
+                               myslots = {}
+                               modified_slots[myroot] = myslots
+                               final_db = self.mydbapi[myroot]
+                               slot_node_map = self._slot_node_map[myroot]
+                               for slot_atom, mynode in slot_node_map.iteritems():
+                                       mytype, myroot, mycpv, mystatus = mynode
+                                       if mystatus == "merge":
+                                               modified_slots[myroot][slot_atom] = mycpv
+
                for blocker in self.blocker_parents.keys():
                        mytype, myroot, mydep = blocker
                        initial_db = self.trees[myroot]["vartree"].dbapi
@@ -1545,6 +1568,7 @@ class depgraph:
                                pdbapi = self.trees[proot][self.pkg_tree_map[ptype]].dbapi
                                pslot = pdbapi.aux_get(pcpv, ["SLOT"])[0]
                                pslot_atom = "%s:%s" % (portage.dep_getkey(pcpv), pslot)
+                               parent_static = pslot_atom not in modified_slots[proot]
                                unresolved_blocks = False
                                depends_on_order = set()
                                for cpv in blocked_initial:
@@ -1555,6 +1579,11 @@ class depgraph:
                                                # node matters.  In any case, this particular block is
                                                # automatically resolved.
                                                continue
+                                       if parent_static and \
+                                               slot_atom not in modified_slots[myroot]:
+                                               # This blocker will be handled the next time that a
+                                               # merge of either package is triggered.
+                                               continue
                                        if pstatus == "merge" and \
                                                slot_atom not in blocked_slots_final.values():
                                                upgrade_matches = final_db.match(slot_atom)
@@ -1574,6 +1603,11 @@ class depgraph:
                                                # The parent blocks itself, so the merge order does not
                                                # need to be enforced.
                                                continue
+                                       if parent_static and \
+                                               slot_atom not in modified_slots[myroot]:
+                                               # This blocker will be handled the next time that a
+                                               # merge of either package is triggered.
+                                               continue
                                        # None of the above blocker resolutions techniques apply,
                                        # so apparently this one is unresolvable.
                                        unresolved_blocks = True
@@ -3984,7 +4018,7 @@ def action_depclean(settings, trees, ldpath_mtimes,
        xterm_titles = "notitles" not in settings.features
        portdb = trees[settings["ROOT"]]["porttree"].dbapi
        real_vartree = trees[settings["ROOT"]]["vartree"]
-       vartree = FakeVartree(real_vartree)
+       vartree = FakeVartree(real_vartree, portdb)
        trees[settings["ROOT"]]["vartree"] = vartree
        vardb = vartree.dbapi
        syslist = getlist(settings, "system")
@@ -4018,26 +4052,9 @@ def action_depclean(settings, trees, ldpath_mtimes,
        unresolveable = {}
        aux_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
 
-       fake_vardb = portage.fakedbapi(settings=settings)
-       fakedb_auxkeys = aux_keys[:]
-       fakedb_auxkeys.append("SLOT")
-       global_updates = None
-       for cpv in myvarlist:
-               try:
-                       # Prefer live ebuild metadata when available.
-                       aux_vals = portdb.aux_get(cpv, fakedb_auxkeys)
-                       # SLOT always comes from the vardb, for multislot.
-                       aux_vals[-1] = vardb.aux_get(cpv, ["SLOT"])[0]
-                       live_ebuild = True
-               except KeyError:
-                       aux_vals = vardb.aux_get(cpv, fakedb_auxkeys)
-                       live_ebuild = False
-               fake_vardb.cpv_inject(
-                       cpv, metadata=dict(izip(fakedb_auxkeys, aux_vals)))
-               if not live_ebuild:
-                       if global_updates is None:
-                               global_updates = grab_global_updates(settings["PORTDIR"])
-                       perform_global_updates(cpv, fake_vardb, global_updates)
+       # Now that FakeVartree pulls metadata from the portdb and performs global
+       # updates when necessary, fake_vardb and vardb are equivalent.
+       fake_vardb = vardb
 
        # HACK: Ensure that installed packages are preferenced by dep_check().
        trees[settings["ROOT"]]["porttree"].dbapi = fake_vardb
index 36c7e3f5397d49292dd3b34bea4b1778bfd8a451..171073281850a889c2975658698a082a66da8fea 100644 (file)
@@ -2170,6 +2170,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
        can_fetch=True
 
+       for var_name in ("FETCHCOMMAND", "RESUMECOMMAND"):
+               if not mysettings.get(var_name, None):
+                       can_fetch = False
+
        if not listonly:
                dirmode  = 02070
                filemode =   060
@@ -2316,6 +2320,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
                                                else:
                                                        writemsg("!!! File %s isn't fully fetched, but unable to complete it\n" % myfile,
                                                                noiselevel=-1)
+                                               for var_name in ("FETCHCOMMAND", "RESUMECOMMAND"):
+                                                       if not mysettings.get(var_name, None):
+                                                               writemsg(("!!! %s is unset.  It should " + \
+                                                               "have been defined in /etc/make.globals.\n") \
+                                                                % var_name, noiselevel=-1)
                                                return 0
                                        else:
                                                continue
@@ -4333,6 +4342,12 @@ class dbapi:
        def cp_list(self,cp,use_cache=1):
                return
 
+       def cpv_all(self):
+               cpv_list = []
+               for cp in self.cp_all():
+                       cpv_list.extend(self.cp_list(cp))
+               return cpv_list
+
        def aux_get(self,mycpv,mylist):
                "stub code for returning auxiliary db information, such as SLOT, DEPEND, etc."
                'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
@@ -5524,12 +5539,6 @@ class portdbapi(dbapi):
                l.sort()
                return l
 
-       def cpv_all(self):
-               cpv_list = []
-               for cp in self.cp_all():
-                       cpv_list.extend(self.cp_list(cp))
-               return cpv_list
-
        def p_list(self,mycp):
                d={}
                for oroot in self.porttrees: