From: Fabian Groffen <grobian@gentoo.org>
Date: Sun, 10 Dec 2006 11:23:47 +0000 (-0000)
Subject: Merged from trunk 5247:5256
X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=afddd1ec1e3504ab96bd39533a0bdd25e008fa23;p=portage.git

Merged from trunk 5247:5256

For bug #128809, process blockers for all installed packages.  Due to
the performance penalty incurred by additional dep_check calls, this is
only enabled with the --deep option.

Avoid unnecessary dbapi.cp_list() calls inside cpv_expand().

Cache the results of match_from_list() calls.

Cache results of dep_getcpv() calls.

Use dep_getcpv() inside dep_expand() for code sharing and caching.

Cache the results of fakedbapi.match() calls and clear the cache
automatically when necessary.

Initialize the match cache in the bindbapi constructor.

For bug #128809, optimize dep_check calls and enable the blocker
detection by default since the performance now seems acceptable.

Make sure the SLOT comes from the vardb instead of the portdb (for
multislot,   etc..). Thanks to jakub for reporting.


svn path=/main/branches/prefix/; revision=5259
---

diff --git a/bin/emerge b/bin/emerge
index c296e8c2b..6d744fe12 100755
--- a/bin/emerge
+++ b/bin/emerge
@@ -1191,7 +1191,8 @@ class depgraph:
 					missing += 1
 					print "Missing binary for:",xs[2]
 
-		self.validate_blockers()
+		if not self.validate_blockers():
+			return False, myfavorites
 		
 		# We're true here unless we are missing binaries.
 		return (not missing,myfavorites)
@@ -1438,6 +1439,67 @@ class depgraph:
 		correct merge order such that mutually blocking packages are never
 		installed simultaneously."""
 
+		#if "deep" in self.myparams:
+		if True:
+			# Pull in blockers from all installed packages that haven't already
+			# been pulled into the depgraph.  This is not enabled by default
+			# due to the performance penalty that is incurred by all the
+			# additional dep_check calls that are required.
+			dep_keys = ["DEPEND","RDEPEND","PDEPEND"]
+			for myroot in self.trees:
+				pkg_node_map = self.pkg_node_map[myroot]
+				vardb = self.trees[myroot]["vartree"].dbapi
+				portdb = self.trees[myroot]["porttree"].dbapi
+				pkgsettings = self.pkgsettings[myroot]
+				final_db = self.mydbapi[myroot]
+				for pkg in self.trees[myroot]["vartree"].dbapi.cpv_all():
+					blocker_atoms = None
+					matching_node = pkg_node_map.get(pkg, None)
+					if not matching_node or \
+						matching_node[3] == "merge":
+						# If this node has any blockers, create a "nomerge"
+						# node for it so that they can be enforced.
+						self.spinner.update()
+						try:
+							dep_vals = portdb.aux_get(pkg, dep_keys)
+							ptype = "ebuild"
+						except KeyError:
+							if myroot not in self.global_updates:
+								self.global_updates[myroot] = \
+									grab_global_updates(pkgsettings["PORTDIR"])
+							perform_global_updates(
+								pkg, vardb, self.global_updates[myroot])
+							dep_vals = vardb.aux_get(pkg, dep_keys)
+							ptype = "installed"
+						myuse = vardb.aux_get(pkg, ["USE"])[0]
+						depstr = " ".join(dep_vals)
+						# It is crucial to pass in final_db here in order to
+						# optimize dep_check calls by eliminating atoms via
+						# dep_wordreduce and dep_eval calls.
+						success, atoms = portage.dep_check(depstr, final_db,
+							pkgsettings, myuse=myuse, trees=self.trees,
+							myroot=myroot)
+						if not success:
+							print "\n\n"
+							print "Error occurred while processing",pkg
+							print str(atoms)
+							return False
+						blocker_atoms = [myatom for myatom in atoms \
+							if myatom.startswith("!")]
+					if blocker_atoms:
+						# Don't store this parent in pkg_node_map, because it's
+						# not needed there and it might overwrite a "merge"
+						# node with the same cpv.
+						myparent = (ptype, myroot, pkg, "nomerge")
+						for myatom in blocker_atoms:
+							blocker = ("blocks", myroot, myatom[1:])
+							myparents = \
+								self.blocker_parents.get(blocker, None)
+							if not myparents:
+								myparents = set()
+								self.blocker_parents[blocker] = myparents
+							myparents.add(myparent)
+
 		for blocker in self.blocker_parents.keys():
 			mytype, myroot, mydep = blocker
 			initial_db = self.trees[myroot]["vartree"].dbapi
@@ -1510,6 +1572,7 @@ class depgraph:
 		# Validate blockers that depend on merge order.
 		if not self.blocker_digraph.empty():
 			self.altlist()
+		return True
 
 	def altlist(self, reversed=False):
 		if reversed in self._altlist_cache:
@@ -1711,7 +1774,8 @@ class depgraph:
 			except ValueError:
 				missing_atoms.append(mydep)
 
-		self.validate_blockers()
+		if not self.validate_blockers():
+			return False
 
 		if world_problems:
 			print >> sys.stderr, "\n!!! Problems have been detected with your world file"
@@ -1861,7 +1925,7 @@ class depgraph:
 					pkg_key, mydb=vardb, settings=pkgsettings)
 				print "["+x[0]+" "+addl+"]",red(resolved),
 				block_parents = self.blocker_parents[tuple(x)]
-				block_parents = [pnode[2] for pnode in block_parents]
+				block_parents = set([pnode[2] for pnode in block_parents])
 				block_parents = ", ".join(block_parents)
 				if resolved!=x[2]:
 					print bad("(\"%s\" is blocking %s)") % \
@@ -3936,6 +4000,8 @@ def action_depclean(settings, trees, ldpath_mtimes,
 		try:
 			# Prefer live ebuild metadata when available.
 			aux_vals = portdb.aux_get(cpv, fakedb_auxkeys)
+			# SLOT always comes from the vardb, for multislot.
+			aux_vals[-1] = vardb.aux_get(cpv, ["SLOT"])[0]
 			live_ebuild = True
 		except KeyError:
 			aux_vals = vardb.aux_get(cpv, fakedb_auxkeys)
diff --git a/pym/portage.py b/pym/portage.py
index d66dfee66..d7678ecc0 100644
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -3844,21 +3844,11 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
 		return mydep
 	if mydep[0]=="*":
 		mydep=mydep[1:]
-	prefix=""
-	postfix=""
-	if mydep[-1]=="*":
-		mydep=mydep[:-1]
-		postfix="*"
-	if mydep[:2] in [ ">=", "<=" ]:
-		prefix=mydep[:2]
-		mydep=mydep[2:]
-	elif mydep[:1] in "=<>~!":
-		prefix=mydep[:1]
-		mydep=mydep[1:]
-	colon = mydep.rfind(":")
-	if colon != -1:
-		postfix = mydep[colon:]
-		mydep = mydep[:colon]
+	orig_dep = mydep
+	mydep = dep_getcpv(orig_dep)
+	myindex = orig_dep.index(mydep)
+	prefix = orig_dep[:myindex]
+	postfix = orig_dep[myindex+len(mydep):]
 	return prefix + cpv_expand(
 		mydep, mydb=mydb, use_cache=use_cache, settings=settings) + postfix
 
@@ -4032,10 +4022,10 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
 			mykey=myslash[0]+"/"+mysplit[0]
 		else:
 			mykey=mycpv
-		if mydb:
+		if mydb and virts and mykey in virts:
 			writemsg("mydb.__class__: %s\n" % (mydb.__class__), 1)
 			if type(mydb)==types.InstanceType:
-				if (not mydb.cp_list(mykey,use_cache=use_cache)) and virts and virts.has_key(mykey):
+				if not mydb.cp_list(mykey, use_cache=use_cache):
 					writemsg("virts[%s]: %s\n" % (str(mykey),virts[mykey]), 1)
 					mykey_orig = mykey[:]
 					for vkey in virts[mykey]:
@@ -4387,6 +4377,19 @@ class fakedbapi(dbapi):
 		if settings is None:
 			settings = globals()["settings"]
 		self.settings = settings
+		self._match_cache = {}
+
+	def _clear_cache(self):
+		if self._match_cache:
+			self._match_cache = {}
+
+	def match(self, origdep, use_cache=1):
+		result = self._match_cache.get(origdep, None)
+		if result is not None:
+			return result[:]
+		result = dbapi.match(self, origdep, use_cache=use_cache)
+		self._match_cache[origdep] = result
+		return result[:]
 
 	def cpv_exists(self,mycpv):
 		return self.cpvdict.has_key(mycpv)
@@ -4408,6 +4411,7 @@ class fakedbapi(dbapi):
 
 	def cpv_inject(self, mycpv, metadata=None):
 		"""Adds a cpv from the list of available packages."""
+		self._clear_cache()
 		mycp=cpv_getkey(mycpv)
 		self.cpvdict[mycpv] = metadata
 		myslot = None
@@ -4439,6 +4443,7 @@ class fakedbapi(dbapi):
 
 	def cpv_remove(self,mycpv):
 		"""Removes a cpv from the list of available packages."""
+		self._clear_cache()
 		mycp=cpv_getkey(mycpv)
 		if self.cpvdict.has_key(mycpv):
 			del	self.cpvdict[mycpv]
@@ -4458,6 +4463,7 @@ class fakedbapi(dbapi):
 		return [metadata.get(x, "") for x in wants]
 
 	def aux_update(self, cpv, values):
+		self._clear_cache()
 		self.cpvdict[cpv].update(values)
 
 class bindbapi(fakedbapi):
@@ -4468,6 +4474,7 @@ class bindbapi(fakedbapi):
 		if settings is None:
 			settings = globals()["settings"]
 		self.settings = settings
+		self._match_cache = {}
 
 	def match(self, *pargs, **kwargs):
 		if self.bintree and not self.bintree.populated:
diff --git a/pym/portage_dep.py b/pym/portage_dep.py
index 6459137e0..72fa3f0c3 100644
--- a/pym/portage_dep.py
+++ b/pym/portage_dep.py
@@ -257,6 +257,8 @@ def get_operator(mydep):
 
 	return operator
 
+_dep_getcpv_cache = {}
+
 def dep_getcpv(mydep):
 	"""
 	Return the category-package-version with any operators/slot specifications stripped off
@@ -270,6 +272,11 @@ def dep_getcpv(mydep):
 	@rtype: String
 	@return: The depstring with the operator removed
 	"""
+	global _dep_getcpv_cache
+	retval = _dep_getcpv_cache.get(mydep, None)
+	if retval is not None:
+		return retval
+	mydep_orig = mydep
 	if mydep and mydep[0] == "*":
 		mydep = mydep[1:]
 	if mydep and mydep[-1] == "*":
@@ -282,7 +289,8 @@ def dep_getcpv(mydep):
 		mydep = mydep[1:]
 	colon = mydep.rfind(":")
 	if colon != -1:
-		return mydep[:colon]
+		mydep = mydep[:colon]
+	_dep_getcpv_cache[mydep_orig] = mydep
 	return mydep
 
 def dep_getslot(mydep):
@@ -478,6 +486,8 @@ def best_match_to_list(mypkg, mylist):
 			bestm  = x
 	return bestm
 
+_match_from_list_cache = {}
+
 def match_from_list(mydep, candidate_list):
 	"""
 	Searches list for entries that matches the package.
@@ -490,6 +500,11 @@ def match_from_list(mydep, candidate_list):
 	@return: A list of package atoms that match the given package atom
 	"""
 
+	global _match_from_list_cache
+	mylist = _match_from_list_cache.get((mydep, tuple(candidate_list)), None)
+	if mylist is not None:
+		return mylist[:]
+
 	from portage_util import writemsg
 	if mydep[0] == "!":
 		mydep = mydep[1:]
@@ -585,4 +600,5 @@ def match_from_list(mydep, candidate_list):
 	else:
 		raise KeyError("Unknown operator: %s" % mydep)
 
+	_match_from_list_cache[(mydep, tuple(candidate_list))] = mylist
 	return mylist