From d4a2d1d68cd961b1b1462e6c74b07f754766501d Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 18 Dec 2006 20:22:40 +0000 Subject: [PATCH] Merged from trunk 5295:5305 Cache blocked slots for better efficiency. For bug #157897, show parent nodes in order to help the user decide about what action to take. Cache KEYWORDS and EAPI metadata values in order to speed up portdbapi.gvisible() calls. This greatly improves repoman performance (help emerge a little too). Make sure eapi is defined in order to prevent a NameError. Split ACCEPT_KEYWORDS before entering the loop for better efficiency. Make repoman cache per-profile xmatch results. For bug #158177, force configure scripts that automatically detect ccache to respect FEATURES="-ccache". Thanks to Horst Schirmeier for this suggestion. Fix global updates error handling to prevent a traceback with python-2.5 as reported in bug #158264. Make dep_check validate all depstring atoms in order to prevent problems from occuring later due to unexpected invalid atoms. Make use_reduce raise an InvalidDependString exception for use? conditionals that aren't followed by parenthesis and add appropriate exception handling in dep_check. svn path=/main/branches/prefix/; revision=5319 --- bin/ebuild.sh | 4 +++ bin/emerge | 27 ++++++++++++----- bin/repoman | 8 +++-- pym/portage.py | 68 +++++++++++++++++++++++++------------------ pym/portage_dep.py | 8 +++-- pym/portage_update.py | 5 ++-- 6 files changed, 76 insertions(+), 44 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index f681c7784..944eb83d4 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1560,6 +1560,10 @@ if [[ ${EBUILD_SH_ARGS} != "depend" ]] && [[ ${EBUILD_SH_ARGS} != "clean" ]] && addwrite "${CCACHE_DIR}" [ -n "${CCACHE_SIZE}" ] && ccache -M ${CCACHE_SIZE} &> /dev/null + else + # Force configure scripts that automatically detect ccache to respect + # FEATURES="-ccache" + export CCACHE_DISABLE=1 fi # XXX: Load up the helper functions. diff --git a/bin/emerge b/bin/emerge index 171e1bc15..6f9cb449f 100755 --- a/bin/emerge +++ b/bin/emerge @@ -863,9 +863,15 @@ class depgraph: "package slot have been " print "!!! pulled into the dependency graph:" print - for cpv in packages: - print " ", cpv - print + for node, parents in packages: + print node, + if parents: + print "pulled in by" + for parent in parents: + print " ", parent + else: + print "(no parents)" + print msg = [] msg.append("It may be possible to solve this problem ") @@ -1001,7 +1007,13 @@ class depgraph: priority=priority) return 1 else: - self._show_slot_collision_notice((mykey, e_cpv)) + e_parents = self.digraph.parent_nodes(existing_node) + myparents = [] + if myparent: + myparents.append(myparent) + self._show_slot_collision_notice( + ((tuple(mybigkey), myparents), + (existing_node, e_parents))) return 0 jbigkey = tuple(mybigkey) @@ -1317,6 +1329,7 @@ class depgraph: if not mycheck[0]: sys.stderr.write("\n\n%s\n\n" % mycheck[1]) + sys.stderr.write("Parent: %s\n\n" % str(myparent)) sys.stderr.write("%s\n\n" % depstring) return 0 mymerge = mycheck[1] @@ -1498,9 +1511,6 @@ class depgraph: # The existing node can be reused. selected_pkg = [e_type, myroot, e_cpv, self.useFlags[myroot][e_cpv]] - else: - self._show_slot_collision_notice((mycpv, e_cpv)) - return 0 if myparent: #we are a dependency, so we want to be unconditionally added @@ -1627,6 +1637,7 @@ class depgraph: blocked_slots_final[cpv] = \ "%s:%s" % (portage.dep_getkey(cpv), final_db.aux_get(cpv, ["SLOT"])[0]) + blocked_slots_final_values = set(blocked_slots_final.itervalues()) for parent in list(self.blocker_parents[blocker]): ptype, proot, pcpv, pstatus = parent pdbapi = self.trees[proot][self.pkg_tree_map[ptype]].dbapi @@ -1649,7 +1660,7 @@ class depgraph: # merge of either package is triggered. continue if pstatus == "merge" and \ - slot_atom not in blocked_slots_final.values(): + slot_atom not in blocked_slots_final_values: upgrade_matches = final_db.match(slot_atom) if upgrade_matches: # Apparently an upgrade may be able to invalidate diff --git a/bin/repoman b/bin/repoman index 0cf726abc..ad02c3d59 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1254,13 +1254,16 @@ for x in scanlist: profdir = portdir+"/profiles/"+prof[0] if prof[0] in arch_caches: - dep_settings = arch_caches[prof[0]] + dep_settings, xcache = arch_caches[prof[0]] else: dep_settings = portage.config( config_profile_path=profdir, config_incrementals=portage_const.INCREMENTALS, local_config=False) - arch_caches[prof[0]] = dep_settings + portdb.melt() + portdb.freeze() + xcache = portdb.xcache + arch_caches[prof[0]] = (dep_settings, xcache) while True: try: # Protect ACCEPT_KEYWORDS from config.regenerate() @@ -1271,6 +1274,7 @@ for x in scanlist: trees["/"]["porttree"].settings = dep_settings portdb.mysettings = dep_settings + portdb.xcache = xcache # for package.use.mask support inside dep_check dep_settings.setcpv("/".join((catdir, y))) dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) diff --git a/pym/portage.py b/pym/portage.py index 171073281..4326c715f 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -3654,6 +3654,9 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", newsplit.append(_expand_new_virtuals(x, edebug, mydbapi, mysettings, myroot=myroot, trees=trees, **kwargs)) continue + if not isvalidatom(x, allow_blockers=True): + raise portage_exception.ParseError( + "invalid atom: '%s'" % x) mykey = dep_getkey(x) if not mykey.startswith("virtual/"): newsplit.append(x) @@ -3888,26 +3891,27 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, #convert parenthesis to sublists mysplit = portage_dep.paren_reduce(depstring) - if mysettings: - mymasks = set() - useforce = set([mysettings["ARCH"]]) - if use == "all": - # These masks are only for repoman. In other cases, relevant masks - # should have already been applied via config.regenerate(). Also, - # binary or installed packages may have been built with flags that - # are now masked, and it would be inconsistent to mask them now. - # Additionally, myuse may consist of flags from a parent package - # that is being merged to a $ROOT that is different from the one - # that mysettings represents. - mymasks.update(mysettings.usemask) - mymasks.update(mysettings.archlist()) - mymasks.discard(mysettings["ARCH"]) - useforce.update(mysettings.useforce) - useforce.difference_update(mymasks) + mymasks = set() + useforce = set() + useforce.add(mysettings["ARCH"]) + if use == "all": + # This masking/forcing is only for repoman. In other cases, relevant + # masking/forcing should have already been applied via + # config.regenerate(). Also, binary or installed packages may have + # been built with flags that are now masked, and it would be + # inconsistent to mask them now. Additionally, myuse may consist of + # flags from a parent package that is being merged to a $ROOT that is + # different from the one that mysettings represents. + mymasks.update(mysettings.usemask) + mymasks.update(mysettings.archlist()) + mymasks.discard(mysettings["ARCH"]) + useforce.update(mysettings.useforce) + useforce.difference_update(mymasks) + try: mysplit = portage_dep.use_reduce(mysplit, uselist=myusesplit, masklist=mymasks, matchall=(use=="all"), excludeall=useforce) - else: - mysplit = portage_dep.use_reduce(mysplit,uselist=myusesplit,matchall=(use=="all")) + except portage_exception.InvalidDependString, e: + return [0, str(e)] # Do the || conversions mysplit=portage_dep.dep_opconvert(mysplit) @@ -5170,6 +5174,7 @@ class portdbapi(dbapi): for x in self.porttrees: # location, label, auxdbkeys self.auxdb[x] = self.auxdbmodule(self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid) + self._gvisible_aux_cache = {} def _init_cache_dirs(self): """Create /var/cache/edb/dep and adjust permissions for the portage @@ -5684,27 +5689,34 @@ class portdbapi(dbapi): return [] newlist=[] + accept_keywords = self.mysettings["ACCEPT_KEYWORDS"].split() pkgdict = self.mysettings.pkeywordsdict for mycpv in mylist: #we need to update this next line when we have fully integrated the new db api auxerr=0 keys = None - try: - keys, eapi = self.aux_get(mycpv, ["KEYWORDS", "EAPI"]) - except KeyError: - pass - except portage_exception.PortageException, e: - writemsg("!!! Error: aux_get('%s', ['KEYWORDS', 'EAPI'])\n" % mycpv, - noiselevel=-1) - writemsg("!!! %s\n" % str(e), - noiselevel=-1) + eapi = None + aux_cache = self._gvisible_aux_cache.get(mycpv) + if aux_cache is not None: + keys, eapi = aux_cache + else: + try: + keys, eapi = self.aux_get(mycpv, ["KEYWORDS", "EAPI"]) + except KeyError: + pass + except portage_exception.PortageException, e: + writemsg("!!! Error: aux_get('%s', ['KEYWORDS', 'EAPI'])\n" % mycpv, + noiselevel=-1) + writemsg("!!! %s\n" % str(e), + noiselevel=-1) + self._gvisible_aux_cache[mycpv] = (keys, eapi) if not keys: # KEYWORDS="" #print "!!! No KEYWORDS for "+str(mycpv)+" -- Untested Status" continue mygroups=keys.split() # Repoman may modify this attribute as necessary. - pgroups = self.mysettings["ACCEPT_KEYWORDS"].split() + pgroups = accept_keywords[:] match=0 cp = dep_getkey(mycpv) if pkgdict.has_key(cp): diff --git a/pym/portage_dep.py b/pym/portage_dep.py index d5c60e658..feb499cf0 100644 --- a/pym/portage_dep.py +++ b/pym/portage_dep.py @@ -167,8 +167,9 @@ def use_reduce(deparray, uselist=[], masklist=[], matchall=0, excludeall=[]): for head in newdeparray[:-1]: head = head[:-1] if head[0] == "!": - head = head[1:] - if not matchall and head in uselist or head in excludeall: + head_key = head[1:] + if not matchall and head_key in uselist or \ + head_key in excludeall: ismatch = False break elif head not in masklist: @@ -186,7 +187,8 @@ def use_reduce(deparray, uselist=[], masklist=[], matchall=0, excludeall=[]): if additions: rlist.append(additions) else: - rlist += [target] + raise portage_exception.InvalidDependString( + "Conditional without parenthesis: '%s?'" % head) else: rlist += [head] diff --git a/pym/portage_update.py b/pym/portage_update.py index 57aa8b43b..f156c39bc 100644 --- a/pym/portage_update.py +++ b/pym/portage_update.py @@ -71,9 +71,8 @@ def grab_updates(updpath, prev_mtimes=None): mylist = os.listdir(updpath) except OSError, oe: if oe.errno == errno.ENOENT: - raise DirectoryNotFound(oe) - else: - raise oe + raise DirectoryNotFound(updpath) + raise if prev_mtimes is None: prev_mtimes = {} # validate the file name (filter out CVS directory, etc...) -- 2.26.2