From: Fabian Groffen Date: Sun, 11 Jan 2009 14:28:25 +0000 (-0000) Subject: Merged from trunk -r12404:12420 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=125998c12654b1348af1ef9feac194ba63c36084;p=portage.git Merged from trunk -r12404:12420 | 12405 | Add Blocker -> Blocked Package edges to | | zmedico | depgraph._unsolvable_blockers since these edges will be | | | useful for implementing an enhanced blocker display. | | 12406 | Remove unused myblocker_parents variable inside | | zmedico | depgraph._serialize_tasks(). | | 12407 | Revert r12405 since it will be more convenient to use a | | zmedico | separate graph to store blocked package edges. | | 12408 | When there are unresolved blockers, display the conflicting | | zmedico | packages along with the packages that pulled them in | | | (similar to the slot conflict display). This is helpful for | | | troubleshooting cases in which blockers don't solve | | | automatically and the reasons are not apparent from the | | | normal merge list display. | | 12409 | * Enable the recursive option in profile package.use | | zmedico | parsing. * Enable package.umask in all the same locations as | | | package.mask. | | 12410 | Bug #55321 - Add support for package.keywords in profiles. | | zmedico | In profiles, package.keywords modifies effective KEYWORDS | | | values for a given ebuild. This behavior is notably | | | different from /etc/portage/package.keywords, which instead | | | modifies effective ACCEPT_KEYWORDS. | | 12411 | Enable package.keywords as-directory in profiles. | | zmedico | | | 12412 | Fix broken code inside config._getKeywords(). | | zmedico | | | 12413 | Add a new 'metadata.warning' category and a check which | | zmedico | detects local USE descriptions that are not used in any | | | ebuilds. Thanks to Markus Meier for the initial | | | patch. | | 12414 | Inside config._getKeywords(), ignore -* in KEYWORDS, to | | zmedico | prevent it from causing preceeding values to be discarded. | | 12415 | Use ebegin/eend to indicate when waiting for a lock and when | | zmedico | it's acquired. Thanks to Petteri Räty for | | | the suggestion. | | 12416 | Add a "quiet" parameter to the EOutput constructor, and also | | zmedico | flush stderr/stdout stream in the constructor. | | 12417 | Inside action_sync(), check for the git binary when | | zmedico | necessary and bail out with a helpful message if it doesn't | | | exist. Thanks to Caleb Cushing for | | | the initial patch. | | 12418 | Bug #254421 - Make dir_get_metadata() handle 'Connection | | zmedico | refused' socket.error from ftplib connection attempt. | | 12419 | Enable use.{mask,force} and package.use.{mask,forc} | | zmedico | as-directory in profiles. | | 12420 | When displaying parents of unsolved blockers, reduce noise | | zmedico | by pruning packages that are only pulled in by other | | | conflict packages. | svn path=/main/branches/prefix/; revision=12424 --- diff --git a/bin/repoman b/bin/repoman index 8c253d24f..1f6be769d 100755 --- a/bin/repoman +++ b/bin/repoman @@ -313,6 +313,7 @@ qahelp={ "ebuild.badheader":"This ebuild has a malformed header", "metadata.missing":"Missing metadata.xml files", "metadata.bad":"Bad metadata.xml files", + "metadata.warning":"Warnings in metadata.xml files", "virtual.versioned":"PROVIDE contains virtuals with versions", "virtual.exists":"PROVIDE contains existing package names", "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default", @@ -351,6 +352,7 @@ qawarnings = set(( "file.size", "inherit.autotools", "java.eclassesnotused", +"metadata.warning", "virtual.versioned", "virtual.exists", "virtual.unavailable", @@ -1063,11 +1065,14 @@ for x in scanlist: fails["metadata.bad"].append(x+"/metadata.xml") del metadata_bad + muselist = frozenset(muselist) changelog_path = os.path.join(checkdir_relative, "ChangeLog") changelog_modified = changelog_path in modified_changelogs allmasked = True + # detect unused local USE-descriptions + used_useflags = set() for y in ebuildlist: relative_path = os.path.join(x, y + ".ebuild") @@ -1368,6 +1373,7 @@ for x in scanlist: default_use.append(myflag) if flag_name not in uselist: myuse.append(flag_name) + used_useflags.update(myuse) # uselist checks - metadata for mypos in range(len(myuse)-1,-1,-1): @@ -1576,6 +1582,12 @@ for x in scanlist: stats["ebuild.allmasked"]+=1 fails["ebuild.allmasked"].append(x) + # check if there are unused local USE-descriptions in metadata.xml + for myflag in muselist.difference(used_useflags): + stats["metadata.warning"] += 1 + fails["metadata.warning"].append( + "unused local USE-description: '%s'" % myflag) + if options.mode == "manifest": sys.exit(dofail) diff --git a/man/portage.5 b/man/portage.5 index d95d28d83..fac8c90eb 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -29,8 +29,10 @@ eapi make.defaults packages packages.build +package.keywords package.mask package.provided +package.unmask package.use package.use.force package.use.mask @@ -584,6 +586,26 @@ A list of all the packages which will be displayed when you run `emerge info`. .BR info_vars A list of all the variables which will be displayed when you run `emerge info`. .TP +.BR package.keywords +Per\-profile KEYWORDS. Useful for cases in which the effective KEYWORDS of a +given package should vary depending on which profile the user has selected. + +.I Format: +.nf +\- comment lines begin with # (no inline comments) +\- one DEPEND atom per line followed by additional KEYWORDS +.fi + +.I Example: +.nf +# add stable keyword to libgd +media\-libs/libgd x86 +# remove stable keyword from mplayer and add unstable keyword +media\-video/mplayer \-x86 ~x86 +# remove all keywords from netcat +net-analyzer/netcat -* +.fi +.TP .BR package.mask This contains a list of DEPEND atoms for packages that should not be installed in any profile. Useful for adding the latest KDE betas and making sure no diff --git a/man/repoman.1 b/man/repoman.1 index 0380adfd6..754214049 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -263,6 +263,9 @@ Bad metadata.xml files .B metadata.missing Missing metadata.xml files .TP +.B metadata.warning +Warnings in metadata.xml files +.TP .B upstream.workaround The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index a95eb995c..7c235b3a6 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -4511,6 +4511,12 @@ class depgraph(object): self._irrelevant_blockers = digraph() # Contains only unsolvable Package -> Blocker edges self._unsolvable_blockers = digraph() + # Contains all Blocker -> Blocked Package edges + self._blocked_pkgs = digraph() + # Contains world packages that have been protected from + # uninstallation but may not have been added to the graph + # if the graph is not complete yet. + self._blocked_world_pkgs = {} self._slot_collision_info = {} # Slot collision nodes are not allowed to block other packages since # blocker validation is only able to account for one package per slot. @@ -6475,6 +6481,9 @@ class depgraph(object): # is already done and this would be likely to # confuse users if displayed like a normal blocker. continue + + self._blocked_pkgs.add(pkg, blocker) + if parent.operation == "merge": # Maybe the blocked package can be replaced or simply # unmerged to resolve this block. @@ -6493,6 +6502,8 @@ class depgraph(object): # merge of either package is triggered. continue + self._blocked_pkgs.add(pkg, blocker) + # Maybe the blocking package can be # unmerged to resolve this block. if parent.operation == "merge" and pkg.installed: @@ -6545,7 +6556,7 @@ class depgraph(object): def _accept_blocker_conflicts(self): acceptable = False for x in ("--buildpkgonly", "--fetchonly", - "--fetch-all-uri", "--nodeps", "--pretend"): + "--fetch-all-uri", "--nodeps"): if x in self.myopts: acceptable = True break @@ -6676,7 +6687,6 @@ class depgraph(object): ignored_uninstall_tasks = set() have_uninstall_task = False complete = "complete" in self.myparams - myblocker_parents = self._blocker_parents.copy() asap_nodes = [] def get_nodes(**kwargs): @@ -7009,6 +7019,7 @@ class depgraph(object): break if not satisfied: skip = True + self._blocked_world_pkgs[inst_pkg] = atom break except portage.exception.InvalidDependString, e: portage.writemsg("!!! Invalid PROVIDE in " + \ @@ -7240,6 +7251,89 @@ class depgraph(object): portage.writemsg("\n", noiselevel=-1) for line in wrap(msg, 70): portage.writemsg(prefix + line + "\n", noiselevel=-1) + + # Display the conflicting packages along with the packages + # that pulled them in. This is helpful for troubleshooting + # cases in which blockers don't solve automatically and + # the reasons are not apparent from the normal merge list + # display. + + conflict_pkgs = {} + for blocker in blockers: + for pkg in chain(self._blocked_pkgs.child_nodes(blocker), \ + self._blocker_parents.parent_nodes(blocker)): + parent_atoms = self._parent_atoms.get(pkg) + if not parent_atoms: + atom = self._blocked_world_pkgs.get(pkg) + if atom is not None: + parent_atoms = set([("@world", atom)]) + if parent_atoms: + conflict_pkgs[pkg] = parent_atoms + + if conflict_pkgs: + # Reduce noise by pruning packages that are only + # pulled in by other conflict packages. + pruned_pkgs = set() + for pkg, parent_atoms in conflict_pkgs.iteritems(): + relevant_parent = False + for parent, atom in parent_atoms: + if parent not in conflict_pkgs: + relevant_parent = True + break + if not relevant_parent: + pruned_pkgs.add(pkg) + for pkg in pruned_pkgs: + del conflict_pkgs[pkg] + + if conflict_pkgs: + msg = [] + msg.append("\n") + indent = " " + # Max number of parents shown, to avoid flooding the display. + max_parents = 3 + for pkg, parent_atoms in conflict_pkgs.iteritems(): + + pruned_list = set() + + # Prefer conflict packages over others. + for parent_atom in parent_atoms: + if len(pruned_list) >= max_parents: + break + parent, atom = parent_atom + if parent in conflict_pkgs: + pruned_list.add(parent_atom) + + for parent_atom in parent_atoms: + if len(pruned_list) >= max_parents: + break + pruned_list.add(parent_atom) + + omitted_parents = len(parent_atoms) - len(pruned_list) + msg.append(indent + "%s pulled in by\n" % pkg) + + for parent_atom in pruned_list: + parent, atom = parent_atom + msg.append(2*indent) + if isinstance(parent, + (PackageArg, AtomArg)): + # For PackageArg and AtomArg types, it's + # redundant to display the atom attribute. + msg.append(str(parent)) + else: + # Display the specific atom from SetArg or + # Package types. + msg.append("%s required by %s" % (atom, parent)) + msg.append("\n") + + if omitted_parents: + msg.append(2*indent) + msg.append("(and %d more)\n" % omitted_parents) + + msg.append("\n") + + sys.stderr.write("".join(msg)) + sys.stderr.flush() + if "--quiet" not in self.myopts: show_blocker_docs_link() @@ -11911,6 +12005,13 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # going to trust the user and assume that the user is in the branch # that he/she wants updated. We'll let the user manage branches with # git directly. + if portage.process.find_binary("git") is None: + msg = ["Command not found: git", + "Type \"emerge dev-util/git\" to enable git support."] + for l in msg: + writemsg_level("!!! %s\n" % l, + level=logging.ERROR, noiselevel=-1) + return 1 msg = ">>> Starting git pull in %s..." % myportdir emergelog(xterm_titles, msg ) writemsg_level(msg + "\n") diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index f205a83a3..d6aae6b69 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1139,6 +1139,7 @@ class config(object): self.pusedict = copy.deepcopy(clone.pusedict) self.categories = copy.deepcopy(clone.categories) self.pkeywordsdict = copy.deepcopy(clone.pkeywordsdict) + self._pkeywords_list = copy.deepcopy(clone._pkeywords_list) self.pmaskdict = copy.deepcopy(clone.pmaskdict) self.punmaskdict = copy.deepcopy(clone.punmaskdict) self.prevmaskdict = copy.deepcopy(clone.prevmaskdict) @@ -1268,9 +1269,19 @@ class config(object): else: self.prevmaskdict[mycatpkg].append(x) - # get profile-masked use flags -- INCREMENTAL Child over parent - self.usemask_list = [grabfile(os.path.join(x, "use.mask")) \ + self._pkeywords_list = [] + rawpkeywords = [grabdict_package( + os.path.join(x, "package.keywords"), recursive=1) \ for x in self.profiles] + for i in xrange(len(self.profiles)): + cpdict = {} + for k, v in rawpkeywords[i].iteritems(): + cpdict.setdefault(dep_getkey(k), {})[k] = v + self._pkeywords_list.append(cpdict) + + # get profile-masked use flags -- INCREMENTAL Child over parent + self.usemask_list = [grabfile(os.path.join(x, "use.mask"), + recursive=1) for x in self.profiles] self.usemask = set(stack_lists( self.usemask_list, incremental=True)) use_defs_lists = [grabdict(os.path.join(x, "use.defaults")) for x in self.profiles] @@ -1278,9 +1289,8 @@ class config(object): del use_defs_lists self.pusemask_list = [] - rawpusemask = [grabdict_package( - os.path.join(x, "package.use.mask")) \ - for x in self.profiles] + rawpusemask = [grabdict_package(os.path.join(x, "package.use.mask"), + recursive=1) for x in self.profiles] for i in xrange(len(self.profiles)): cpdict = {} for k, v in rawpusemask[i].iteritems(): @@ -1289,9 +1299,8 @@ class config(object): del rawpusemask self.pkgprofileuse = [] - rawprofileuse = [grabdict_package( - os.path.join(x, "package.use"), juststrings=True) \ - for x in self.profiles] + rawprofileuse = [grabdict_package(os.path.join(x, "package.use"), + juststrings=True, recursive=1) for x in self.profiles] for i in xrange(len(self.profiles)): cpdict = {} for k, v in rawprofileuse[i].iteritems(): @@ -1299,14 +1308,14 @@ class config(object): self.pkgprofileuse.append(cpdict) del rawprofileuse - self.useforce_list = [grabfile(os.path.join(x, "use.force")) \ - for x in self.profiles] + self.useforce_list = [grabfile(os.path.join(x, "use.force"), + recursive=1) for x in self.profiles] self.useforce = set(stack_lists( self.useforce_list, incremental=True)) self.puseforce_list = [] rawpuseforce = [grabdict_package( - os.path.join(x, "package.use.force")) \ + os.path.join(x, "package.use.force"), recursive=1) \ for x in self.profiles] for i in xrange(len(self.profiles)): cpdict = {} @@ -1527,17 +1536,6 @@ class config(object): self._plicensedict[cp] = cp_dict cp_dict[k] = self.expandLicenseTokens(v) - #package.unmask - pkgunmasklines = grabfile_package( - os.path.join(abs_user_config, "package.unmask"), - recursive=1) - for x in pkgunmasklines: - mycatpkg=dep_getkey(x) - if mycatpkg in self.punmaskdict: - self.punmaskdict[mycatpkg].append(x) - else: - self.punmaskdict[mycatpkg]=[x] - #getting categories from an external file now categories = [grabfile(os.path.join(x, "categories")) for x in locations] self.categories = stack_lists(categories, incremental=1) @@ -1547,12 +1545,16 @@ class config(object): archlist = stack_lists(archlist, incremental=1) self.configdict["conf"]["PORTAGE_ARCHLIST"] = " ".join(archlist) - #package.mask + # package.mask and package.unmask pkgmasklines = [] + pkgunmasklines = [] for x in pmask_locations: pkgmasklines.append(grabfile_package( os.path.join(x, "package.mask"), recursive=1)) + pkgunmasklines.append(grabfile_package( + os.path.join(x, "package.unmask"), recursive=1)) pkgmasklines = stack_lists(pkgmasklines, incremental=1) + pkgunmasklines = stack_lists(pkgunmasklines, incremental=1) self.pmaskdict = {} for x in pkgmasklines: @@ -1562,6 +1564,13 @@ class config(object): else: self.pmaskdict[mycatpkg]=[x] + for x in pkgunmasklines: + mycatpkg=dep_getkey(x) + if mycatpkg in self.punmaskdict: + self.punmaskdict[mycatpkg].append(x) + else: + self.punmaskdict[mycatpkg]=[x] + pkgprovidedlines = [grabfile(os.path.join(x, "package.provided")) for x in self.profiles] pkgprovidedlines = stack_lists(pkgprovidedlines, incremental=1) has_invalid_data = False @@ -2260,6 +2269,25 @@ class config(object): return x return None + def _getKeywords(self, cpv, metadata): + cp = dep_getkey(cpv) + pkg = "%s:%s" % (cpv, metadata["SLOT"]) + keywords = [[x for x in metadata["KEYWORDS"].split() if x != "-*"]] + pos = len(keywords) + for i in xrange(len(self.profiles)): + cpdict = self._pkeywords_list[i].get(cp, None) + if cpdict: + keys = list(cpdict) + while keys: + best_match = best_match_to_list(pkg, keys) + if best_match: + keys.remove(best_match) + keywords.insert(pos, cpdict[best_match]) + else: + break + pos = len(keywords) + return stack_lists(keywords, incremental=True) + def _getMissingKeywords(self, cpv, metadata): """ Take a package and return a list of any KEYWORDS that the user may @@ -2281,7 +2309,7 @@ class config(object): # object (bug #139600) egroups = self.configdict["backupenv"].get( "ACCEPT_KEYWORDS", "").split() - mygroups = metadata["KEYWORDS"].split() + mygroups = self._getKeywords(cpv, metadata) # Repoman may modify this attribute as necessary. pgroups = self["ACCEPT_KEYWORDS"].split() match=0 @@ -7011,7 +7039,7 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): # keywords checking eapi = metadata["EAPI"] - mygroups = metadata["KEYWORDS"] + mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] slot = metadata["SLOT"] if eapi.startswith("-"): @@ -7022,7 +7050,6 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): return ["EAPI %s" % eapi] egroups = settings.configdict["backupenv"].get( "ACCEPT_KEYWORDS", "").split() - mygroups = mygroups.split() pgroups = settings["ACCEPT_KEYWORDS"].split() myarch = settings["ARCH"] if pgroups and myarch not in pgroups: diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 6414d0c04..bf00ebd65 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -9,6 +9,7 @@ import portage.xpak import HTMLParser import sys import os +import socket import time import tempfile import base64 @@ -475,7 +476,14 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= if makepickle is None: makepickle = CACHE_PATH+"/metadata.idx.most_recent" - conn,protocol,address,params,headers = create_conn(baseurl, conn) + try: + conn, protocol, address, params, headers = create_conn(baseurl, conn) + except socket.error, e: + # ftplib.FTP(host) can raise errors like this: + # socket.error: (111, 'Connection refused') + sys.stderr.write("!!! %s\n" % (e,)) + return {} + out = sys.stdout try: metadatafile = open(CACHE_PATH+"/remote_metadata.pickle") diff --git a/pym/portage/locks.py b/pym/portage/locks.py index 830c06b00..659ad3d4b 100644 --- a/pym/portage/locks.py +++ b/pym/portage/locks.py @@ -11,6 +11,7 @@ import errno, os, stat, time, types from portage.exception import DirectoryNotFound, FileNotFound, \ InvalidData, TryAgain from portage.data import portage_gid +from portage.output import EOutput from portage.util import writemsg from portage.localization import _ import portage.const @@ -93,19 +94,21 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, raise TryAgain(mypath) global _quiet - if _quiet: - pass - elif waiting_msg is None: + out = EOutput() + out.quiet = _quiet + if waiting_msg is None: if isinstance(mypath, int): - writemsg("waiting for lock on fd %i\n" % myfd, - noiselevel=-1) + waiting_msg = "waiting for lock on fd %i" % myfd else: - writemsg("waiting for lock on %s\n" % lockfilename, - noiselevel=-1) - elif waiting_msg: - writemsg(waiting_msg + "\n", noiselevel=-1) + waiting_msg = "waiting for lock on %s\n" % lockfilename + out.ebegin(waiting_msg) # try for the exclusive lock now. - fcntl.lockf(myfd,fcntl.LOCK_EX) + try: + fcntl.lockf(myfd, fcntl.LOCK_EX) + except EnvironmentError, e: + out.eend(1, str(e)) + raise + out.eend(os.EX_OK) elif e.errno == errno.ENOLCK: # We're not allowed to lock on this FS. os.close(myfd) diff --git a/pym/portage/output.py b/pym/portage/output.py index d52063e25..db22084df 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -437,10 +437,10 @@ class EOutput(object): @type term_columns: IntType """ - def __init__(self): + def __init__(self, quiet=False): self.__last_e_cmd = "" self.__last_e_len = 0 - self.quiet = False + self.quiet = quiet lines, columns = get_term_size() if columns <= 0: columns = 80 @@ -448,6 +448,8 @@ class EOutput(object): if os.environ.get("TERM") in ("cons25", "dumb"): columns = columns - 1 self.term_columns = columns + sys.stdout.flush() + sys.stderr.flush() def __eend(self, caller, errno, msg): if errno == 0: @@ -461,10 +463,11 @@ class EOutput(object): self.ewarn(msg[0]) if self.__last_e_cmd != "ebegin": self.__last_e_len = 0 - out = sys.stdout - out.write("%*s%s\n" % \ - ((self.term_columns - self.__last_e_len - 6), "", status_brackets)) - out.flush() + if not self.quiet: + out = sys.stdout + out.write("%*s%s\n" % ((self.term_columns - self.__last_e_len - 6), + "", status_brackets)) + out.flush() def ebegin(self, msg): """