From: Fabian Groffen Date: Wed, 20 Dec 2006 18:11:14 +0000 (-0000) Subject: Merged from trunk 5320:5333 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=f891d4fe60e9a8c1d4f51203ecce036043607bdc;p=portage.git Merged from trunk 5320:5333 For bug #158503, allow --nodeps to override blockers. Increase cache hits in portdbapi.xmatch(). Use a separate digraph to store parent/child relationships (PDEPEND relationships are not reversed) so that relationship is always known in case a slot collision occurs. New elog modules for dismerging messages into a single file/mail (genone) Make binarytree.move_ent() properly update PF. For bug #158558, fix broken ebuild rename logic in vardbapi.move_ent(). For bug #158558, make the unmerge phase do automatic cleanup for broken vardbapi.move_ent() in portage versions before 2.1.2. Fix an import statement typo. For bug #158608, don't attempt to use distlocks when it's detected that the filesystem is readonly (unless "skiprocheck" is in FEATURES for some strange reason). Add missing newlines to the warning message that's printed when the user has distlocks enabled together with skiprocheck and the filesystem is readonly. Give the user some advice whenever an invalid dependency string is encountered. Fix the invalid dependency message so there's no confusion about which package has the problem. svn path=/main/branches/prefix/; revision=5334 --- diff --git a/NEWS b/NEWS index fe150b4a8..9e466b550 100644 --- a/NEWS +++ b/NEWS @@ -19,6 +19,11 @@ portage-2.1.2 * DEPEND atoms support SLOT dependencies of the form ${CATEGORY}/${PN}:${SLOT}. * Development: Extend PYTHONPATH support to allow overriding the hardcoded /usr/lib/portage/pym for development/testing purposes +* New "finalize" hook for elog modules enabling modules to perform actions on + shutdown (like dispatching collected messages). +* New elog modules mail_summary and save_summary that act like the mail/save + modules except that they merge messages from multiple packages in a single + file/mail. portage-2.1.1 ------------ diff --git a/bin/emerge b/bin/emerge index f309b4efd..2db414e07 100755 --- a/bin/emerge +++ b/bin/emerge @@ -786,6 +786,38 @@ def perform_global_updates(mycpv, mydb, mycommands): if updates: mydb.aux_update(mycpv, updates) + +def show_invalid_depstring_notice(parent_node, depstring, error_msg): + + from formatter import AbstractFormatter, DumbWriter + f = AbstractFormatter(DumbWriter(maxcol=72)) + + print "\n\n!!! Invalid or corrupt dependency specification: " + print + print error_msg + print + print parent_node + print + print depstring + print + p_type, p_root, p_key, p_status = parent_node + msg = [] + if p_status == "nomerge": + msg.append("Portage is unable to process the dependencies of the ") + msg.append("'%s' package. " % p_key) + msg.append("In order to correct this problem, the package ") + msg.append("should be uninstalled, reinstalled, or upgraded. ") + msg.append("As a temporary workaround, the --nodeps option can ") + msg.append("be used to ignore all dependencies.") + else: + msg.append("This package can not be installed. ") + msg.append("Please notify the '%s' package maintainer " % p_key) + msg.append("about this problem.") + + for x in msg: + f.add_flowing_data(x) + f.end_paragraph(1) + class depgraph: pkg_tree_map = { @@ -841,6 +873,9 @@ class depgraph: self.missingbins=[] self.digraph=portage.digraph() + # Tracks simple parent/child relationships (PDEPEND relationships are + # not reversed). + self._parent_child_digraph = digraph() self.orderedkeys=[] self.outdatedpackages=[] self.args_keys = [] @@ -913,6 +948,7 @@ class depgraph: if addme: existing_node = self.pkg_node_map[myroot].get(mykey) if existing_node: + self._parent_child_digraph.add(existing_node, myparent) if existing_node != myparent: # Refuse to make a node depend on itself so that the we don't # don't create a bogus circular dependency in self.altlist(). @@ -926,7 +962,9 @@ class depgraph: self.spinner.update() if mytype == "blocks": - if addme and "--buildpkgonly" not in self.myopts and myparent: + if myparent and \ + "--buildpkgonly" not in self.myopts and \ + "--nodeps" not in self.myopts: mybigkey[1] = myparent[1] self.blocker_parents.setdefault( tuple(mybigkey), set()).add(myparent) @@ -983,6 +1021,7 @@ class depgraph: mybigkey.append("merge") else: mybigkey.append("nomerge") + jbigkey = tuple(mybigkey) if addme: metadata = dict(izip(self._mydbapi_keys, @@ -998,6 +1037,7 @@ class depgraph: e_type, myroot, e_cpv, e_status = existing_node if mykey == e_cpv: # The existing node can be reused. + self._parent_child_digraph.add(existing_node, myparent) if rev_dep and myparent: ptype, proot, pkey, pstatus = myparent self.digraph.addnode(myparent, existing_node, @@ -1007,16 +1047,16 @@ class depgraph: priority=priority) return 1 else: - e_parents = self.digraph.parent_nodes(existing_node) + e_parents = self._parent_child_digraph.parent_nodes( + existing_node) myparents = [] if myparent: myparents.append(myparent) self._show_slot_collision_notice( - ((tuple(mybigkey), myparents), + ((jbigkey, myparents), (existing_node, e_parents))) return 0 - jbigkey = tuple(mybigkey) self._slot_node_map[myroot][slot_atom] = jbigkey self.pkg_node_map[myroot][mykey] = jbigkey self.useFlags[myroot][mykey] = myuse @@ -1029,6 +1069,11 @@ class depgraph: self.digraph.addnode(jbigkey, myparent, priority=priority) + # Do this even when addme is False (--onlydeps) so that the + # parent/child relationship is always known in case + # self._show_slot_collision_notice() needs to be called later. + self._parent_child_digraph.add(jbigkey, myparent) + """ This section determines whether we go deeper into dependencies or not. We want to go deeper on a few occasions: Installing package A, we need to make sure package A's deps are met. @@ -1328,9 +1373,7 @@ class depgraph: myroot=myroot, trees=self.trees) if not mycheck[0]: - sys.stderr.write("\n\n%s\n\n" % mycheck[1]) - sys.stderr.write("Parent: %s\n\n" % str(myparent)) - sys.stderr.write("%s\n\n" % depstring) + show_invalid_depstring_notice(myparent, depstring, mycheck[1]) return 0 mymerge = mycheck[1] @@ -1539,7 +1582,8 @@ class depgraph: correct merge order such that mutually blocking packages are never installed simultaneously.""" - if "--buildpkgonly" in self.myopts: + if "--buildpkgonly" in self.myopts or \ + "--nodeps" in self.myopts: return True #if "deep" in self.myparams: @@ -1586,9 +1630,9 @@ class depgraph: pkgsettings, myuse=myuse, trees=dep_check_trees, myroot=myroot) if not success: - print "\n\n" - print "Error occurred while processing",pkg - print str(atoms) + show_invalid_depstring_notice( + ("installed", myroot, pkg, "nomerge"), + depstr, atoms) return False blocker_atoms = [myatom for myatom in atoms \ if myatom.startswith("!")] @@ -2509,7 +2553,7 @@ class MergeTask(object): portage.config(clone=trees["/"]["vartree"].settings) def merge(self, mylist, favorites, mtimedb): - returnme=0 + failed_fetches = [] mymergelist=[] ldpath_mtimes = mtimedb["ldpath"] xterm_titles = "notitles" not in self.settings.features @@ -2657,7 +2701,7 @@ class MergeTask(object): print print "!!! Fetch for",y,"failed, continuing..." print - returnme=1 + failed_fetches.append(pkg_key) continue portage.doebuild_environment(y, "setup", myroot, @@ -2883,9 +2927,15 @@ class MergeTask(object): #by doing an exit this way, --fetchonly can continue to try to #fetch everything even if a particular download fails. if "--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts: - if returnme: - print "\n\n!!! Some fetch errors were encountered. Please see above for details.\n\n" - sys.exit(returnme) + if failed_fetches: + sys.stderr.write("\n\n!!! Some fetch errors were " + \ + "encountered. Please see above for details.\n\n") + for cpv in failed_fetches: + sys.stderr.write(" ") + sys.stderr.write(cpv) + sys.stderr.write("\n") + sys.stderr.write("\n") + sys.exit(1) else: sys.exit(0) return os.EX_OK @@ -4180,9 +4230,9 @@ def action_depclean(settings, trees, ldpath_mtimes, success, atoms = portage.dep_check(depstr, None, settings, myuse=usedef, trees=dep_check_trees, myroot=myroot) if not success: - print "\n\n" - print "Error occurred while processing",pkg - print str(atoms) + show_invalid_depstring_notice( + ("installed", myroot, pkg, "nomerge"), + depstr, atoms) return if "--debug" in myopts: diff --git a/cnf/make.conf b/cnf/make.conf index b0c7d147f..b1f87c0f7 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -333,6 +333,12 @@ PORTAGE_ELOG_CLASSES="warn error log" # syslog (sends all messages to syslog) # mail (send all messages to the mailserver defined # in $PORTAGE_ELOG_MAILURI) +# save_summary (like "save" but merges all messages +# in $PORT_LOGDIR/elog/summary.log, +# /var/log/portage/elog/summary.log if +# $PORT_LOGDIR is unset) +# mail_summary (like "mail" but sends all messages in +# a single mail when emerge exits) # To use elog you should enable at least one module #PORTAGE_ELOG_SYSTEM="save mail" diff --git a/pym/elog_modules/mod_mail_summary.py b/pym/elog_modules/mod_mail_summary.py new file mode 100644 index 000000000..53628b920 --- /dev/null +++ b/pym/elog_modules/mod_mail_summary.py @@ -0,0 +1,39 @@ +# portage.py -- core Portage functionality +# Copyright 1998-2004 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id: mod_mail.py 3484 2006-06-10 22:38:44Z genone $ + +import portage_mail, socket, os, time +from email.MIMEText import MIMEText as TextMessage + +_items = {} +def process(mysettings, cpv, logentries, fulltext): + header = ">>> Messages generated for package %s by process %d on %s:\n\n" % \ + (cpv, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))) + _items[cpv] = header + fulltext + +def finalize(mysettings): + if len(_items) == 0: + return + elif len(_items) == 1: + count = "one package" + else: + count = "multiple packages" + if mysettings.has_key("PORTAGE_ELOG_MAILURI"): + myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0] + else: + myrecipient = "root@localhost" + + myfrom = mysettings["PORTAGE_ELOG_MAILFROM"] + mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"] + mysubject = mysubject.replace("${PACKAGE}", count) + mysubject = mysubject.replace("${HOST}", socket.getfqdn()) + + mybody = "elog messages for the following packages generated by process %d on host %s:\n" % (count, os.getpid(), socket.getfqdn()) + for cpv in _items.keys(): + mybody += "- %s\n" % cpv + + mymessage = portage_mail.create_message(myfrom, myrecipient, mysubject, mybody, attachments=_items.values()) + portage_mail.send_mail(mysettings, mymessage) + + return diff --git a/pym/elog_modules/mod_save_summary.py b/pym/elog_modules/mod_save_summary.py new file mode 100644 index 000000000..dd4dad138 --- /dev/null +++ b/pym/elog_modules/mod_save_summary.py @@ -0,0 +1,23 @@ +import os, time +from portage_data import portage_uid, portage_gid + +def process(mysettings, cpv, logentries, fulltext): + if mysettings["PORT_LOGDIR"] != "": + elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog") + else: + elogdir = os.path.join(os.sep, "var", "log", "portage", "elog") + if not os.path.exists(elogdir): + os.makedirs(elogdir) + os.chown(elogdir, portage_uid, portage_gid) + os.chmod(elogdir, 02770) + + # TODO: Locking + elogfilename = elogdir+"/summary.log" + elogfile = open(elogfilename, "a") + elogfile.write(">>> Messages generated by process %d on %s:\n\n" % \ + (os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())))) + elogfile.write(fulltext) + elogfile.write("\n") + elogfile.close() + + return elogfilename diff --git a/pym/portage.py b/pym/portage.py index 5fa1c47f7..69845e12d 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -467,7 +467,7 @@ class digraph: print "(%s)" % self.nodes[node][0][child] - +_elog_atexit_handlers = [] def elog_process(cpv, mysettings): mylogfiles = listdir(mysettings["T"]+"/logging/") # shortcut for packages without any messages @@ -513,6 +513,9 @@ def elog_process(cpv, mysettings): logmodule = __import__("elog_modules.mod_"+s) m = getattr(logmodule, "mod_"+s) m.process(mysettings, cpv, mylogentries, fulllog) + if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: + _elog_atexit_handlers.append(m.finalize) + atexit_register(m.finalize, mysettings) except (ImportError, AttributeError), e: print "!!! Error while importing logging modules while loading \"mod_%s\":" % s print e @@ -2074,10 +2077,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro: if use_locks: - writemsg(red("!!! You are fetching to a read-only filesystem, you should turn locking off"), - noiselevel=-1) - writemsg("!!! This can be done by adding -distlocks to FEATURES in /etc/make.conf", - noiselevel=-1) + writemsg(red("!!! For fetching to a read-only filesystem, " + \ + "locking should be turned off.\n"), noiselevel=-1) + writemsg("!!! This can be done by adding -distlocks to " + \ + "FEATURES in /etc/make.conf\n", noiselevel=-1) # use_locks = 0 # local mirrors are always added @@ -2170,11 +2173,21 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", can_fetch=True + if listonly: + can_fetch = False + for var_name in ("FETCHCOMMAND", "RESUMECOMMAND"): if not mysettings.get(var_name, None): can_fetch = False - if not listonly: + if can_fetch and \ + not fetch_to_ro and \ + not os.access(mysettings["DISTDIR"], os.W_OK): + writemsg("!!! No write access to '%s'\n" % mysettings["DISTDIR"], + noiselevel=-1) + can_fetch = False + + if can_fetch: dirmode = 02070 filemode = 060 modemask = 02 @@ -2201,12 +2214,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", writemsg("!!! Directory Not Found: DISTDIR='%s'\n" % mysettings["DISTDIR"], noiselevel=-1) writemsg("!!! Fetching will fail!\n", noiselevel=-1) - if not os.access(mysettings["DISTDIR"]+"/",os.W_OK): - if not fetch_to_ro: - print "!!! No write access to %s" % mysettings["DISTDIR"]+"/" - can_fetch=False - else: - if use_locks and locks_in_subdir: + if can_fetch and use_locks and locks_in_subdir: distlocks_subdir = os.path.join(mysettings["DISTDIR"], locks_in_subdir) if not os.access(distlocks_subdir, os.W_OK): writemsg("!!! No write access to write to %s. Aborting.\n" % distlocks_subdir, @@ -2329,16 +2337,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", else: continue - # check if we can actually write to the directory/existing file. - if fetched!=2 and os.path.exists(mysettings["DISTDIR"]+"/"+myfile) != \ - os.access(mysettings["DISTDIR"]+"/"+myfile, os.W_OK) and not fetch_to_ro: - writemsg( red("***") + \ - " Lack write access to %s, failing fetch\n" % \ - os.path.join(mysettings["DISTDIR"], myfile), - noiselevel=-1) - fetched=0 - break - elif fetched!=2: + if fetched != 2: #we either need to resume or start the download #you can't use "continue" when you're inside a "try" block if fetched==1: @@ -4669,13 +4668,17 @@ class vardbapi(dbapi): os.rename(origpath, newpath) # We need to rename the ebuild now. - old_eb_path = newpath+"/"+mycpsplit[1] +"-"+mycpsplit[2] - new_eb_path = newpath+"/"+mycpsplit_new[1]+"-"+mycpsplit[2] - if mycpsplit[3] != "r0": - old_eb_path += "-"+mycpsplit[3] - new_eb_path += "-"+mycpsplit[3] - if os.path.exists(old_eb_path+".ebuild"): - os.rename(old_eb_path+".ebuild", new_eb_path+".ebuild") + old_pf = catsplit(mycpv)[1] + new_pf = catsplit(mynewcpv)[1] + if new_pf != old_pf: + try: + os.rename(os.path.join(newpath, old_pf + ".ebuild"), + os.path.join(newpath, new_pf + ".ebuild")) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") fixdbentries([mylist], newpath) @@ -5622,7 +5625,8 @@ class portdbapi(dbapi): myval=match_from_list(mydep,mylist) elif level=="match-visible": #dep match -- find all visible matches - myval=match_from_list(mydep,self.xmatch("list-visible",None,mydep=mydep,mykey=mykey)) + myval = match_from_list(mydep, + self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)) #get all visible packages, then get the matching ones elif level=="match-all": #match *all* visible *and* masked packages @@ -5636,6 +5640,8 @@ class portdbapi(dbapi): if self.aux_get(cpv, ["SLOT"])[0] == myslot] if self.frozen and (level not in ["match-list","bestmatch-list"]): self.xcache[level][mydep]=myval + if origdep and origdep != mydep: + self.xcache[level][origdep] = myval return myval def match(self,mydep,use_cache=1): @@ -5830,6 +5836,7 @@ class binarytree(packagetree): if mynewpkg != myoldpkg: mydata[mynewpkg+".ebuild"] = mydata[myoldpkg+".ebuild"] del mydata[myoldpkg+".ebuild"] + mydata["PF"] = mynewpkg + "\n" mytbz2.recompose_mem(xpak.xpak_mem(mydata)) self.dbapi.cpv_remove(mycpv) @@ -6343,7 +6350,12 @@ class dblink: mystuff = listdir(self.dbdir, EmptyOnError=1) for x in mystuff: if x.endswith(".ebuild"): - myebuildpath = os.path.join(self.dbdir, x) + myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild") + if x[:-7] != self.pkg: + # Clean up after vardbapi.move_ent() breakage in + # portage versions before 2.1.2 + os.rename(os.path.join(self.dbdir, x), myebuildpath) + write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") break self.settings.load_infodir(self.dbdir)