* Enable "echo" and "save_summary" elog modules by default
* Fix -* handling in package.keywords to work as intended (reset the accepted
keywords list), also see RELEASE-NOTES.
+* Experimental support for preserving old library files on package upgrades
+ based on FEATURES=preserve-libs, USE AT YOUR OWN RISK!!!
+* Make elog functionality available to python code
+* Add support for news items (GLEP 42)
portage-2.1.2
-------------
import portage
del os.environ["PORTAGE_LEGACY_GLOBALS"]
from portage import digraph, portdbapi
-from portage.const import NEWS_LIB_PATH
+from portage.const import NEWS_LIB_PATH, CACHE_PATH
import emerge.help
import portage.xpak, commands, errno, re, socket, time, types
print " "+green("*")+" Processed",icount,"info files."
+def display_news_notification(settings):
+ target_root = settings["ROOT"]
+ NEWS_PATH = os.path.join("metadata", "news")
+ UNREAD_PATH = os.path.join(target_root, NEWS_LIB_PATH, "news")
+ porttree = portdbapi(porttree_root=settings["PORTDIR"], mysettings=settings)
+ newsReaderDisplay = False
+
+ print
+ for repo in porttree.getRepositories():
+ unreadItems = checkUpdatedNewsItems(target_root, NEWS_PATH, UNREAD_PATH, repo)
+ if unreadItems:
+ print colorize("WARN", " * IMPORTANT:"),
+ print "%s news items need reading for repository '%s'." % (unreadItems, repo)
+ newsReaderDisplay = True
+
+ if newsReaderDisplay:
+ print colorize("WARN", " *"),
+ print "Use " + colorize("GOOD", "eselect news") + " to read news items."
+
+ print
def post_emerge(settings, mtimedb, retval):
"""
Misc. things to run at the end of a merge session.
Update Config Files
Update News Items
Commit mtimeDB
+ Display preserved libs warnings
Exit Emerge
@param settings: Configuration settings (typically portage.settings)
chk_updated_info_files(normalize_path(target_root + EPREFIX), infodirs, info_mtimes, retval)
chk_updated_cfg_files(normalize_path(target_root + EPREFIX), config_protect)
- NEWS_PATH = os.path.join( "metadata", "news" )
- UNREAD_PATH = os.path.join( target_root, NEWS_LIB_PATH, 'news')
- porttree = portdbapi( porttree_root = settings["PORTDIR"], mysettings = settings )
- newsReaderDisplay = False
+ display_news_notification(settings)
- for repo in porttree.getRepositories():
- unreadItems = checkUpdatedNewsItems(target_root, NEWS_PATH, UNREAD_PATH, repo)
- if unreadItems:
- print colorize("WARN", " * IMPORTANT:"),
- print "%s news items need reading for repository '%s'." % (unreadItems, repo)
- newsReaderDisplay = True
-
- if newsReaderDisplay:
- print colorize("WARN", " *"),
- print "Use " + colorize("GOOD", "eselect news") + " to read news items."
+ from portage.dbapi.vartree import PreservedLibsRegistry
+ plib_registry = PreservedLibsRegistry(os.path.join(target_root, CACHE_PATH, "preserved_libs_registry"))
+ if plib_registry.hasEntries():
+ print colorize("WARN", "!!!") + " existing preserved libs:"
+ plibdata = plib_registry.getPreservedLibs()
+ for cpv in plibdata.keys():
+ print colorize("WARN", ">>>") + " package: %s" % cpv
+ for f in plibdata[cpv]:
+ print colorize("WARN", " * ") + " - %s" % f
+ print "Use " + colorize("GOOD", "revdep-rebuild") + " to rebuild packages using these libraries"
+ print "and then remerge the packages listed above."
mtimedb.commit()
sys.exit(retval)
print red(" * ")+"configuration files."
print red(" * ")+"To update portage, run 'emerge portage'."
print
+
+ display_news_notification(settings)
def action_metadata(settings, portdb, myopts):
portage.writemsg_stdout("\n>>> Updating Portage cache: ")
# "update", "system", or just process files:
else:
validate_ebuild_environment(trees)
+ if "--pretend" not in myopts:
+ display_news_notification(settings)
action_build(settings, trees, mtimedb,
myopts, myaction, myfiles, spinner)
if "--pretend" not in myopts:
post_emerge(settings, mtimedb, 0)
+ else:
+ display_news_notification(settings)
if __name__ == "__main__":
retval = emerge_main()
mysplit = curdb["USE"].split()
for x in mysplit:
if x == "-*":
- myflags = use_expand_protected[:]
+ myflags = []
continue
if x[0] == "+":
# from shells and from binaries that belong to portage (the number of entry
# points is minimized). The "tee" binary is not among the allowed entry
# points, so it is spawned outside of the sesandbox domain and reads from a
- # pipe between two domains.
+ # pseudo-terminal that connects two domains.
logfile = keywords.get("logfile")
mypids = []
- pw = None
+ slave_fd = None
+ output_pid = None
+ input_pid = None
if logfile:
del keywords["logfile"]
fd_pipes = keywords.get("fd_pipes")
fd_pipes = {0:0, 1:1, 2:2}
elif 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
- pr, pw = os.pipe()
- mypids.extend(portage.process.spawn(('tee', '-i', '-a', logfile),
- returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]}))
- os.close(pr)
- fd_pipes[1] = pw
- fd_pipes[2] = pw
+ from pty import openpty
+ master_fd, slave_fd = openpty()
+ # Disable the ECHO attribute so the terminal behaves properly
+ # if the subprocess needs to read input from stdin.
+ import termios
+ term_attr = termios.tcgetattr(master_fd)
+ term_attr[3] &= ~termios.ECHO
+ termios.tcsetattr(master_fd, termios.TCSAFLUSH, term_attr)
+ # tee will always exit with an IO error, so ignore it's stderr.
+ null_file = open('/dev/null', 'w')
+ mypids.extend(portage.process.spawn(['tee', '-i', '-a', logfile],
+ returnpid=True, fd_pipes={0:master_fd, 1:fd_pipes[1],
+ 2:null_file.fileno()}))
+ output_pid = mypids[-1]
+ mypids.extend(portage.process.spawn(['cat'],
+ returnpid=True, fd_pipes={0:fd_pipes[0], 1:master_fd,
+ 2:null_file.fileno()}))
+ input_pid = mypids[-1]
+ os.close(master_fd)
+ null_file.close()
+ fd_pipes[0] = slave_fd
+ fd_pipes[1] = slave_fd
+ fd_pipes[2] = slave_fd
keywords["fd_pipes"] = fd_pipes
features = mysettings.features
- # XXX: Negative RESTRICT word
- droppriv=(droppriv and ("userpriv" in features) and not \
- (("nouserpriv" in mysettings["RESTRICT"].split()) or \
- ("userpriv" in mysettings["RESTRICT"].split())))
-
+ restrict = mysettings.get("RESTRICT", "").split()
+ droppriv=(droppriv and "userpriv" in features and not \
+ ("nouserpriv" in restrict or "userpriv" in restrict))
if droppriv and not uid and portage_gid and portage_uid:
- keywords.update({"uid":portage_uid,"gid":portage_gid,"groups":userpriv_groups,"umask":002})
-
+ keywords.update({"uid":portage_uid,"gid":portage_gid,
+ "groups":userpriv_groups,"umask":002})
if not free:
free=((droppriv and "usersandbox" not in features) or \
- (not droppriv and "sandbox" not in features and "usersandbox" not in features))
+ (not droppriv and "sandbox" not in features and \
+ "usersandbox" not in features))
if free:
keywords["opt_name"] += " bash"
if sesandbox:
con = selinux.getcontext()
- con = con.replace(mysettings["PORTAGE_T"], mysettings["PORTAGE_SANDBOX_T"])
+ con = con.replace(mysettings["PORTAGE_T"],
+ mysettings["PORTAGE_SANDBOX_T"])
selinux.setexec(con)
returnpid = keywords.get("returnpid")
try:
mypids.extend(spawn_func(mystring, env=env, **keywords))
finally:
- if pw:
- os.close(pw)
+ if slave_fd:
+ os.close(slave_fd)
if sesandbox:
selinux.setexec(None)
if returnpid:
return mypids
- while mypids:
- pid = mypids.pop(0)
- retval = os.waitpid(pid, 0)[1]
- portage.process.spawned_pids.remove(pid)
- if retval != os.EX_OK:
- for pid in mypids:
- if os.waitpid(pid, os.WNOHANG) == (0,0):
- import signal
- os.kill(pid, signal.SIGTERM)
- os.waitpid(pid, 0)
- portage.process.spawned_pids.remove(pid)
- if retval & 0xff:
- return (retval & 0xff) << 8
- return retval >> 8
- return os.EX_OK
+ if output_pid:
+ # tee will exit when the other end of the pseudo-terminal is closed.
+ os.waitpid(output_pid, 0)
+ portage.process.spawned_pids.remove(output_pid)
+ if input_pid:
+ # cat is blocking on stdin, so it must be killed.
+ import signal
+ try:
+ os.kill(input_pid, signal.SIGTERM)
+ except OSError:
+ pass # it died by itself
+ os.waitpid(input_pid, 0)
+ portage.process.spawned_pids.remove(input_pid)
+ pid = mypids[-1]
+ retval = os.waitpid(pid, 0)[1]
+ portage.process.spawned_pids.remove(pid)
+ if retval != os.EX_OK:
+ if retval & 0xff:
+ return (retval & 0xff) << 8
+ return retval >> 8
+ return retval
def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
"fetch files. Will use digest file if available."
import os, sys, stat, cPickle, errno, commands, copy, time
from itertools import izip
+
+class PreservedLibsRegistry(object):
+ """ This class handles the tracking of preserved library objects """
+ def __init__(self, filename, autocommit=True):
+ """ @param filename: absolute path for saving the preserved libs records
+ @type filename: String
+ @param autocommit: determines if the file is written after every update
+ @type autocommit: Boolean
+ """
+ self._filename = filename
+ self._autocommit = autocommit
+ self.load()
+
+ def load(self):
+ """ Reload the registry data from file """
+ try:
+ self._data = cPickle.load(open(self._filename, "r"))
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ self._data = {}
+ else:
+ raise e
+
+ def store(self):
+ """ Store the registry data to file. No need to call this if autocommit
+ was enabled.
+ """
+ cPickle.dump(self._data, open(self._filename, "w"))
+
+ def register(self, cpv, slot, counter, paths):
+ """ Register new objects in the registry. If there is a record with the
+ same packagename (internally derived from cpv) and slot it is
+ overwritten with the new data.
+ @param cpv: package instance that owns the objects
+ @type cpv: CPV (as String)
+ @param slot: the value of SLOT of the given package instance
+ @type slot: String
+ @param counter: vdb counter value for the package instace
+ @type counter: Integer
+ @param paths: absolute paths of objects that got preserved during an update
+ @type paths: List
+ """
+ cp = "/".join(catpkgsplit(cpv)[:2])
+ cps = cp+":"+slot
+ if len(paths) == 0 and self._data.has_key(cps) \
+ and self._data[cps][0] == cpv and int(self._data[cps][1]) == int(counter):
+ del self._data[cps]
+ elif len(paths) > 0:
+ self._data[cps] = (cpv, counter, paths)
+ if self._autocommit:
+ self.store()
+
+ def unregister(self, cpv, slot, counter):
+ """ Remove a previous registration of preserved objects for the given package.
+ @param cpv: package instance whose records should be removed
+ @type cpv: CPV (as String)
+ @param slot: the value of SLOT of the given package instance
+ @type slot: String
+ """
+ self.register(cpv, slot, counter, [])
+
+ def pruneNonExisting(self):
+ """ Remove all records for objects that no longer exist on the filesystem. """
+ for cps in self._data.keys():
+ cpv, counter, paths = self._data[cps]
+ paths = [f for f in paths if os.path.exists(f)]
+ if len(paths) > 0:
+ self._data[cps] = (cpv, counter, paths)
+ else:
+ del self._data[cps]
+ if self._autocommit:
+ self.store()
+
+ def hasEntries(self):
+ """ Check if this registry contains any records. """
+ return (len(self._data.keys()) > 0)
+
+ def getPreservedLibs(self):
+ """ Return a mapping of packages->preserved objects.
+ @returns mapping of package instances to preserved objects
+ @rtype Dict cpv->list-of-paths
+ """
+ rValue = {}
+ for cps in self._data.keys():
+ rValue[self._data[cps][0]] = self._data[cps][2]
+ return rValue
+
+class LibraryPackageMap(object):
+ """ This class provides a library->consumer mapping generated from VDB data """
+ def __init__(self, filename, vardbapi):
+ self._filename = filename
+ self._dbapi = vardbapi
+
+ def get(self):
+ """ Read the global library->consumer map for the given vdb instance.
+ @returns mapping of library objects (just basenames) to consumers (absolute paths)
+ @rtype filename->list-of-paths
+ """
+ if not os.path.exists(self._filename):
+ self.update()
+ rValue = {}
+ for l in open(self._filename, "r").read().split("\n"):
+ mysplit = l.split()
+ if len(mysplit) > 1:
+ rValue[mysplit[0]] = mysplit[1].split(",")
+ return rValue
+
+ def update(self):
+ """ Update the global library->consumer map for the given vdb instance. """
+ obj_dict = {}
+ for cpv in self._dbapi.cpv_all():
+ needed_list = self._dbapi.aux_get(cpv, ["NEEDED"])[0]
+ for l in needed_list.split("\n"):
+ mysplit = l.split()
+ if len(mysplit) < 2:
+ continue
+ libs = mysplit[1].split(",")
+ for lib in libs:
+ if not obj_dict.has_key(lib):
+ obj_dict[lib] = [mysplit[0]]
+ else:
+ obj_dict[lib].append(mysplit[0])
+ mapfile = open(self._filename, "w")
+ for lib in obj_dict.keys():
+ mapfile.write(lib+" "+",".join(obj_dict[lib])+"\n")
+ mapfile.close()
+
class vardbapi(dbapi):
def __init__(self, root, categories=None, settings=None, vartree=None):
self.root = root[:]
self._aux_cache_filename = os.path.join(self.root,
CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle")
+ self.libmap = LibraryPackageMap(os.path.join(self.root, CACHE_PATH, "library_consumers"), self)
+ self.plib_registry = PreservedLibsRegistry(os.path.join(self.root, CACHE_PATH, "preserved_libs_registry"))
+
def getpath(self, mykey, filename=None):
rValue = os.path.join(self.root, VDB_PATH, mykey)
if filename != None:
write_atomic(cpath, str(counter))
return counter
- def get_library_map(self):
- """ Read the global library->consumer map for this vdb instance """
- mapfilename = os.path.join(self.root, CACHE_PATH, "library_consumers")
- if not os.path.exists(mapfilename):
- self.update_library_map()
- rValue = {}
- for l in open(mapfilename, "r").read().split("\n"):
- mysplit = l.split()
- if len(mysplit) > 1:
- rValue[mysplit[0]] = mysplit[1].split(",")
- return rValue
-
- def update_library_map(self):
- """ Update the global library->consumer map for this vdb instance. """
- mapfilename = os.path.join(self.root, CACHE_PATH, "library_consumers")
- obj_dict = {}
- for cpv in self.cpv_all():
- needed_list = self.aux_get(cpv, ["NEEDED"])[0]
- for l in needed_list.split("\n"):
- mysplit = l.split()
- if len(mysplit) < 2:
- continue
- libs = mysplit[1].split(",")
- for lib in libs:
- if not obj_dict.has_key(lib):
- obj_dict[lib] = [mysplit[0]]
- else:
- obj_dict[lib].append(mysplit[0])
- mapfile = open(mapfilename, "w")
- for lib in obj_dict.keys():
- mapfile.write(lib+" "+",".join(obj_dict[lib])+"\n")
- mapfile.close()
-
class vartree(object):
"this tree will scan a var/db/pkg database located at root (passed to init)"
def __init__(self, root="/", virtual=None, clone=None, categories=None,
return retval
self._unmerge_pkgfiles(pkgfiles)
+
+ # Remove the registration of preserved libs for this pkg instance
+ self.vartree.dbapi.plib_registry.unregister(self.mycpv, self.settings["SLOT"], self.settings["COUNTER"])
if myebuildpath:
retval = doebuild(myebuildpath, "postrm", self.myroot,
vartree=self.vartree)
# regenerate reverse NEEDED map
- self.vartree.dbapi.update_library_map()
+ self.vartree.dbapi.libmap.update()
finally:
if builddir_lock:
return True
- def _preserve_libs(self, srcroot, destroot, mycontents):
+ def _preserve_libs(self, srcroot, destroot, mycontents, counter):
# read global reverse NEEDED map
- libmap = self.vartree.dbapi.get_library_map()
+ libmap = self.vartree.dbapi.libmap.get()
# get list of libraries from old package instance
old_contents = self._installed_instance.getcontents().keys()
preserve_paths.append(linktarget)
else:
shutil.copy2(os.path.join(destroot, x), os.path.join(srcroot, x.lstrip(os.sep)))
+
+ # keep track of the libs we preserved
+ self.vartree.dbapi.plib_registry.register(self.mycpv, self.settings["SLOT"], counter, preserve_paths)
+
del preserve_paths
def _collision_protect(self, srcroot, destroot, otherversions, mycontents, mysymlinks):
catsplit(slot_matches[0])[1], destroot, self.settings,
vartree=self.vartree)
+ # get current counter value (counter_tick also takes care of incrementing it)
+ # XXX Need to make this destroot, but it needs to be initialized first. XXX
+ # XXX bis: leads to some invalidentry() call through cp_all().
+ # Note: The counter is generated here but written later because preserve_libs
+ # needs the counter value but has to be before dbtmpdir is made (which
+ # has to be before the counter is written) - genone
+ counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv)
+
myfilelist = None
mylinklist = None
# Preserve old libs if they are still in use
if slot_matches and "preserve-libs" in self.settings.features:
myfilelist = listdir(srcroot, recursive=1, filesonly=1, followSymlinks=False)
- mylinklist = filter(os.path.islink, listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False))
- self._preserve_libs(srcroot, destroot, myfilelist+mylinklist)
+ mylinklist = filter(os.path.islink, [os.path.join(srcroot, x) for x in listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False)])
+ mylinklist = [x[len(srcroot):] for x in mylinklist]
+ self._preserve_libs(srcroot, destroot, myfilelist+mylinklist, counter)
# check for package collisions
if "collision-protect" in self.settings.features:
if myfilelist == None:
myfilelist = listdir(srcroot, recursive=1, filesonly=1, followSymlinks=False)
if mylinklist == None:
- mylinklist = filter(os.path.islink, listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False))
+ mylinklist = filter(os.path.islink, [os.path.join(srcroot, x) for x in listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False)])
+ mylinklist = [x[len(srcroot):] for x in mylinklist]
self._collision_protect(srcroot, destroot, otherversions, myfilelist+mylinklist, mylinklist)
if os.stat(srcroot).st_dev == os.stat(destroot).st_dev:
for x in listdir(inforoot):
self.copyfile(inforoot+"/"+x)
- # get current counter value (counter_tick also takes care of incrementing it)
- # XXX Need to make this destroot, but it needs to be initialized first. XXX
- # XXX bis: leads to some invalidentry() call through cp_all().
- counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv)
# write local package counter for recording
lcfile = open(os.path.join(self.dbtmpdir, "COUNTER"),"w")
lcfile.write(str(counter))
del conf_mem_file
# regenerate reverse NEEDED map
- self.vartree.dbapi.update_library_map()
+ self.vartree.dbapi.libmap.update()
#do postinst script
a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0,
from portage.process import atexit_register
from portage.util import writemsg
-from portage import listdir
+from portage.elog.messages import collect_ebuild_messages, collect_messages
+from portage.elog.filtering import filter_loglevels
import os
+def _merge_logentries(a, b):
+ rValue = {}
+ phases = set(a.keys()+b.keys())
+ for p in phases:
+ rValue[p] = []
+ if a.has_key(p):
+ for x in a[p]:
+ rValue[p].append(x)
+ if b.has_key(p):
+ for x in b[p]:
+ rValue[p].append(x)
+ return rValue
+
+def _combine_logentries(logentries):
+ # generate a single string with all log messages
+ rValue = ""
+ for phase in EBUILD_PHASES:
+ if not phase in logentries:
+ continue
+ for msgtype, msgcontent in logentries[phase]:
+ rValue += "%s: %s\n" % (msgtype, phase)
+ for line in msgcontent:
+ rValue += line
+ rValue += "\n"
+ return rValue
+
_elog_atexit_handlers = []
def elog_process(cpv, mysettings):
- mylogfiles = listdir(mysettings["T"]+"/logging/")
- # shortcut for packages without any messages
- if len(mylogfiles) == 0:
- return
- # exploit listdir() file order so we process log entries in chronological order
- mylogfiles.reverse()
- all_logentries = {}
- for f in mylogfiles:
- msgfunction, msgtype = f.split(".")
- if msgfunction not in EBUILD_PHASES:
- writemsg("!!! can't process invalid log file: %s\n" % f,
- noiselevel=-1)
- continue
- if not msgfunction in all_logentries:
- all_logentries[msgfunction] = []
- msgcontent = open(mysettings["T"]+"/logging/"+f, "r").readlines()
- all_logentries[msgfunction].append((msgtype, msgcontent))
+ ebuild_logentries = collect_ebuild_messages(os.path.join(mysettings["T"], "logging"))
+ all_logentries = collect_messages()
+ if all_logentries.has_key(cpv):
+ all_logentries[cpv] = _merge_logentries(ebuild_logentries, all_logentries[cpv])
+ else:
+ all_logentries[cpv] = ebuild_logentries
- def filter_loglevels(logentries, loglevels):
- # remove unwanted entries from all logentries
- rValue = {}
- loglevels = map(str.upper, loglevels)
- for phase in logentries.keys():
- for msgtype, msgcontent in logentries[phase]:
- if msgtype.upper() in loglevels or "*" in loglevels:
- if not rValue.has_key(phase):
- rValue[phase] = []
- rValue[phase].append((msgtype, msgcontent))
- return rValue
-
my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split())
- default_logentries = filter_loglevels(all_logentries, my_elog_classes)
- # in case the filters matched all messages and no module overrides exist
- if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")):
- return
- def combine_logentries(logentries):
- # generate a single string with all log messages
- rValue = ""
- for phase in EBUILD_PHASES:
- if not phase in logentries:
- continue
- for msgtype, msgcontent in logentries[phase]:
- rValue += "%s: %s\n" % (msgtype, phase)
- for line in msgcontent:
- rValue += line
- rValue += "\n"
- return rValue
-
- default_fulllog = combine_logentries(default_logentries)
+ for key in all_logentries.keys():
+ default_logentries = filter_loglevels(all_logentries[key], my_elog_classes)
- # pass the processing to the individual modules
- logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split()
- for s in logsystems:
- # allow per module overrides of PORTAGE_ELOG_CLASSES
- if ":" in s:
- s, levels = s.split(":", 1)
- levels = levels.split(",")
- mod_logentries = filter_loglevels(all_logentries, levels)
- mod_fulllog = combine_logentries(mod_logentries)
- else:
- mod_logentries = default_logentries
- mod_fulllog = default_fulllog
- if len(mod_logentries) == 0:
- continue
- # - is nicer than _ for module names, so allow people to use it.
- s = s.replace("-", "_")
- try:
- # FIXME: ugly ad.hoc import code
- # TODO: implement a common portage module loader
- name = "portage.elog.mod_" + s
- m = __import__(name)
- for comp in name.split(".")[1:]:
- m = getattr(m, comp)
- def timeout_handler(signum, frame):
- raise PortageException("Timeout in elog_process for system '%s'" % s)
- import signal
- signal.signal(signal.SIGALRM, timeout_handler)
- # Timeout after one minute (in case something like the mail
- # module gets hung).
- signal.alarm(60)
+ # in case the filters matched all messages and no module overrides exist
+ if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")):
+ return
+
+ default_fulllog = _combine_logentries(default_logentries)
+
+ # pass the processing to the individual modules
+ logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split()
+ for s in logsystems:
+ # allow per module overrides of PORTAGE_ELOG_CLASSES
+ if ":" in s:
+ s, levels = s.split(":", 1)
+ levels = levels.split(",")
+ mod_logentries = filter_loglevels(all_logentries[key], levels)
+ mod_fulllog = combine_logentries(mod_logentries)
+ else:
+ mod_logentries = default_logentries
+ mod_fulllog = default_fulllog
+ if len(mod_logentries) == 0:
+ continue
+ # - is nicer than _ for module names, so allow people to use it.
+ s = s.replace("-", "_")
try:
- m.process(mysettings, cpv, mod_logentries, mod_fulllog)
- finally:
- signal.alarm(0)
- if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers:
- _elog_atexit_handlers.append(m.finalize)
- atexit_register(m.finalize, mysettings)
- except (ImportError, AttributeError), e:
- writemsg("!!! Error while importing logging modules " + \
- "while loading \"mod_%s\":\n" % str(s))
- writemsg("%s\n" % str(e), noiselevel=-1)
- except PortageException, e:
- writemsg("%s\n" % str(e), noiselevel=-1)
+ # FIXME: ugly ad.hoc import code
+ # TODO: implement a common portage module loader
+ name = "portage.elog.mod_" + s
+ m = __import__(name)
+ for comp in name.split(".")[1:]:
+ m = getattr(m, comp)
+ def timeout_handler(signum, frame):
+ raise PortageException("Timeout in elog_process for system '%s'" % s)
+ import signal
+ signal.signal(signal.SIGALRM, timeout_handler)
+ # Timeout after one minute (in case something like the mail
+ # module gets hung).
+ signal.alarm(60)
+ try:
+ m.process(mysettings, str(key), mod_logentries, mod_fulllog)
+ finally:
+ signal.alarm(0)
+ if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers:
+ _elog_atexit_handlers.append(m.finalize)
+ atexit_register(m.finalize, mysettings)
+ except (ImportError, AttributeError), e:
+ writemsg("!!! Error while importing logging modules " + \
+ "while loading \"mod_%s\":\n" % str(s))
+ writemsg("%s\n" % str(e), noiselevel=-1)
+ except PortageException, e:
+ writemsg("%s\n" % str(e), noiselevel=-1)
- # clean logfiles to avoid repetitions
- for f in mylogfiles:
- try:
- os.unlink(os.path.join(mysettings["T"], "logging", f))
- except OSError:
- pass
--- /dev/null
+# elog/messages.py - elog core functions
+# Copyright 2006-2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id: __init__.py 6458 2007-04-30 02:31:30Z genone $
+
+def filter_loglevels(logentries, loglevels):
+ # remove unwanted entries from all logentries
+ rValue = {}
+ loglevels = map(str.upper, loglevels)
+ for phase in logentries.keys():
+ for msgtype, msgcontent in logentries[phase]:
+ if msgtype.upper() in loglevels or "*" in loglevels:
+ if not rValue.has_key(phase):
+ rValue[phase] = []
+ rValue[phase].append((msgtype, msgcontent))
+ return rValue
+
--- /dev/null
+# elog/messages.py - elog core functions
+# Copyright 2006-2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id: __init__.py 6458 2007-04-30 02:31:30Z genone $
+
+from portage.output import colorize
+from portage.const import EBUILD_PHASES
+from portage.util import writemsg
+
+from portage import listdir
+
+import os
+
+def collect_ebuild_messages(path):
+ """ Collect elog messages generated by the bash logging function stored
+ at 'path'.
+ """
+ mylogfiles = listdir(path)
+ # shortcut for packages without any messages
+ if len(mylogfiles) == 0:
+ return {}
+ # exploit listdir() file order so we process log entries in chronological order
+ mylogfiles.reverse()
+ logentries = {}
+ for f in mylogfiles:
+ msgfunction, msgtype = f.split(".")
+ if msgfunction not in EBUILD_PHASES:
+ writemsg("!!! can't process invalid log file: %s\n" % f,
+ noiselevel=-1)
+ continue
+ if not msgfunction in logentries:
+ logentries[msgfunction] = []
+ msgcontent = open(os.path.join(path, f), "r").readlines()
+ logentries[msgfunction].append((msgtype, msgcontent))
+ # clean logfiles to avoid repetitions
+ for f in mylogfiles:
+ try:
+ os.unlink(os.path.join(path, f))
+ except OSError:
+ pass
+ return logentries
+
+_msgbuffer = {}
+def _elog_base(level, msg, phase="other", key=None, color=None):
+ """ Backend for the other messaging functions, should not be called
+ directly.
+ """
+ if color == None:
+ color = "GOOD"
+ print colorize(color, " * ")+msg
+ if not _msgbuffer.has_key(key):
+ _msgbuffer[key] = {}
+ if not _msgbuffer[key].has_key(phase):
+ _msgbuffer[key][phase] = []
+ _msgbuffer[key][phase].append((level, msg))
+
+ #raise NotImplementedError()
+
+def collect_messages():
+ rValue = _msgbuffer
+ _reset_buffer()
+ return rValue
+
+def _reset_buffer():
+ """ Reset the internal message buffer when it has been processed,
+ should not be called directly.
+ """
+ _msgbuffer = {}
+
+# creating and exporting the actual messaging functions
+_functions = { "einfo": ("INFO", "GOOD"),
+ "elog": ("LOG", "GOOD"),
+ "ewarn": ("WARN", "WARN"),
+ "eqawarn": ("QA", "WARN"),
+ "eerror": ("ERROR", "ERROR"),
+}
+
+def _make_msgfunction(level, color):
+ def _elog(msg, phase="other", key=None):
+ """ Display and log a message assigned to the given key/cpv
+ (or unassigned if no key is given).
+ """
+ _elog_base(level, msg, phase=phase, key=key, color=color)
+ return _elog
+
+import sys
+for f in _functions.keys():
+ setattr(sys.modules[__name__], f, _make_msgfunction(_functions[f][0], _functions[f][1]))
+del f, _functions
import portage.elog_modules.mod_save, portage.process, portage.exception
-def process(mysettings, cpv, logentries, fulltext):
- elogfilename = portage.elog_modules.mod_save.process(mysettings, cpv, logentries, fulltext)
+def process(mysettings, key, logentries, fulltext):
+ elogfilename = portage.elog_modules.mod_save.process(mysettings, key, logentries, fulltext)
if (not "PORTAGE_ELOG_COMMAND" in mysettings.keys()) \
or len(mysettings["PORTAGE_ELOG_COMMAND"]) == 0:
else:
mylogcmd = mysettings["PORTAGE_ELOG_COMMAND"]
mylogcmd = mylogcmd.replace("${LOGFILE}", elogfilename)
- mylogcmd = mylogcmd.replace("${PACKAGE}", cpv)
+ mylogcmd = mylogcmd.replace("${PACKAGE}", key)
retval = portage.process.spawn_bash(mylogcmd)
if retval != 0:
raise portage.exception.PortageException("!!! PORTAGE_ELOG_COMMAND failed with exitcode %d" % retval)
from portage.const import EBUILD_PHASES
_items = {}
-def process(mysettings, cpv, logentries, fulltext):
- _items[cpv] = logentries
+def process(mysettings, key, logentries, fulltext):
+ _items[key] = logentries
def finalize(mysettings):
printer = EOutput()
- for cpv in _items.keys():
+ for key in _items.keys():
print
- printer.einfo("Messages for package %s:" % cpv)
+ printer.einfo("Messages for package %s:" % key)
print
for phase in EBUILD_PHASES:
- if not phase in _items[cpv]:
+ if not phase in _items[key]:
continue
- for msgtype, msgcontent in _items[cpv][phase]:
+ for msgtype, msgcontent in _items[key][phase]:
fmap = {"INFO": printer.einfo,
"WARN": printer.ewarn,
"ERROR": printer.eerror,
import portage.mail, socket
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
if mysettings.has_key("PORTAGE_ELOG_MAILURI"):
myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0]
else:
myfrom = mysettings["PORTAGE_ELOG_MAILFROM"]
mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"]
- mysubject = mysubject.replace("${PACKAGE}", cpv)
+ mysubject = mysubject.replace("${PACKAGE}", key)
mysubject = mysubject.replace("${HOST}", socket.getfqdn())
mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, fulltext)
from email.MIMEText import MIMEText as TextMessage
_items = {}
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
header = ">>> Messages generated for package %s by process %d on %s:\n\n" % \
- (cpv, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())))
- _items[cpv] = header + fulltext
+ (key, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())))
+ _items[key] = header + fulltext
def finalize(mysettings):
if len(_items) == 0:
mybody = "elog messages for the following packages generated by " + \
"process %d on host %s:\n" % (os.getpid(), socket.getfqdn())
- for cpv in _items.keys():
- mybody += "- %s\n" % cpv
+ for key in _items.keys():
+ mybody += "- %s\n" % key
mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, mybody, attachments=_items.values())
portage.mail.send_mail(mysettings, mymessage)
import os, time
from portage.data import portage_uid, portage_gid
-def process(mysettings, cpv, logentries, fulltext):
- cpv_path = cpv.replace("/", ":")
+def process(mysettings, key, logentries, fulltext):
+ path = key.replace("/", ":")
if mysettings["PORT_LOGDIR"] != "":
elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog")
os.chown(elogdir, portage_uid, portage_gid)
os.chmod(elogdir, 02770)
- elogfilename = elogdir+"/"+cpv_path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log"
+ elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log"
elogfile = open(elogfilename, "w")
elogfile.write(fulltext)
elogfile.close()
from portage.data import portage_uid, portage_gid
from portage.const import EPREFIX
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
if mysettings["PORT_LOGDIR"] != "":
elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog")
else:
elogfilename = elogdir+"/summary.log"
elogfile = open(elogfilename, "a")
elogfile.write(">>> Messages generated by process %d on %s for package %s:\n\n" % \
- (os.getpid(), time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), cpv))
+ (os.getpid(), time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), key))
elogfile.write(fulltext)
elogfile.write("\n")
elogfile.close()
import syslog
from portage.const import EBUILD_PHASES
-def process(mysettings, cpv, logentries, fulltext):
+def process(mysettings, key, logentries, fulltext):
syslog.openlog("portage", syslog.LOG_ERR | syslog.LOG_WARNING | syslog.LOG_INFO | syslog.LOG_NOTICE, syslog.LOG_LOCAL5)
for phase in EBUILD_PHASES:
if not phase in logentries:
"LOG": syslog.LOG_NOTICE,
"QA": syslog.LOG_WARNING}
msgtext = "".join(msgcontent)
- syslog.syslog(pri[msgtype], "%s: %s: %s" % (cpv, phase, msgtext))
+ syslog.syslog(pri[msgtype], "%s: %s: %s" % (key, phase, msgtext))
syslog.closelog()
TIMESTAMP_FILE = "news-timestamp"
- def __init__( self, root, NEWS_PATH, UNREAD_PATH, LANGUAGE_ID='en' ):
+ def __init__(self, root, NEWS_PATH, UNREAD_PATH, LANGUAGE_ID='en'):
self.NEWS_PATH = NEWS_PATH
self.UNREAD_PATH = UNREAD_PATH
- self.TIMESTAMP_PATH = os.path.join( root, NEWS_LIB_PATH, NewsManager.TIMESTAMP_FILE )
+ self.TIMESTAMP_PATH = os.path.join(root, NEWS_LIB_PATH, NewsManager.TIMESTAMP_FILE)
self.target_root = root
self.LANGUAGE_ID = LANGUAGE_ID
- self.config = config( config_root = os.environ.get("PORTAGE_CONFIGROOT", "/"),
+ self.config = config(config_root = os.environ.get("PORTAGE_CONFIGROOT", "/"),
target_root = root, config_incrementals = INCREMENTALS)
- self.vdb = vardbapi( settings = self.config, root = root,
- vartree = vartree( root = root, settings = self.config ) )
- self.portdb = portdbapi( porttree_root = self.config["PORTDIR"], mysettings = self.config )
+ self.vdb = vardbapi(settings = self.config, root = root,
+ vartree = vartree(root = root, settings = self.config))
+ self.portdb = portdbapi(porttree_root = self.config["PORTDIR"], mysettings = self.config)
# Ensure that the unread path exists and is writable.
dirmode = 02070
modemask = 02
ensure_dirs(self.UNREAD_PATH, mode=dirmode, mask=modemask, gid=portage_gid)
- def updateItems( self, repoid ):
+ def updateItems(self, repoid):
"""
Figure out which news items from NEWS_PATH are both unread and relevant to
the user (according to the GLEP 42 standards of relevancy). Then add these
timestamp_file = self.TIMESTAMP_PATH + repoid
if os.path.exists(timestamp_file):
# Make sure the timestamp has correct permissions.
- apply_permissions( filename=timestamp_file,
- uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=664 )
+ apply_permissions(filename=timestamp_file,
+ uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=0664)
timestamp = os.stat(timestamp_file).st_mtime
else:
timestamp = 0
- path = os.path.join( self.portdb.getRepositoryPath( repoid ), self.NEWS_PATH )
+ path = os.path.join(self.portdb.getRepositoryPath(repoid), self.NEWS_PATH)
newsdir_lock = None
try:
- newsdir_lock = lockdir( self.portdb.getRepositoryPath(repoid) )
+ newsdir_lock = lockdir(self.portdb.getRepositoryPath(repoid))
# Skip reading news for repoid if the news dir does not exist. Requested by
# NightMorph :)
- if not os.path.exists( path ):
+ if not os.path.exists(path):
return None
- news = os.listdir( path )
+ news = os.listdir(path)
updates = []
- for item in news:
+ for itemid in news:
try:
- file = os.path.join( path, item, item + "." + self.LANGUAGE_ID + ".txt")
- tmp = NewsItem( file , timestamp )
- except TypeError:
+ filename = os.path.join(path, itemid, itemid + "." + self.LANGUAGE_ID + ".txt")
+ item = NewsItem(filename, itemid, timestamp)
+ except (TypeError, ValueError), e:
continue
-
- if tmp.isRelevant( profile=os.readlink(PROFILE_PATH), config=config, vardb=self.vdb):
- updates.append( tmp )
+ if item.isRelevant(profile=os.readlink(PROFILE_PATH), config=config, vardb=self.vdb):
+ updates.append(item)
finally:
if newsdir_lock:
unlockdir(newsdir_lock)
del path
- path = os.path.join( self.UNREAD_PATH, "news-" + repoid + ".unread" )
+ path = os.path.join(self.UNREAD_PATH, "news-"+repoid+".unread")
try:
- unread_lock = lockfile( path )
- if not os.path.exists( path ):
+ unread_lock = lockfile(path)
+ if not os.path.exists(path):
#create the file if it does not exist
- open( path, "w" )
+ open(path, "w")
# Ensure correct perms on the unread file.
apply_permissions( filename=path,
- uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=664 )
+ uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=0664)
# Make sure we have the correct permissions when created
- unread_file = open( path, "a" )
+ unread_file = open(path, "a")
for item in updates:
- unread_file.write( item.path + "\n" )
+ unread_file.write(item.name + "\n")
unread_file.close()
finally:
unlockfile(unread_lock)
f = open(timestamp_file, "w")
f.close()
- def getUnreadItems( self, repoid, update=False ):
+ def getUnreadItems(self, repoid, update=False):
"""
Determine if there are unread relevant items in news.repoid.unread.
If there are unread items return their number.
"""
if update:
- self.updateItems( repoid )
+ self.updateItems(repoid)
- unreadfile = os.path.join( self.UNREAD_PATH, "news-"+ repoid +".unread" )
+ unreadfile = os.path.join(self.UNREAD_PATH, "news-"+repoid+".unread")
try:
try:
unread_lock = lockfile(unreadfile)
# Set correct permissions on the news-repoid.unread file
- apply_permissions( filename=unreadfile,
- uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=0664 )
+ apply_permissions(filename=unreadfile,
+ uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=0664)
- if os.path.exists( unreadfile ):
- unread = open( unreadfile ).readlines()
+ if os.path.exists(unreadfile):
+ unread = open(unreadfile).readlines()
if len(unread):
return len(unread)
except FileNotFound:
"""
- def __init__( self, path, cache_mtime = 0 ):
+ def __init__(self, path, name, cache_mtime = 0):
"""
For a given news item we only want if it path is a file and it's
mtime is newer than the cache'd timestamp.
"""
- if not os.path.isfile( path ):
- raise TypeError
- if not os.stat( path ).st_mtime > cache_mtime:
- raise TypeError
+ if not os.path.isfile(path):
+ raise TypeError("%s is no regular file" % path)
+ if not os.stat(path).st_mtime > cache_mtime:
+ raise ValueError("%s / %s timestamp mismatch" % (str(os.stat(path).st_mtime), str(cache_mtime)))
self.path = path
+ self.name = name
self._parsed = False
- def isRelevant( self, vardb, config, profile ):
+ def isRelevant(self, vardb, config, profile):
"""
This function takes a dict of keyword arguments; one should pass in any
objects need to do to lookups (like what keywords we are on, what profile,
'profile' : profile }
for restriction in self.restrictions:
- if restriction.checkRestriction( **kwargs ):
+ if restriction.checkRestriction(**kwargs):
return True
return False # No restrictions were met; thus we aren't relevant :(
- def parse( self ):
+ def parse(self):
lines = open(self.path).readlines()
self.restrictions = []
for line in lines:
for regex, restriction in restricts.iteritems():
match = regex.match(line)
if match:
- self.restrictions.append( restriction( match.groups()[0].strip() ) )
+ self.restrictions.append(restriction(match.groups()[0].strip()))
continue
self._parsed = True
- def __getattr__( self, attr ):
+ def __getattr__(self, attr):
if not self._parsed:
self.parse()
return self.__dict__[attr]
are met, then it is displayed
"""
- def checkRestriction( self, **kwargs ):
+ def checkRestriction(self, **kwargs):
raise NotImplementedError("Derived class should over-ride this method")
class DisplayProfileRestriction(DisplayRestriction):
if the user is running a specific profile.
"""
- def __init__( self, profile ):
+ def __init__(self, profile):
self.profile = profile
- def checkRestriction( self, **kwargs ):
+ def checkRestriction(self, **kwargs):
if self.profile == kwargs['profile']:
return True
return False
if the user is running a specific keyword.
"""
- def __init__( self, keyword ):
+ def __init__(self, keyword):
self.keyword = keyword
- def checkRestriction( self, **kwargs ):
+ def checkRestriction(self, **kwargs):
if kwargs['config']["ARCH"] == self.keyword:
return True
return False
if the user has that item installed.
"""
- def __init__( self, cpv ):
+ def __init__(self, cpv):
self.cpv = cpv
- def checkRestriction( self, **kwargs ):
+ def checkRestriction(self, **kwargs):
vdb = kwargs['vardb']
- if vdb.match( self.cpv ):
+ if vdb.match(self.cpv):
return True
return False
projects / portage.git / commitdiff