From 093cd82b06d587896c8e08bed309d052c73e41e6 Mon Sep 17 00:00:00 2001 From: Marius Mauch Date: Fri, 18 May 2007 17:16:55 +0000 Subject: [PATCH] Enable elog functionality for the python side of portage svn path=/main/trunk/; revision=6548 --- pym/portage/elog/__init__.py | 180 +++++++++++++-------------- pym/portage/elog/filtering.py | 17 +++ pym/portage/elog/messages.py | 89 +++++++++++++ pym/portage/elog/mod_custom.py | 6 +- pym/portage/elog/mod_echo.py | 12 +- pym/portage/elog/mod_mail.py | 4 +- pym/portage/elog/mod_mail_summary.py | 10 +- pym/portage/elog/mod_save.py | 6 +- pym/portage/elog/mod_save_summary.py | 4 +- pym/portage/elog/mod_syslog.py | 4 +- 10 files changed, 213 insertions(+), 119 deletions(-) create mode 100644 pym/portage/elog/filtering.py create mode 100644 pym/portage/elog/messages.py diff --git a/pym/portage/elog/__init__.py b/pym/portage/elog/__init__.py index be7eb5b66..77bdd1b94 100644 --- a/pym/portage/elog/__init__.py +++ b/pym/portage/elog/__init__.py @@ -8,111 +8,99 @@ from portage.exception import PortageException from portage.process import atexit_register from portage.util import writemsg -from portage import listdir +from portage.elog.messages import collect_ebuild_messages, collect_messages +from portage.elog.filtering import filter_loglevels import os +def _merge_logentries(a, b): + rValue = {} + phases = set(a.keys()+b.keys()) + for p in phases: + rValue[p] = [] + if a.has_key(p): + for x in a[p]: + rValue[p].append(x) + if b.has_key(p): + for x in b[p]: + rValue[p].append(x) + return rValue + +def _combine_logentries(logentries): + # generate a single string with all log messages + rValue = "" + for phase in EBUILD_PHASES: + if not phase in logentries: + continue + for msgtype, msgcontent in logentries[phase]: + rValue += "%s: %s\n" % (msgtype, phase) + for line in msgcontent: + rValue += line + rValue += "\n" + return rValue + _elog_atexit_handlers = [] def elog_process(cpv, mysettings): - mylogfiles = listdir(mysettings["T"]+"/logging/") - # shortcut for packages without any messages - if len(mylogfiles) == 0: - return - # exploit listdir() file order so we process log entries in chronological order - mylogfiles.reverse() - all_logentries = {} - for f in mylogfiles: - msgfunction, msgtype = f.split(".") - if msgfunction not in EBUILD_PHASES: - writemsg("!!! can't process invalid log file: %s\n" % f, - noiselevel=-1) - continue - if not msgfunction in all_logentries: - all_logentries[msgfunction] = [] - msgcontent = open(mysettings["T"]+"/logging/"+f, "r").readlines() - all_logentries[msgfunction].append((msgtype, msgcontent)) + ebuild_logentries = collect_ebuild_messages(os.path.join(mysettings["T"], "logging")) + all_logentries = collect_messages() + if all_logentries.has_key(cpv): + all_logentries[cpv] = _merge_logentries(ebuild_logentries, all_logentries[cpv]) + else: + all_logentries[cpv] = ebuild_logentries - def filter_loglevels(logentries, loglevels): - # remove unwanted entries from all logentries - rValue = {} - loglevels = map(str.upper, loglevels) - for phase in logentries.keys(): - for msgtype, msgcontent in logentries[phase]: - if msgtype.upper() in loglevels or "*" in loglevels: - if not rValue.has_key(phase): - rValue[phase] = [] - rValue[phase].append((msgtype, msgcontent)) - return rValue - my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split()) - default_logentries = filter_loglevels(all_logentries, my_elog_classes) - # in case the filters matched all messages and no module overrides exist - if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")): - return - def combine_logentries(logentries): - # generate a single string with all log messages - rValue = "" - for phase in EBUILD_PHASES: - if not phase in logentries: - continue - for msgtype, msgcontent in logentries[phase]: - rValue += "%s: %s\n" % (msgtype, phase) - for line in msgcontent: - rValue += line - rValue += "\n" - return rValue - - default_fulllog = combine_logentries(default_logentries) + for key in all_logentries.keys(): + default_logentries = filter_loglevels(all_logentries[key], my_elog_classes) - # pass the processing to the individual modules - logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split() - for s in logsystems: - # allow per module overrides of PORTAGE_ELOG_CLASSES - if ":" in s: - s, levels = s.split(":", 1) - levels = levels.split(",") - mod_logentries = filter_loglevels(all_logentries, levels) - mod_fulllog = combine_logentries(mod_logentries) - else: - mod_logentries = default_logentries - mod_fulllog = default_fulllog - if len(mod_logentries) == 0: - continue - # - is nicer than _ for module names, so allow people to use it. - s = s.replace("-", "_") - try: - # FIXME: ugly ad.hoc import code - # TODO: implement a common portage module loader - name = "portage.elog.mod_" + s - m = __import__(name) - for comp in name.split(".")[1:]: - m = getattr(m, comp) - def timeout_handler(signum, frame): - raise PortageException("Timeout in elog_process for system '%s'" % s) - import signal - signal.signal(signal.SIGALRM, timeout_handler) - # Timeout after one minute (in case something like the mail - # module gets hung). - signal.alarm(60) + # in case the filters matched all messages and no module overrides exist + if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")): + return + + default_fulllog = _combine_logentries(default_logentries) + + # pass the processing to the individual modules + logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split() + for s in logsystems: + # allow per module overrides of PORTAGE_ELOG_CLASSES + if ":" in s: + s, levels = s.split(":", 1) + levels = levels.split(",") + mod_logentries = filter_loglevels(all_logentries[key], levels) + mod_fulllog = combine_logentries(mod_logentries) + else: + mod_logentries = default_logentries + mod_fulllog = default_fulllog + if len(mod_logentries) == 0: + continue + # - is nicer than _ for module names, so allow people to use it. + s = s.replace("-", "_") try: - m.process(mysettings, cpv, mod_logentries, mod_fulllog) - finally: - signal.alarm(0) - if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: - _elog_atexit_handlers.append(m.finalize) - atexit_register(m.finalize, mysettings) - except (ImportError, AttributeError), e: - writemsg("!!! Error while importing logging modules " + \ - "while loading \"mod_%s\":\n" % str(s)) - writemsg("%s\n" % str(e), noiselevel=-1) - except PortageException, e: - writemsg("%s\n" % str(e), noiselevel=-1) + # FIXME: ugly ad.hoc import code + # TODO: implement a common portage module loader + name = "portage.elog.mod_" + s + m = __import__(name) + for comp in name.split(".")[1:]: + m = getattr(m, comp) + def timeout_handler(signum, frame): + raise PortageException("Timeout in elog_process for system '%s'" % s) + import signal + signal.signal(signal.SIGALRM, timeout_handler) + # Timeout after one minute (in case something like the mail + # module gets hung). + signal.alarm(60) + try: + m.process(mysettings, str(key), mod_logentries, mod_fulllog) + finally: + signal.alarm(0) + if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: + _elog_atexit_handlers.append(m.finalize) + atexit_register(m.finalize, mysettings) + except (ImportError, AttributeError), e: + writemsg("!!! Error while importing logging modules " + \ + "while loading \"mod_%s\":\n" % str(s)) + writemsg("%s\n" % str(e), noiselevel=-1) + except PortageException, e: + writemsg("%s\n" % str(e), noiselevel=-1) - # clean logfiles to avoid repetitions - for f in mylogfiles: - try: - os.unlink(os.path.join(mysettings["T"], "logging", f)) - except OSError: - pass diff --git a/pym/portage/elog/filtering.py b/pym/portage/elog/filtering.py new file mode 100644 index 000000000..f4748fede --- /dev/null +++ b/pym/portage/elog/filtering.py @@ -0,0 +1,17 @@ +# elog/messages.py - elog core functions +# Copyright 2006-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id: __init__.py 6458 2007-04-30 02:31:30Z genone $ + +def filter_loglevels(logentries, loglevels): + # remove unwanted entries from all logentries + rValue = {} + loglevels = map(str.upper, loglevels) + for phase in logentries.keys(): + for msgtype, msgcontent in logentries[phase]: + if msgtype.upper() in loglevels or "*" in loglevels: + if not rValue.has_key(phase): + rValue[phase] = [] + rValue[phase].append((msgtype, msgcontent)) + return rValue + diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py new file mode 100644 index 000000000..bfe428030 --- /dev/null +++ b/pym/portage/elog/messages.py @@ -0,0 +1,89 @@ +# elog/messages.py - elog core functions +# Copyright 2006-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id: __init__.py 6458 2007-04-30 02:31:30Z genone $ + +from portage.output import colorize +from portage.const import EBUILD_PHASES +from portage.util import writemsg + +from portage import listdir + +import os + +def collect_ebuild_messages(path): + """ Collect elog messages generated by the bash logging function stored + at 'path'. + """ + mylogfiles = listdir(path) + # shortcut for packages without any messages + if len(mylogfiles) == 0: + return {} + # exploit listdir() file order so we process log entries in chronological order + mylogfiles.reverse() + logentries = {} + for f in mylogfiles: + msgfunction, msgtype = f.split(".") + if msgfunction not in EBUILD_PHASES: + writemsg("!!! can't process invalid log file: %s\n" % f, + noiselevel=-1) + continue + if not msgfunction in logentries: + logentries[msgfunction] = [] + msgcontent = open(os.path.join(path, f), "r").readlines() + logentries[msgfunction].append((msgtype, msgcontent)) + # clean logfiles to avoid repetitions + for f in mylogfiles: + try: + os.unlink(os.path.join(path, f)) + except OSError: + pass + return logentries + +_msgbuffer = {} +def _elog_base(level, msg, phase="other", key=None, color=None): + """ Backend for the other messaging functions, should not be called + directly. + """ + if color == None: + color = "GOOD" + print colorize(color, " * ")+msg + if not _msgbuffer.has_key(key): + _msgbuffer[key] = {} + if not _msgbuffer[key].has_key(phase): + _msgbuffer[key][phase] = [] + _msgbuffer[key][phase].append((level, msg)) + + #raise NotImplementedError() + +def collect_messages(): + rValue = _msgbuffer + _reset_buffer() + return rValue + +def _reset_buffer(): + """ Reset the internal message buffer when it has been processed, + should not be called directly. + """ + _msgbuffer = {} + +# creating and exporting the actual messaging functions +_functions = { "einfo": ("INFO", "GOOD"), + "elog": ("LOG", "GOOD"), + "ewarn": ("WARN", "WARN"), + "eqawarn": ("QA", "WARN"), + "eerror": ("ERROR", "ERROR"), +} + +def _make_msgfunction(level, color): + def _elog(msg, phase="other", key=None): + """ Display and log a message assigned to the given key/cpv + (or unassigned if no key is given). + """ + _elog_base(level, msg, phase=phase, key=key, color=color) + return _elog + +import sys +for f in _functions.keys(): + setattr(sys.modules[__name__], f, _make_msgfunction(_functions[f][0], _functions[f][1])) +del f, _functions diff --git a/pym/portage/elog/mod_custom.py b/pym/portage/elog/mod_custom.py index 3433bbbf1..a486c8931 100644 --- a/pym/portage/elog/mod_custom.py +++ b/pym/portage/elog/mod_custom.py @@ -5,8 +5,8 @@ import portage.elog_modules.mod_save, portage.process, portage.exception -def process(mysettings, cpv, logentries, fulltext): - elogfilename = portage.elog_modules.mod_save.process(mysettings, cpv, logentries, fulltext) +def process(mysettings, key, logentries, fulltext): + elogfilename = portage.elog_modules.mod_save.process(mysettings, key, logentries, fulltext) if (not "PORTAGE_ELOG_COMMAND" in mysettings.keys()) \ or len(mysettings["PORTAGE_ELOG_COMMAND"]) == 0: @@ -14,7 +14,7 @@ def process(mysettings, cpv, logentries, fulltext): else: mylogcmd = mysettings["PORTAGE_ELOG_COMMAND"] mylogcmd = mylogcmd.replace("${LOGFILE}", elogfilename) - mylogcmd = mylogcmd.replace("${PACKAGE}", cpv) + mylogcmd = mylogcmd.replace("${PACKAGE}", key) retval = portage.process.spawn_bash(mylogcmd) if retval != 0: raise portage.exception.PortageException("!!! PORTAGE_ELOG_COMMAND failed with exitcode %d" % retval) diff --git a/pym/portage/elog/mod_echo.py b/pym/portage/elog/mod_echo.py index 73928b437..125a8c40e 100644 --- a/pym/portage/elog/mod_echo.py +++ b/pym/portage/elog/mod_echo.py @@ -7,19 +7,19 @@ from portage.output import EOutput from portage.const import EBUILD_PHASES _items = {} -def process(mysettings, cpv, logentries, fulltext): - _items[cpv] = logentries +def process(mysettings, key, logentries, fulltext): + _items[key] = logentries def finalize(mysettings): printer = EOutput() - for cpv in _items.keys(): + for key in _items.keys(): print - printer.einfo("Messages for package %s:" % cpv) + printer.einfo("Messages for package %s:" % key) print for phase in EBUILD_PHASES: - if not phase in _items[cpv]: + if not phase in _items[key]: continue - for msgtype, msgcontent in _items[cpv][phase]: + for msgtype, msgcontent in _items[key][phase]: fmap = {"INFO": printer.einfo, "WARN": printer.ewarn, "ERROR": printer.eerror, diff --git a/pym/portage/elog/mod_mail.py b/pym/portage/elog/mod_mail.py index 4695e8ec4..d0d2b33ac 100644 --- a/pym/portage/elog/mod_mail.py +++ b/pym/portage/elog/mod_mail.py @@ -5,7 +5,7 @@ import portage.mail, socket -def process(mysettings, cpv, logentries, fulltext): +def process(mysettings, key, logentries, fulltext): if mysettings.has_key("PORTAGE_ELOG_MAILURI"): myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0] else: @@ -13,7 +13,7 @@ def process(mysettings, cpv, logentries, fulltext): myfrom = mysettings["PORTAGE_ELOG_MAILFROM"] mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"] - mysubject = mysubject.replace("${PACKAGE}", cpv) + mysubject = mysubject.replace("${PACKAGE}", key) mysubject = mysubject.replace("${HOST}", socket.getfqdn()) mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, fulltext) diff --git a/pym/portage/elog/mod_mail_summary.py b/pym/portage/elog/mod_mail_summary.py index a1f4ed57f..a8e694935 100644 --- a/pym/portage/elog/mod_mail_summary.py +++ b/pym/portage/elog/mod_mail_summary.py @@ -7,10 +7,10 @@ import portage.mail, socket, os, time from email.MIMEText import MIMEText as TextMessage _items = {} -def process(mysettings, cpv, logentries, fulltext): +def process(mysettings, key, logentries, fulltext): header = ">>> Messages generated for package %s by process %d on %s:\n\n" % \ - (cpv, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))) - _items[cpv] = header + fulltext + (key, os.getpid(), time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))) + _items[key] = header + fulltext def finalize(mysettings): if len(_items) == 0: @@ -31,8 +31,8 @@ def finalize(mysettings): mybody = "elog messages for the following packages generated by " + \ "process %d on host %s:\n" % (os.getpid(), socket.getfqdn()) - for cpv in _items.keys(): - mybody += "- %s\n" % cpv + for key in _items.keys(): + mybody += "- %s\n" % key mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, mybody, attachments=_items.values()) portage.mail.send_mail(mysettings, mymessage) diff --git a/pym/portage/elog/mod_save.py b/pym/portage/elog/mod_save.py index eff40edd9..31c410fac 100644 --- a/pym/portage/elog/mod_save.py +++ b/pym/portage/elog/mod_save.py @@ -6,8 +6,8 @@ import os, time from portage.data import portage_uid, portage_gid -def process(mysettings, cpv, logentries, fulltext): - cpv_path = cpv.replace("/", ":") +def process(mysettings, key, logentries, fulltext): + path = key.replace("/", ":") if mysettings["PORT_LOGDIR"] != "": elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog") @@ -18,7 +18,7 @@ def process(mysettings, cpv, logentries, fulltext): os.chown(elogdir, portage_uid, portage_gid) os.chmod(elogdir, 02770) - elogfilename = elogdir+"/"+cpv_path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log" + elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log" elogfile = open(elogfilename, "w") elogfile.write(fulltext) elogfile.close() diff --git a/pym/portage/elog/mod_save_summary.py b/pym/portage/elog/mod_save_summary.py index aebc6f9aa..25f8ae77a 100644 --- a/pym/portage/elog/mod_save_summary.py +++ b/pym/portage/elog/mod_save_summary.py @@ -6,7 +6,7 @@ import os, time from portage.data import portage_uid, portage_gid -def process(mysettings, cpv, logentries, fulltext): +def process(mysettings, key, logentries, fulltext): if mysettings["PORT_LOGDIR"] != "": elogdir = os.path.join(mysettings["PORT_LOGDIR"], "elog") else: @@ -20,7 +20,7 @@ def process(mysettings, cpv, logentries, fulltext): elogfilename = elogdir+"/summary.log" elogfile = open(elogfilename, "a") elogfile.write(">>> Messages generated by process %d on %s for package %s:\n\n" % \ - (os.getpid(), time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), cpv)) + (os.getpid(), time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), key)) elogfile.write(fulltext) elogfile.write("\n") elogfile.close() diff --git a/pym/portage/elog/mod_syslog.py b/pym/portage/elog/mod_syslog.py index 28aa15b3c..bb66b2b74 100644 --- a/pym/portage/elog/mod_syslog.py +++ b/pym/portage/elog/mod_syslog.py @@ -6,7 +6,7 @@ import syslog from portage.const import EBUILD_PHASES -def process(mysettings, cpv, logentries, fulltext): +def process(mysettings, key, logentries, fulltext): syslog.openlog("portage", syslog.LOG_ERR | syslog.LOG_WARNING | syslog.LOG_INFO | syslog.LOG_NOTICE, syslog.LOG_LOCAL5) for phase in EBUILD_PHASES: if not phase in logentries: @@ -18,5 +18,5 @@ def process(mysettings, cpv, logentries, fulltext): "LOG": syslog.LOG_NOTICE, "QA": syslog.LOG_WARNING} msgtext = "".join(msgcontent) - syslog.syslog(pri[msgtype], "%s: %s: %s" % (cpv, phase, msgtext)) + syslog.syslog(pri[msgtype], "%s: %s: %s" % (key, phase, msgtext)) syslog.closelog() -- 2.26.2