From: Fabian Groffen Date: Fri, 14 Aug 2009 20:21:04 +0000 (-0000) Subject: Merged from trunk -r13957:13967 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=2b17d1d689b1476de3f4164332178d1e0ef30a1b;p=portage.git Merged from trunk -r13957:13967 | 13958 | Inside collect_ebuild_messages(), open elog files in text | | zmedico | mode (unicode). Thanks to jlec for reporting. | | 13959 | Bug #280690 - Fix incorrect idendification of start quote as | | zmedico | end quote when the start quote is immediately followed by a | | | newline. | | 13960 | Always use encoding='utf_8' for unicode() calls. | | zmedico | | | 13961 | Write log files in text mode (unicode). Thanks to Arfrever | | zmedico | for reporting. | | 13962 | Encode unicode messages as bytes in order to avoid TypeError | | zmedico | from syslog.syslog() with python-2.x. | | 13963 | Fix typo in comment. Thanks to Arfrever. | | zmedico | | | 13964 | Fix EOutput to safely write unicode to stderr/stdout. Thanks | | zmedico | to Arfrever for reporting. | | 13965 | Make everything safe for unicode (this should fix the elog | | zmedico | modules that send mail). | | 13966 | Remove unused import. | | zmedico | | svn path=/main/branches/prefix/; revision=14037 --- diff --git a/bin/filter-bash-environment.py b/bin/filter-bash-environment.py index 16ee0bc85..f6605acdc 100755 --- a/bin/filter-bash-environment.py +++ b/bin/filter-bash-environment.py @@ -48,7 +48,12 @@ def filter_bash_environment(pattern, file_in, file_out): quote = var_assign_match.group(3) filter_this = pattern.match(var_assign_match.group(2)) \ is not None - if quote is not None and not have_end_quote(quote, line): + # Exclude the start quote when searching for the end quote, + # to ensure that the start quote is not misidentified as the + # end quote (happens if there is a newline immediately after + # the start quote). + if quote is not None and not \ + have_end_quote(quote, line[var_assign_match.end(2)+2:]): multi_line_quote = quote multi_line_quote_filter = filter_this if not filter_this: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 967d7bec7..4f849722f 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -248,7 +248,7 @@ def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymli ftype = [] for x in list: if not isinstance(x, unicode): - x = unicode(x, errors='replace') + x = unicode(x, encoding='utf_8', errors='replace') try: if followSymlinks: pathstat = os.stat(mypath+"/"+x) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 9f36af569..d9b85b18a 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -879,7 +879,7 @@ class portdbapi(dbapi): continue for x in file_list: if not isinstance(x, unicode): - x = unicode(x, errors='replace') + x = unicode(x, encoding='utf_8', errors='replace') pf = None if glep55: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 18a9533f0..a5958db69 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1595,7 +1595,7 @@ class vardbapi(dbapi): returnme = [] for x in dir_list: if not isinstance(x, unicode): - x = unicode(x, errors='replace') + x = unicode(x, encoding='utf_8', errors='replace') if self._excluded_dirs.match(x) is not None: continue ps = pkgsplit(x) @@ -1628,7 +1628,8 @@ class vardbapi(dbapi): def listdir(p, **kwargs): try: return [isinstance(x, unicode) and x or \ - unicode(x, errors='replace') for x in os.listdir(p) \ + unicode(x, encoding='utf_8', errors='replace') \ + for x in os.listdir(p) \ if os.path.isdir(os.path.join(p, x))] except EnvironmentError, e: if e.errno == PermissionDenied.errno: diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index 3b44e0fe8..341f74c0e 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -12,6 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage.const import EBUILD_PHASES from portage.localization import _ +import codecs import os import sys @@ -40,7 +41,8 @@ def collect_ebuild_messages(path): logentries[msgfunction] = [] lastmsgtype = None msgcontent = [] - for l in open(filename): + for l in codecs.open(filename, mode='r', + encoding='utf_8', errors='replace'): if not l: continue try: diff --git a/pym/portage/elog/mod_mail_summary.py b/pym/portage/elog/mod_mail_summary.py index 90a6ed2f8..dc0143c82 100644 --- a/pym/portage/elog/mod_mail_summary.py +++ b/pym/portage/elog/mod_mail_summary.py @@ -7,7 +7,6 @@ import portage.mail, socket, os, time from portage.exception import PortageException from portage.localization import _ from portage.util import writemsg -from email.MIMEText import MIMEText as TextMessage _items = {} def process(mysettings, key, logentries, fulltext): diff --git a/pym/portage/elog/mod_save.py b/pym/portage/elog/mod_save.py index cccd63796..741e94e14 100644 --- a/pym/portage/elog/mod_save.py +++ b/pym/portage/elog/mod_save.py @@ -3,6 +3,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import codecs import os, time from portage.data import portage_uid, portage_gid from portage.util import ensure_dirs @@ -18,7 +19,8 @@ def process(mysettings, key, logentries, fulltext): ensure_dirs(elogdir, uid=portage_uid, gid=portage_gid, mode=02770) elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log" - elogfile = open(elogfilename, "w") + elogfile = codecs.open(elogfilename, mode='w', + encoding='utf_8', errors='replace') elogfile.write(fulltext) elogfile.close() diff --git a/pym/portage/elog/mod_save_summary.py b/pym/portage/elog/mod_save_summary.py index 1b749f24b..f35bb70a3 100644 --- a/pym/portage/elog/mod_save_summary.py +++ b/pym/portage/elog/mod_save_summary.py @@ -3,6 +3,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import codecs import os, time from portage.data import portage_uid, portage_gid from portage.localization import _ @@ -18,7 +19,8 @@ def process(mysettings, key, logentries, fulltext): # TODO: Locking elogfilename = elogdir+"/summary.log" - elogfile = open(elogfilename, "a") + elogfile = codecs.open(elogfilename, mode='a', + encoding='utf_8', errors='replace') apply_permissions(elogfilename, mode=060, mask=0) elogfile.write(_(">>> Messages generated by process %(pid)d on %(time)s for package %(pkg)s:\n\n") % {"pid": os.getpid(), "time": time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime(time.time())), "pkg": key}) diff --git a/pym/portage/elog/mod_syslog.py b/pym/portage/elog/mod_syslog.py index bb66b2b74..0fe205644 100644 --- a/pym/portage/elog/mod_syslog.py +++ b/pym/portage/elog/mod_syslog.py @@ -3,20 +3,28 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import sys import syslog from portage.const import EBUILD_PHASES +_pri = { + "INFO" : syslog.LOG_INFO, + "WARN" : syslog.LOG_WARNING, + "ERROR" : syslog.LOG_ERR, + "LOG" : syslog.LOG_NOTICE, + "QA" : syslog.LOG_WARNING +} + def process(mysettings, key, logentries, fulltext): syslog.openlog("portage", syslog.LOG_ERR | syslog.LOG_WARNING | syslog.LOG_INFO | syslog.LOG_NOTICE, syslog.LOG_LOCAL5) for phase in EBUILD_PHASES: if not phase in logentries: continue for msgtype,msgcontent in logentries[phase]: - pri = {"INFO": syslog.LOG_INFO, - "WARN": syslog.LOG_WARNING, - "ERROR": syslog.LOG_ERR, - "LOG": syslog.LOG_NOTICE, - "QA": syslog.LOG_WARNING} msgtext = "".join(msgcontent) - syslog.syslog(pri[msgtype], "%s: %s: %s" % (key, phase, msgtext)) + msgtext = "%s: %s: %s" % (key, phase, msgtext) + if sys.hexversion < 0x3000000 and isinstance(msgtext, unicode): + # Avoid TypeError from syslog.syslog() + msgtext = msgtext.encode('utf_8', 'replace') + syslog.syslog(_pri[msgtype], msgtext) syslog.closelog() diff --git a/pym/portage/mail.py b/pym/portage/mail.py index 5f1cc11a6..9779fac86 100644 --- a/pym/portage/mail.py +++ b/pym/portage/mail.py @@ -10,6 +10,17 @@ from email.MIMEBase import MIMEBase as BaseMessage from email.header import Header def create_message(sender, recipient, subject, body, attachments=None): + + if sys.hexversion < 0x3000000: + if isinstance(sender, unicode): + sender = sender.encode('utf_8', 'replace') + if isinstance(recipient, unicode): + recipient = recipient.encode('utf_8', 'replace') + if isinstance(subject, unicode): + subject = subject.encode('utf_8', 'replace') + if isinstance(body, unicode): + body = body.encode('utf_8', 'replace') + if attachments == None: mymessage = TextMessage(body) else: @@ -19,6 +30,8 @@ def create_message(sender, recipient, subject, body, attachments=None): if isinstance(x, BaseMessage): mymessage.attach(x) elif isinstance(x, basestring): + if sys.hexversion < 0x3000000 and isinstance(x, unicode): + x = x.encode('utf_8', 'replace') mymessage.attach(TextMessage(x)) else: raise portage.exception.PortageException("Can't handle type of attachment: %s" % type(x)) @@ -66,7 +79,21 @@ def send_mail(mysettings, message): myrecipient = mysettings["PORTAGE_ELOG_MAILURI"] myfrom = message.get("From") - + + if sys.hexversion < 0x3000000: + if isinstance(myrecipient, unicode): + myrecipient = myrecipient.encode('utf_8', 'replace') + if isinstance(mymailhost, unicode): + mymailhost = mymailhost.encode('utf_8', 'replace') + if isinstance(mymailport, unicode): + mymailport = mymailport.encode('utf_8', 'replace') + if isinstance(myfrom, unicode): + myfrom = myfrom.encode('utf_8', 'replace') + if isinstance(mymailuser, unicode): + mymailuser = mymailuser.encode('utf_8', 'replace') + if isinstance(mymailpasswd, unicode): + mymailpasswd = mymailpasswd.encode('utf_8', 'replace') + # user wants to use a sendmail binary instead of smtp if mymailhost[0] == os.sep and os.path.exists(mymailhost): fd = os.popen(mymailhost+" -f "+myfrom+" "+myrecipient, "w") diff --git a/pym/portage/output.py b/pym/portage/output.py index ab28b2e4a..258fa7562 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -466,6 +466,13 @@ class EOutput(object): sys.stdout.flush() sys.stderr.flush() + def _write(self, f, s): + if sys.hexversion < 0x3000000 and isinstance(s, unicode): + # avoid potential UnicodeEncodeError + s = s.encode('utf_8', 'replace') + f.write(s) + f.flush() + def __eend(self, caller, errno, msg): if errno == 0: status_brackets = colorize("BRACKET", "[ ") + colorize("GOOD", "ok") + colorize("BRACKET", " ]") @@ -480,9 +487,9 @@ class EOutput(object): self.__last_e_len = 0 if not self.quiet: out = sys.stdout - out.write("%*s%s\n" % ((self.term_columns - self.__last_e_len - 6), + self._write(out, + "%*s%s\n" % ((self.term_columns - self.__last_e_len - 6), "", status_brackets)) - out.flush() def ebegin(self, msg): """ @@ -524,9 +531,8 @@ class EOutput(object): out = sys.stderr if not self.quiet: if self.__last_e_cmd == "ebegin": - out.write("\n") - out.write(colorize("BAD", " * ") + msg + "\n") - out.flush() + self._write(out, "\n") + self._write(out, colorize("BAD", " * ") + msg + "\n") self.__last_e_cmd = "eerror" def einfo(self, msg): @@ -539,9 +545,8 @@ class EOutput(object): out = sys.stdout if not self.quiet: if self.__last_e_cmd == "ebegin": - out.write("\n") - out.write(colorize("GOOD", " * ") + msg + "\n") - out.flush() + self._write(out, "\n") + self._write(out, colorize("GOOD", " * ") + msg + "\n") self.__last_e_cmd = "einfo" def einfon(self, msg): @@ -554,9 +559,8 @@ class EOutput(object): out = sys.stdout if not self.quiet: if self.__last_e_cmd == "ebegin": - out.write("\n") - out.write(colorize("GOOD", " * ") + msg) - out.flush() + self._write(out, "\n") + self._write(out, colorize("GOOD", " * ") + msg) self.__last_e_cmd = "einfon" def ewarn(self, msg): @@ -569,9 +573,8 @@ class EOutput(object): out = sys.stderr if not self.quiet: if self.__last_e_cmd == "ebegin": - out.write("\n") - out.write(colorize("WARN", " * ") + msg + "\n") - out.flush() + self._write(out, "\n") + self._write(out, colorize("WARN", " * ") + msg + "\n") self.__last_e_cmd = "ewarn" def ewend(self, errno, *msg): diff --git a/pym/portage/sets/shell.py b/pym/portage/sets/shell.py index 43a8fe02f..25ccb03c7 100644 --- a/pym/portage/sets/shell.py +++ b/pym/portage/sets/shell.py @@ -35,7 +35,8 @@ class CommandOutputSet(PackageSet): pipe = subprocess.Popen(self._command, stdout=subprocess.PIPE, shell=True) stdout, stderr = pipe.communicate() if pipe.wait() == os.EX_OK: - self._setAtoms(unicode(stdout, errors='replace').splitlines()) + self._setAtoms(unicode(stdout, + encoding='utf_8', errors='replace').splitlines()) def singleBuilder(self, options, settings, trees): if not "command" in options: