Merged from trunk -r13957:13967
authorFabian Groffen <grobian@gentoo.org>
Fri, 14 Aug 2009 20:21:04 +0000 (20:21 -0000)
committerFabian Groffen <grobian@gentoo.org>
Fri, 14 Aug 2009 20:21:04 +0000 (20:21 -0000)
   | 13958   | Inside collect_ebuild_messages(), open elog files in text    |
   | zmedico | mode (unicode). Thanks to jlec for reporting.                |

   | 13959   | Bug #280690 - Fix incorrect idendification of start quote as |
   | zmedico | end quote when the start quote is immediately followed by a  |
   |         | newline.                                                     |

   | 13960   | Always use encoding='utf_8' for unicode() calls.             |
   | zmedico |                                                              |

   | 13961   | Write log files in text mode (unicode). Thanks to Arfrever   |
   | zmedico | for reporting.                                               |

   | 13962   | Encode unicode messages as bytes in order to avoid TypeError |
   | zmedico | from syslog.syslog() with python-2.x.                        |

   | 13963   | Fix typo in comment. Thanks to Arfrever.                     |
   | zmedico |                                                              |

   | 13964   | Fix EOutput to safely write unicode to stderr/stdout. Thanks |
   | zmedico | to Arfrever for reporting.                                   |

   | 13965   | Make everything safe for unicode (this should fix the elog   |
   | zmedico | modules that send mail).                                     |

   | 13966   | Remove unused import.                                        |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=14037

12 files changed:
bin/filter-bash-environment.py
pym/portage/__init__.py
pym/portage/dbapi/porttree.py
pym/portage/dbapi/vartree.py
pym/portage/elog/messages.py
pym/portage/elog/mod_mail_summary.py
pym/portage/elog/mod_save.py
pym/portage/elog/mod_save_summary.py
pym/portage/elog/mod_syslog.py
pym/portage/mail.py
pym/portage/output.py
pym/portage/sets/shell.py

index 16ee0bc8594cb6ebf0f61818ea91099c4db6691d..f6605acdc2fe58e296ed692427d354ceec0011f1 100755 (executable)
@@ -48,7 +48,12 @@ def filter_bash_environment(pattern, file_in, file_out):
                                quote = var_assign_match.group(3)
                                filter_this = pattern.match(var_assign_match.group(2)) \
                                        is not None
-                               if quote is not None and not have_end_quote(quote, line):
+                               # Exclude the start quote when searching for the end quote,
+                               # to ensure that the start quote is not misidentified as the
+                               # end quote (happens if there is a newline immediately after
+                               # the start quote).
+                               if quote is not None and not \
+                                       have_end_quote(quote, line[var_assign_match.end(2)+2:]):
                                        multi_line_quote = quote
                                        multi_line_quote_filter = filter_this
                                if not filter_this:
index 967d7bec7bd231cdc749e824f94a06f54bd8d1b4..4f849722f0479638cb754fb89875d88e6e39bcb5 100644 (file)
@@ -248,7 +248,7 @@ def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymli
                ftype = []
                for x in list:
                        if not isinstance(x, unicode):
-                               x = unicode(x, errors='replace')
+                               x = unicode(x, encoding='utf_8', errors='replace')
                        try:
                                if followSymlinks:
                                        pathstat = os.stat(mypath+"/"+x)
index 9f36af5699b9cae9d589e6d880626e0102bf429a..d9b85b18abdbc981abc62cf6c95f88caf238d7f6 100644 (file)
@@ -879,7 +879,7 @@ class portdbapi(dbapi):
                                continue
                        for x in file_list:
                                if not isinstance(x, unicode):
-                                       x = unicode(x, errors='replace')
+                                       x = unicode(x, encoding='utf_8', errors='replace')
 
                                pf = None
                                if glep55:
index 18a9533f0497a9dbb466c5ed02ca5527cdc7947f..a5958db694ff5b7752995973a2ab9001247ebe36 100644 (file)
@@ -1595,7 +1595,7 @@ class vardbapi(dbapi):
                returnme = []
                for x in dir_list:
                        if not isinstance(x, unicode):
-                               x = unicode(x, errors='replace')
+                               x = unicode(x, encoding='utf_8', errors='replace')
                        if self._excluded_dirs.match(x) is not None:
                                continue
                        ps = pkgsplit(x)
@@ -1628,7 +1628,8 @@ class vardbapi(dbapi):
                        def listdir(p, **kwargs):
                                try:
                                        return [isinstance(x, unicode) and x or \
-                                               unicode(x, errors='replace') for x in os.listdir(p) \
+                                               unicode(x, encoding='utf_8', errors='replace') \
+                                               for x in os.listdir(p) \
                                                if os.path.isdir(os.path.join(p, x))]
                                except EnvironmentError, e:
                                        if e.errno == PermissionDenied.errno:
index 3b44e0fe812f6ab8458ce10679ff9eee17f02a41..341f74c0e1b58db6143d1d19754407df003c2cce 100644 (file)
@@ -12,6 +12,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 from portage.const import EBUILD_PHASES
 from portage.localization import _
 
+import codecs
 import os
 import sys
 
@@ -40,7 +41,8 @@ def collect_ebuild_messages(path):
                        logentries[msgfunction] = []
                lastmsgtype = None
                msgcontent = []
-               for l in open(filename):
+               for l in codecs.open(filename, mode='r',
+                       encoding='utf_8', errors='replace'):
                        if not l:
                                continue
                        try:
index 90a6ed2f895f5828d217047f78bb539523cf4f84..dc0143c82ff470f43c05bb5a223bb50b919e1cd6 100644 (file)
@@ -7,7 +7,6 @@ import portage.mail, socket, os, time
 from portage.exception import PortageException
 from portage.localization import _
 from portage.util import writemsg
-from email.MIMEText import MIMEText as TextMessage
 
 _items = {}
 def process(mysettings, key, logentries, fulltext):
index cccd63796f6b3dfe846a89b9cd20943849bb5e8d..741e94e147b1dde57558c432b7c90f7b52bf63fc 100644 (file)
@@ -3,6 +3,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import codecs
 import os, time
 from portage.data import portage_uid, portage_gid
 from portage.util import ensure_dirs
@@ -18,7 +19,8 @@ def process(mysettings, key, logentries, fulltext):
        ensure_dirs(elogdir, uid=portage_uid, gid=portage_gid, mode=02770)
 
        elogfilename = elogdir+"/"+path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log"
-       elogfile = open(elogfilename, "w")
+       elogfile = codecs.open(elogfilename, mode='w',
+               encoding='utf_8', errors='replace')
        elogfile.write(fulltext)
        elogfile.close()
 
index 1b749f24b1289c6b636c7d6ae23dcaeb36a8c465..f35bb70a3fc0c58aa6caf7a21623c476a38ef893 100644 (file)
@@ -3,6 +3,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import codecs
 import os, time
 from portage.data import portage_uid, portage_gid
 from portage.localization import _
@@ -18,7 +19,8 @@ def process(mysettings, key, logentries, fulltext):
 
        # TODO: Locking
        elogfilename = elogdir+"/summary.log"
-       elogfile = open(elogfilename, "a")
+       elogfile = codecs.open(elogfilename, mode='a',
+               encoding='utf_8', errors='replace')
        apply_permissions(elogfilename, mode=060, mask=0)
        elogfile.write(_(">>> Messages generated by process %(pid)d on %(time)s for package %(pkg)s:\n\n") %
                        {"pid": os.getpid(), "time": time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime(time.time())), "pkg": key})
index bb66b2b74ba6f0b0e27c1f87084b893a73d259dd..0fe2056445531a5e0191a19fa6f88fdd0f605f8c 100644 (file)
@@ -3,20 +3,28 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import sys
 import syslog
 from portage.const import EBUILD_PHASES
 
+_pri = {
+       "INFO"   : syslog.LOG_INFO, 
+       "WARN"   : syslog.LOG_WARNING, 
+       "ERROR"  : syslog.LOG_ERR, 
+       "LOG"    : syslog.LOG_NOTICE,
+       "QA"     : syslog.LOG_WARNING
+}
+
 def process(mysettings, key, logentries, fulltext):
        syslog.openlog("portage", syslog.LOG_ERR | syslog.LOG_WARNING | syslog.LOG_INFO | syslog.LOG_NOTICE, syslog.LOG_LOCAL5)
        for phase in EBUILD_PHASES:
                if not phase in logentries:
                        continue
                for msgtype,msgcontent in logentries[phase]:
-                       pri = {"INFO": syslog.LOG_INFO, 
-                               "WARN": syslog.LOG_WARNING, 
-                               "ERROR": syslog.LOG_ERR, 
-                               "LOG": syslog.LOG_NOTICE,
-                               "QA": syslog.LOG_WARNING}
                        msgtext = "".join(msgcontent)
-                       syslog.syslog(pri[msgtype], "%s: %s: %s" % (key, phase, msgtext))
+                       msgtext = "%s: %s: %s" % (key, phase, msgtext)
+                       if sys.hexversion < 0x3000000 and isinstance(msgtext, unicode):
+                               # Avoid TypeError from syslog.syslog()
+                               msgtext = msgtext.encode('utf_8', 'replace')
+                       syslog.syslog(_pri[msgtype], msgtext)
        syslog.closelog()
index 5f1cc11a6fcd22a7aee36920d461d17f82fbd901..9779fac86b186f8a0a70669e63b719937d493874 100644 (file)
@@ -10,6 +10,17 @@ from email.MIMEBase import MIMEBase as BaseMessage
 from email.header import Header
 
 def create_message(sender, recipient, subject, body, attachments=None):
+
+       if sys.hexversion < 0x3000000:
+               if isinstance(sender, unicode):
+                       sender = sender.encode('utf_8', 'replace')
+               if isinstance(recipient, unicode):
+                       recipient = recipient.encode('utf_8', 'replace')
+               if isinstance(subject, unicode):
+                       subject = subject.encode('utf_8', 'replace')
+               if isinstance(body, unicode):
+                       body = body.encode('utf_8', 'replace')
+
        if attachments == None:
                mymessage = TextMessage(body)
        else:
@@ -19,6 +30,8 @@ def create_message(sender, recipient, subject, body, attachments=None):
                        if isinstance(x, BaseMessage):
                                mymessage.attach(x)
                        elif isinstance(x, basestring):
+                               if sys.hexversion < 0x3000000 and isinstance(x, unicode):
+                                       x = x.encode('utf_8', 'replace')
                                mymessage.attach(TextMessage(x))
                        else:
                                raise portage.exception.PortageException("Can't handle type of attachment: %s" % type(x))
@@ -66,7 +79,21 @@ def send_mail(mysettings, message):
                myrecipient = mysettings["PORTAGE_ELOG_MAILURI"]
        
        myfrom = message.get("From")
-               
+
+       if sys.hexversion < 0x3000000:
+               if isinstance(myrecipient, unicode):
+                       myrecipient = myrecipient.encode('utf_8', 'replace')
+               if isinstance(mymailhost, unicode):
+                       mymailhost = mymailhost.encode('utf_8', 'replace')
+               if isinstance(mymailport, unicode):
+                       mymailport = mymailport.encode('utf_8', 'replace')
+               if isinstance(myfrom, unicode):
+                       myfrom = myfrom.encode('utf_8', 'replace')
+               if isinstance(mymailuser, unicode):
+                       mymailuser = mymailuser.encode('utf_8', 'replace')
+               if isinstance(mymailpasswd, unicode):
+                       mymailpasswd = mymailpasswd.encode('utf_8', 'replace')
+
        # user wants to use a sendmail binary instead of smtp
        if mymailhost[0] == os.sep and os.path.exists(mymailhost):
                fd = os.popen(mymailhost+" -f "+myfrom+" "+myrecipient, "w")
index ab28b2e4aa8108fa644418702f70c02d3fa040b6..258fa756250fc9bad6e8b73dbbade5b9e531c191 100644 (file)
@@ -466,6 +466,13 @@ class EOutput(object):
                sys.stdout.flush()
                sys.stderr.flush()
 
+       def _write(self, f, s):
+               if sys.hexversion < 0x3000000 and isinstance(s, unicode):
+                       # avoid potential UnicodeEncodeError
+                       s = s.encode('utf_8', 'replace')
+               f.write(s)
+               f.flush()
+
        def __eend(self, caller, errno, msg):
                if errno == 0:
                        status_brackets = colorize("BRACKET", "[ ") + colorize("GOOD", "ok") + colorize("BRACKET", " ]")
@@ -480,9 +487,9 @@ class EOutput(object):
                        self.__last_e_len = 0
                if not self.quiet:
                        out = sys.stdout
-                       out.write("%*s%s\n" % ((self.term_columns - self.__last_e_len - 6),
+                       self._write(out,
+                               "%*s%s\n" % ((self.term_columns - self.__last_e_len - 6),
                                "", status_brackets))
-                       out.flush()
 
        def ebegin(self, msg):
                """
@@ -524,9 +531,8 @@ class EOutput(object):
                out = sys.stderr
                if not self.quiet:
                        if self.__last_e_cmd == "ebegin":
-                               out.write("\n")
-                       out.write(colorize("BAD", " * ") + msg + "\n")
-                       out.flush()
+                               self._write(out, "\n")
+                       self._write(out, colorize("BAD", " * ") + msg + "\n")
                self.__last_e_cmd = "eerror"
 
        def einfo(self, msg):
@@ -539,9 +545,8 @@ class EOutput(object):
                out = sys.stdout
                if not self.quiet:
                        if self.__last_e_cmd == "ebegin":
-                               out.write("\n")
-                       out.write(colorize("GOOD", " * ") + msg + "\n")
-                       out.flush()
+                               self._write(out, "\n")
+                       self._write(out, colorize("GOOD", " * ") + msg + "\n")
                self.__last_e_cmd = "einfo"
 
        def einfon(self, msg):
@@ -554,9 +559,8 @@ class EOutput(object):
                out = sys.stdout
                if not self.quiet:
                        if self.__last_e_cmd == "ebegin":
-                               out.write("\n")
-                       out.write(colorize("GOOD", " * ") + msg)
-                       out.flush()
+                               self._write(out, "\n")
+                       self._write(out, colorize("GOOD", " * ") + msg)
                self.__last_e_cmd = "einfon"
 
        def ewarn(self, msg):
@@ -569,9 +573,8 @@ class EOutput(object):
                out = sys.stderr
                if not self.quiet:
                        if self.__last_e_cmd == "ebegin":
-                               out.write("\n")
-                       out.write(colorize("WARN", " * ") + msg + "\n")
-                       out.flush()
+                               self._write(out, "\n")
+                       self._write(out, colorize("WARN", " * ") + msg + "\n")
                self.__last_e_cmd = "ewarn"
 
        def ewend(self, errno, *msg):
index 43a8fe02f722e145eaa3c5fe42cca3c53aee7511..25ccb03c75688fff2d4bb4cb6eae8d14583760b7 100644 (file)
@@ -35,7 +35,8 @@ class CommandOutputSet(PackageSet):
                pipe = subprocess.Popen(self._command, stdout=subprocess.PIPE, shell=True)
                stdout, stderr = pipe.communicate()
                if pipe.wait() == os.EX_OK:
-                       self._setAtoms(unicode(stdout, errors='replace').splitlines())
+                       self._setAtoms(unicode(stdout,
+                               encoding='utf_8', errors='replace').splitlines())
 
        def singleBuilder(self, options, settings, trees):
                if not "command" in options: