Merged from trunk -r13939:13949
authorFabian Groffen <grobian@gentoo.org>
Fri, 14 Aug 2009 20:05:47 +0000 (20:05 -0000)
committerFabian Groffen <grobian@gentoo.org>
Fri, 14 Aug 2009 20:05:47 +0000 (20:05 -0000)
   | 13940   | Fix filter_loglevels() and filter_phases() to work with      |
   | zmedico | unicode.                                                     |

   | 13941   | Make getconfig() return unicode type and remove              |
   | zmedico | corresponding env_update() workaround from bug #280460.      |

   | 13942   | In shlex_split(), don't encode unicode for py3k. Thanks to   |
   | zmedico | Arfrever.                                                    |

   | 13943   | In getconfig(), specify utf_8 encoding for py3k.             |
   | zmedico |                                                              |

   | 13944   | Make spawn decode unicode in order to avoid a potential      |
   | zmedico | UnicodeEncodeError from os.execve().                         |

   | 13945   | Convert environment variables to unicode inside the config   |
   | zmedico | constructor, in order to avoid potential UnicodeDecodeError  |
   |         | exceptions later.                                            |

   | 13946   | In config.__setitem__(), convert keys/values to unicode in   |
   | zmedico | order to avoid potential UnicodeDecodeError exceptions       |
   |         | later.                                                       |

   | 13947   | Open repo_name in text mode (unicode).                       |
   | zmedico |                                                              |

   | 13948   | Use writemsg_stdout() for safe unicode output.               |
   | zmedico |                                                              |

   | 13949   | Account for hardlinks when computing SIZE metadata.          |
   | zmedico |                                                              |

svn path=/main/branches/prefix/; revision=14032

pym/_emerge/depgraph.py
pym/portage/__init__.py
pym/portage/dbapi/porttree.py
pym/portage/elog/filtering.py
pym/portage/process.py
pym/portage/util.py

index 8a5fba520f866fe41430b0ba790adbd60789df63..b3a045db18ef6b2ddbaf97ce1cd6521287aa01d5 100644 (file)
@@ -24,7 +24,7 @@ from portage.output import bold, blue, colorize, create_color_func, darkblue, \
 bad = create_color_func("BAD")
 from portage.sets import SETPREFIX
 from portage.sets.base import InternalPackageSet
-from portage.util import cmp_sort_key, writemsg
+from portage.util import cmp_sort_key, writemsg, writemsg_stdout
 
 from _emerge.AtomArg import AtomArg
 from _emerge.Blocker import Blocker
@@ -4356,16 +4356,7 @@ class depgraph(object):
                if "--changelog" in self._frozen_config.myopts:
                        print
                        for revision,text in changelogs:
-
-                               if sys.hexversion < 0x3000000:
-                                       # avoid potential UnicodeEncodeError
-                                       if isinstance(revision, unicode):
-                                               revision = revision.encode('utf_8', 'replace')
-                                       if isinstance(text, unicode):
-                                               text = text.encode('utf_8', 'replace')
-
-                               print bold('*'+revision)
-                               sys.stdout.write(text)
+                               writemsg_stdout(bold('*'+revision) + '\n' + text)
 
                sys.stdout.flush()
                return os.EX_OK
@@ -5122,13 +5113,8 @@ def show_masked_packages(masked_packages):
 
                print "- "+cpv+" (masked by: "+", ".join(mreasons)+")"
 
-               if sys.hexversion < 0x3000000 and isinstance(comment, unicode):
-                       # avoid potential UnicodeEncodeError
-                       comment = comment.encode('utf_8', 'replace')
-
                if comment and comment not in shown_comments:
-                       print filename+":"
-                       print comment
+                       writemsg_stdout(filename + ":\n" + comment + "\n")
                        shown_comments.add(comment)
                portdb = root_config.trees["porttree"].dbapi
                for l in missing_licenses:
index 50a1a129e4805e9272848a8cff1b692f7cb65cc1..967d7bec7bd231cdc749e824f94a06f54bd8d1b4 100644 (file)
@@ -687,17 +687,6 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None,
                        writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1)
                        continue
 
-               # TODO: Make getconfig() return unicode.
-               unicode_config = {}
-               for k, v in myconfig.iteritems():
-                       if not isinstance(k, unicode):
-                               k = unicode(k, encoding='utf8', errors='replace')
-                       if not isinstance(v, unicode):
-                               v = unicode(v, encoding='utf8', errors='replace')
-                       unicode_config[k] = v
-               myconfig = unicode_config
-               del unicode_config
-
                config_list.append(myconfig)
                if "SPACE_SEPARATED" in myconfig:
                        space_separated.update(myconfig["SPACE_SEPARATED"].split())
@@ -1537,7 +1526,17 @@ class config(object):
                        # backupenv is used for calculating incremental variables.
                        if env is None:
                                env = os.environ
-                       self.backupenv = env.copy()
+
+                       # Avoid potential UnicodeDecodeError exceptions later.
+                       env_unicode = {}
+                       for k, v in env.iteritems():
+                               if not isinstance(k, unicode):
+                                       k = unicode(k, encoding='utf_8', errors='replace')
+                               if not isinstance(v, unicode):
+                                       v = unicode(v, encoding='utf_8', errors='replace')
+                               env_unicode[k] = v
+
+                       self.backupenv = env_unicode
 
                        if env_d:
                                # Remove duplicate values so they don't override updated
@@ -3214,8 +3213,15 @@ class config(object):
                "set a value; will be thrown away at reset() time"
                if not isinstance(myvalue, basestring):
                        raise ValueError("Invalid type being used as a value: '%s': '%s'" % (str(mykey),str(myvalue)))
+
+               # Avoid potential UnicodeDecodeError exceptions later.
+               if not isinstance(mykey, unicode):
+                       mykey = unicode(mykey, encoding='utf_8', errors='replace')
+               if not isinstance(myvalue, unicode):
+                       myvalue = unicode(myvalue, encoding='utf_8', errors='replace')
+
                self.modifying()
-               self.modifiedkeys += [mykey]
+               self.modifiedkeys.append(mykey)
                self.configdict["env"][mykey]=myvalue
 
        def environ(self):
@@ -5192,12 +5198,15 @@ def _post_src_install_uid_fix(mysettings):
                destdir = destdir.encode('utf_8', 'replace')
 
        size = 0
+       counted_inodes = set()
 
        for parent, dirs, files in os.walk(destdir):
                for fname in chain(dirs, files):
                        fpath = os.path.join(parent, fname)
                        mystat = os.lstat(fpath)
-                       if stat.S_ISREG(mystat.st_mode):
+                       if stat.S_ISREG(mystat.st_mode) and \
+                               mystat.st_ino not in counted_inodes:
+                               counted_inodes.add(mystat.st_ino)
                                size += mystat.st_size
                        if mystat.st_uid != portage_uid and \
                                mystat.st_gid != portage_gid:
index 9f427a14825c8d3df19f44e8fcc3c246931e964e..9f36af5699b9cae9d589e6d880626e0102bf429a 100644 (file)
@@ -167,7 +167,8 @@ class portdbapi(dbapi):
                                continue
                        repo_name_path = os.path.join(path, REPO_NAME_LOC)
                        try:
-                               repo_name = open(repo_name_path, 'r').readline().strip()
+                               repo_name = codecs.open(repo_name_path, mode='r',
+                                       encoding='utf_8', errors='replace').readline().strip()
                        except EnvironmentError:
                                # warn about missing repo_name at some other time, since we
                                # don't want to see a warning every time the portage module is
index d33d312374d53a4eb4cc8426a36cc397374d0621..d4d405347d9eaa2c5db72729a69ffdab6aaff44f 100644 (file)
@@ -8,7 +8,7 @@ from portage.const import EBUILD_PHASES
 def filter_loglevels(logentries, loglevels):
        # remove unwanted entries from all logentries
        rValue = {}
-       loglevels = map(str.upper, loglevels)
+       loglevels = [x.upper() for x in loglevels]
        for phase in logentries:
                for msgtype, msgcontent in logentries[phase]:
                        if msgtype.upper() in loglevels or "*" in loglevels:
@@ -20,7 +20,7 @@ def filter_loglevels(logentries, loglevels):
 def filter_phases(logentries, phases):
        rValue1 = {}
        rValue2 = {}
-       phases = map(str.lower, phases)
+       phases = [x.lower() for x in phases]
        for phase in logentries:
                if phase in phases:
                        rValue1[phase] = logentries[phase]
index a88f5bf28f24699f968a815d556b3c47a22843ea..051a59f618fb1c6d3bb948c2148331d0b7fb0033 100644 (file)
@@ -178,6 +178,17 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
        if isinstance(mycommand, basestring):
                mycommand = mycommand.split()
 
+       # Avoid a potential UnicodeEncodeError from os.execve().
+       env_bytes = {}
+       for k, v in env.iteritems():
+               if isinstance(k, unicode):
+                       k = k.encode('utf_8', 'replace')
+               if isinstance(v, unicode):
+                       v = v.encode('utf_8', 'replace')
+               env_bytes[k] = v
+       env = env_bytes
+       del env_bytes
+
        # If an absolute path to an executable file isn't given
        # search for it unless we've been told not to.
        binary = mycommand[0]
index 32ac48884ebea22800de37c41c67682c11e2eb18..513e4a79a24dfeea23437e14896a8781e22bf261 100644 (file)
@@ -356,7 +356,7 @@ def shlex_split(s):
        This is equivalent to shlex.split but it temporarily encodes unicode
        strings to bytes since shlex.split() doesn't handle unicode strings.
        """
-       is_unicode = isinstance(s, unicode)
+       is_unicode = sys.hexversion < 0x3000000 and isinstance(s, unicode)
        if is_unicode:
                s = s.encode('utf_8', 'replace')
        rval = shlex.split(s)
@@ -388,7 +388,11 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True):
                # trailing newline after the source statement
                # NOTE: shex doesn't seem to supported unicode objects
                # (produces spurious \0 characters with python-2.6.2)
-               content = open(mycfg).read()
+               if sys.hexversion < 0x3000000:
+                       content = open(mycfg, 'rb').read()
+               else:
+                       content = open(mycfg, mode='r',
+                               encoding='utf_8', errors='replace').read()
                if content and content[-1] != '\n':
                        content += '\n'
        except IOError, e:
@@ -448,6 +452,10 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True):
                                        raise portage.exception.CorruptionError("ParseError: Unexpected EOF: "+str(mycfg)+": line "+str(lex.lineno))
                                else:
                                        return mykeys
+                       if not isinstance(key, unicode):
+                               key = unicode(key, encoding='utf_8', errors='replace')
+                       if not isinstance(val, unicode):
+                               val = unicode(val, encoding='utf_8', errors='replace')
                        if expand:
                                mykeys[key] = varexpand(val, expand_map)
                                expand_map[key] = mykeys[key]