From 59490adcd003c0d3dafd820c67852d1beb560b2d Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 14 Aug 2009 20:05:47 +0000 Subject: [PATCH] Merged from trunk -r13939:13949 | 13940 | Fix filter_loglevels() and filter_phases() to work with | | zmedico | unicode. | | 13941 | Make getconfig() return unicode type and remove | | zmedico | corresponding env_update() workaround from bug #280460. | | 13942 | In shlex_split(), don't encode unicode for py3k. Thanks to | | zmedico | Arfrever. | | 13943 | In getconfig(), specify utf_8 encoding for py3k. | | zmedico | | | 13944 | Make spawn decode unicode in order to avoid a potential | | zmedico | UnicodeEncodeError from os.execve(). | | 13945 | Convert environment variables to unicode inside the config | | zmedico | constructor, in order to avoid potential UnicodeDecodeError | | | exceptions later. | | 13946 | In config.__setitem__(), convert keys/values to unicode in | | zmedico | order to avoid potential UnicodeDecodeError exceptions | | | later. | | 13947 | Open repo_name in text mode (unicode). | | zmedico | | | 13948 | Use writemsg_stdout() for safe unicode output. | | zmedico | | | 13949 | Account for hardlinks when computing SIZE metadata. | | zmedico | | svn path=/main/branches/prefix/; revision=14032 --- pym/_emerge/depgraph.py | 20 +++---------------- pym/portage/__init__.py | 37 ++++++++++++++++++++++------------- pym/portage/dbapi/porttree.py | 3 ++- pym/portage/elog/filtering.py | 4 ++-- pym/portage/process.py | 11 +++++++++++ pym/portage/util.py | 12 ++++++++++-- 6 files changed, 51 insertions(+), 36 deletions(-) diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 8a5fba520..b3a045db1 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -24,7 +24,7 @@ from portage.output import bold, blue, colorize, create_color_func, darkblue, \ bad = create_color_func("BAD") from portage.sets import SETPREFIX from portage.sets.base import InternalPackageSet -from portage.util import cmp_sort_key, writemsg +from portage.util import cmp_sort_key, writemsg, writemsg_stdout from _emerge.AtomArg import AtomArg from _emerge.Blocker import Blocker @@ -4356,16 +4356,7 @@ class depgraph(object): if "--changelog" in self._frozen_config.myopts: print for revision,text in changelogs: - - if sys.hexversion < 0x3000000: - # avoid potential UnicodeEncodeError - if isinstance(revision, unicode): - revision = revision.encode('utf_8', 'replace') - if isinstance(text, unicode): - text = text.encode('utf_8', 'replace') - - print bold('*'+revision) - sys.stdout.write(text) + writemsg_stdout(bold('*'+revision) + '\n' + text) sys.stdout.flush() return os.EX_OK @@ -5122,13 +5113,8 @@ def show_masked_packages(masked_packages): print "- "+cpv+" (masked by: "+", ".join(mreasons)+")" - if sys.hexversion < 0x3000000 and isinstance(comment, unicode): - # avoid potential UnicodeEncodeError - comment = comment.encode('utf_8', 'replace') - if comment and comment not in shown_comments: - print filename+":" - print comment + writemsg_stdout(filename + ":\n" + comment + "\n") shown_comments.add(comment) portdb = root_config.trees["porttree"].dbapi for l in missing_licenses: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 50a1a129e..967d7bec7 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -687,17 +687,6 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1) continue - # TODO: Make getconfig() return unicode. - unicode_config = {} - for k, v in myconfig.iteritems(): - if not isinstance(k, unicode): - k = unicode(k, encoding='utf8', errors='replace') - if not isinstance(v, unicode): - v = unicode(v, encoding='utf8', errors='replace') - unicode_config[k] = v - myconfig = unicode_config - del unicode_config - config_list.append(myconfig) if "SPACE_SEPARATED" in myconfig: space_separated.update(myconfig["SPACE_SEPARATED"].split()) @@ -1537,7 +1526,17 @@ class config(object): # backupenv is used for calculating incremental variables. if env is None: env = os.environ - self.backupenv = env.copy() + + # Avoid potential UnicodeDecodeError exceptions later. + env_unicode = {} + for k, v in env.iteritems(): + if not isinstance(k, unicode): + k = unicode(k, encoding='utf_8', errors='replace') + if not isinstance(v, unicode): + v = unicode(v, encoding='utf_8', errors='replace') + env_unicode[k] = v + + self.backupenv = env_unicode if env_d: # Remove duplicate values so they don't override updated @@ -3214,8 +3213,15 @@ class config(object): "set a value; will be thrown away at reset() time" if not isinstance(myvalue, basestring): raise ValueError("Invalid type being used as a value: '%s': '%s'" % (str(mykey),str(myvalue))) + + # Avoid potential UnicodeDecodeError exceptions later. + if not isinstance(mykey, unicode): + mykey = unicode(mykey, encoding='utf_8', errors='replace') + if not isinstance(myvalue, unicode): + myvalue = unicode(myvalue, encoding='utf_8', errors='replace') + self.modifying() - self.modifiedkeys += [mykey] + self.modifiedkeys.append(mykey) self.configdict["env"][mykey]=myvalue def environ(self): @@ -5192,12 +5198,15 @@ def _post_src_install_uid_fix(mysettings): destdir = destdir.encode('utf_8', 'replace') size = 0 + counted_inodes = set() for parent, dirs, files in os.walk(destdir): for fname in chain(dirs, files): fpath = os.path.join(parent, fname) mystat = os.lstat(fpath) - if stat.S_ISREG(mystat.st_mode): + if stat.S_ISREG(mystat.st_mode) and \ + mystat.st_ino not in counted_inodes: + counted_inodes.add(mystat.st_ino) size += mystat.st_size if mystat.st_uid != portage_uid and \ mystat.st_gid != portage_gid: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 9f427a148..9f36af569 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -167,7 +167,8 @@ class portdbapi(dbapi): continue repo_name_path = os.path.join(path, REPO_NAME_LOC) try: - repo_name = open(repo_name_path, 'r').readline().strip() + repo_name = codecs.open(repo_name_path, mode='r', + encoding='utf_8', errors='replace').readline().strip() except EnvironmentError: # warn about missing repo_name at some other time, since we # don't want to see a warning every time the portage module is diff --git a/pym/portage/elog/filtering.py b/pym/portage/elog/filtering.py index d33d31237..d4d405347 100644 --- a/pym/portage/elog/filtering.py +++ b/pym/portage/elog/filtering.py @@ -8,7 +8,7 @@ from portage.const import EBUILD_PHASES def filter_loglevels(logentries, loglevels): # remove unwanted entries from all logentries rValue = {} - loglevels = map(str.upper, loglevels) + loglevels = [x.upper() for x in loglevels] for phase in logentries: for msgtype, msgcontent in logentries[phase]: if msgtype.upper() in loglevels or "*" in loglevels: @@ -20,7 +20,7 @@ def filter_loglevels(logentries, loglevels): def filter_phases(logentries, phases): rValue1 = {} rValue2 = {} - phases = map(str.lower, phases) + phases = [x.lower() for x in phases] for phase in logentries: if phase in phases: rValue1[phase] = logentries[phase] diff --git a/pym/portage/process.py b/pym/portage/process.py index a88f5bf28..051a59f61 100644 --- a/pym/portage/process.py +++ b/pym/portage/process.py @@ -178,6 +178,17 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, if isinstance(mycommand, basestring): mycommand = mycommand.split() + # Avoid a potential UnicodeEncodeError from os.execve(). + env_bytes = {} + for k, v in env.iteritems(): + if isinstance(k, unicode): + k = k.encode('utf_8', 'replace') + if isinstance(v, unicode): + v = v.encode('utf_8', 'replace') + env_bytes[k] = v + env = env_bytes + del env_bytes + # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] diff --git a/pym/portage/util.py b/pym/portage/util.py index 32ac48884..513e4a79a 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -356,7 +356,7 @@ def shlex_split(s): This is equivalent to shlex.split but it temporarily encodes unicode strings to bytes since shlex.split() doesn't handle unicode strings. """ - is_unicode = isinstance(s, unicode) + is_unicode = sys.hexversion < 0x3000000 and isinstance(s, unicode) if is_unicode: s = s.encode('utf_8', 'replace') rval = shlex.split(s) @@ -388,7 +388,11 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): # trailing newline after the source statement # NOTE: shex doesn't seem to supported unicode objects # (produces spurious \0 characters with python-2.6.2) - content = open(mycfg).read() + if sys.hexversion < 0x3000000: + content = open(mycfg, 'rb').read() + else: + content = open(mycfg, mode='r', + encoding='utf_8', errors='replace').read() if content and content[-1] != '\n': content += '\n' except IOError, e: @@ -448,6 +452,10 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): raise portage.exception.CorruptionError("ParseError: Unexpected EOF: "+str(mycfg)+": line "+str(lex.lineno)) else: return mykeys + if not isinstance(key, unicode): + key = unicode(key, encoding='utf_8', errors='replace') + if not isinstance(val, unicode): + val = unicode(val, encoding='utf_8', errors='replace') if expand: mykeys[key] = varexpand(val, expand_map) expand_map[key] = mykeys[key] -- 2.26.2