From b6bce323b3bb5af48d8190803fa7e823a9079dca Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 5 Aug 2009 18:10:06 +0000 Subject: [PATCH] Merged from trunk -r13895:13909 | 13898 | Bug #280275 - Fix exception: AttributeError: 'config' object | | zmedico | has no attribute '_license_groups'. | | 13900 | Avoid UnicodeEncodeError in writemsg(). Thanks to Markos | | zmedico | Chandras for reporting. | | 13901 | Open log files in text mode (to avoid UnicodeEncodeError). | | zmedico | Thanks to Markos Chandras for | | | reporting. | | 13902 | Open CONTENTS in text mode (to avoid UnicodeEncodeError). | | zmedico | Thanks to Markos Chandras for | | | reporting. | | 13903 | Bug #280269 - Fix StaticFileSet.multiBuilder() to handle | | zmedico | unicode filenames. | | 13904 | Bug #280269 - Decode commandline arguments to unicode when | | zmedico | necessary. | | 13905 | Bug #280269 - Fix Atom.__str__ so that it doesn't try to | | zmedico | encode a unicode string (resulting in UnicodeEncodeError). | | | If an Atom instance is passed into the constructor, just | | | return the given instance. | | 13906 | Decode arguments to unicode if necessary (needed at least | | zmedico | for unicode file names passed to the owners command). | | 13907 | Always return unicode from vardbapi.aux_get(). | | zmedico | | | 13908 | Fix tar_contents() to open files in binary mode for py3k | | zmedico | compatibility. | | 13909 | Fix vardbapi._aux_get() to always return unicode. | | zmedico | | svn path=/main/branches/prefix/; revision=13922 --- bin/portageq | 9 +++++++-- pym/_emerge/Scheduler.py | 10 +++++++--- pym/_emerge/main.py | 4 ++++ pym/portage/__init__.py | 3 ++- pym/portage/dbapi/vartree.py | 24 +++++++++++++++++------- pym/portage/dep.py | 4 +++- pym/portage/sets/files.py | 3 +++ pym/portage/util.py | 3 +++ subst-install.in | 10 ++++++---- 9 files changed, 52 insertions(+), 18 deletions(-) diff --git a/bin/portageq b/bin/portageq index d154ce609..b8ff3e8aa 100755 --- a/bin/portageq +++ b/bin/portageq @@ -606,10 +606,15 @@ def main(): sys.path.insert(0, pym_path) import portage + args = sys.argv[2:] + if args and not isinstance(args[0], unicode): + for i in xrange(len(args)): + args[i] = unicode(args[i], encoding='utf_8', errors='replace') + try: if uses_root: - sys.argv[2] = portage.settings["ROOT"] - retval = function(sys.argv[2:]) + args[0] = portage.settings["ROOT"] + retval = function(args) if retval: sys.exit(retval) except portage.exception.PermissionDenied, e: diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index ed054b5b5..6016e9478 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import codecs import logging import os import sys @@ -478,7 +479,8 @@ class Scheduler(PollScheduler): return self._pkg(cpv, type_name, root_config, installed=installed) def _append_to_log_path(self, log_path, msg): - f = open(log_path, 'a') + f = codecs.open(log_path, mode='a', + encoding='utf_8', errors='replace') try: f.write(msg) finally: @@ -492,7 +494,8 @@ class Scheduler(PollScheduler): background = self._background if background and log_path is not None: - log_file = open(log_path, 'a') + log_file = codecs.open(log_path, mode='a', + encoding='utf_8', errors='replace') out = log_file try: @@ -867,7 +870,8 @@ class Scheduler(PollScheduler): log_path = self._locate_failure_log(failed_pkg) if log_path is not None: try: - log_file = open(log_path) + log_file = codecs.open(log_path, mode='r', + encoding='utf_8', errors='replace') except IOError: pass diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 0cb30da32..2113c1b6e 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -760,6 +760,10 @@ def parse_opts(tmpcmdline, silent=False): if myaction is None and myoptions.deselect is True: myaction = 'deselect' + if myargs and not isinstance(myargs[0], unicode): + for i in xrange(len(myargs)): + myargs[i] = unicode(myargs[i], encoding='utf_8', errors='replace') + myfiles += myargs return myaction, myopts, myfiles diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index ae17493ef..d884f8cb8 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1212,6 +1212,7 @@ class config(object): self._accept_chost_re = None self._accept_license = None self._accept_license_str = None + self._license_groups = {} self.virtuals = {} self.virts_p = {} @@ -1297,6 +1298,7 @@ class config(object): self._accept_license = copy.deepcopy(clone._accept_license) self._plicensedict = copy.deepcopy(clone._plicensedict) + self._license_groups = copy.deepcopy(clone._license_groups) else: def check_var_directory(varname, var): @@ -1793,7 +1795,6 @@ class config(object): self.pprovideddict[mycatpkg]=[x] # parse licensegroups - self._license_groups = {} for x in locations: self._license_groups.update( grabdict(os.path.join(x, "license_groups"))) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 5c60f5ab4..7f4d20397 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -38,6 +38,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ from portage.cache.mappings import slot_dict_class +import codecs import os, re, shutil, stat, errno, copy, subprocess import logging import shlex @@ -1866,6 +1867,12 @@ class vardbapi(dbapi): cache_mtime, metadata = pkg_data cache_valid = cache_mtime == mydir_mtime if cache_valid: + for k, v in metadata.iteritems(): + if not isinstance(v, unicode): + # Migrate old metadata to unicode. + metadata[k] = unicode(v, + encoding='utf_8', errors='replace') + mydata.update(metadata) pull_me.difference_update(mydata) @@ -1886,7 +1893,7 @@ class vardbapi(dbapi): if not mydata['SLOT']: # Empty slot triggers InvalidAtom exceptions when generating slot # atoms for packages, so translate it to '0' here. - mydata['SLOT'] = '0' + mydata['SLOT'] = u'0' return [mydata[x] for x in wants] def _aux_get(self, mycpv, wants, st=None): @@ -1909,7 +1916,8 @@ class vardbapi(dbapi): results.append(long(st.st_mtime)) continue try: - myf = open(os.path.join(mydir, x), "r") + myf = codecs.open(os.path.join(mydir, x), + mode='r', encoding='utf_8', errors='replace') try: myd = myf.read() finally: @@ -1919,9 +1927,9 @@ class vardbapi(dbapi): if self._aux_multi_line_re.match(x) is None: myd = " ".join(myd.split()) except IOError: - myd = "" + myd = u'' if x == "EAPI" and not myd: - results.append("0") + results.append(u'0') else: results.append(myd) return results @@ -2558,7 +2566,8 @@ class dblink(object): return self.contentscache pkgfiles = {} try: - myc = open(contents_file,"r") + myc = codecs.open(contents_file, mode='r', + encoding='utf_8', errors='replace') except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -4141,7 +4150,8 @@ class dblink(object): lcfile.close() # open CONTENTS file (possibly overwriting old one) for recording - outfile = open(os.path.join(self.dbtmpdir, "CONTENTS"),"w") + outfile = codecs.open(os.path.join(self.dbtmpdir, 'CONTENTS'), + mode='w', encoding='utf_8', errors='replace') self.updateprotect() @@ -4846,7 +4856,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tarinfo.size = 0 tar.addfile(tarinfo) else: - f = open(path) + f = open(path, 'rb') try: tar.addfile(tarinfo, f) finally: diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 64215b68c..f36f78256 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -494,6 +494,8 @@ class _AtomCache(type): identical instances when available. """ def __call__(cls, s): + if isinstance(s, Atom): + return s instance = cls._atoms.get(s) if instance is None: instance = super(_AtomCache, cls).__call__(s) @@ -620,7 +622,7 @@ class Atom(object): return repr(self._str) def __str__(self): - return str(self._str) + return self._str def endswith(self, *pargs, **kargs): return self._str.endswith(*pargs, **kargs) diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 95613ac47..3188025ef 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -131,6 +131,9 @@ class StaticFileSet(EditablePackageSet): if d[:1] == '.': dirs.remove(d) for filename in files: + if not isinstance(filename, unicode): + filename = unicode(filename, + encoding='utf_8', errors='replace') if filename[:1] == '.': continue if filename.endswith(".metadata"): diff --git a/pym/portage/util.py b/pym/portage/util.py index 376b6bac8..0cd40cc2b 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -57,6 +57,9 @@ def writemsg(mystr,noiselevel=0,fd=None): if fd is None: fd = sys.stderr if noiselevel <= noiselimit: + if sys.hexversion < 0x3000000 and isinstance(mystr, unicode): + # avoid potential UnicodeEncodeError + mystr = mystr.encode('utf_8', 'replace') fd.write(mystr) fd.flush() diff --git a/subst-install.in b/subst-install.in index 566555064..050fd9a17 100644 --- a/subst-install.in +++ b/subst-install.in @@ -4,15 +4,17 @@ prefix="@prefix@" exec_prefix="@exec_prefix@" -# for bug #279550 we have to do some nasty trick to make sure that sed +# For bug #279550 we have to do some nasty trick to make sure that sed # doesn't strip the backslash in the replacement value (because it can -# be a backreference) and hence escape those. +# be a backreference) and hence escape those. Eventually in strings we +# need to escape the backslash too, such that the single backslash +# doesn't get lost when considered an invalid escape rootuser="@rootuser@" portagegroup="@portagegroup@" portageuser="@portageuser@" rootuser=${rootuser//\\/\\\\} -portagegroup=${portagegroup//\\/\\\\} -portageuser=${portageuser//\\/\\\\} +portagegroup=${portagegroup//\\/\\\\\\\\} +portageuser=${portageuser//\\/\\\\\\\\} # there are many ways to do this all dynamic, but we only care for raw # speed here, so let configure fill in this list and be done with it -- 2.26.2