From 55052c65aeed37d2eb8a671c017f4205b8732c77 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 14 Aug 2009 20:53:24 +0000 Subject: [PATCH] Merged from trunk -r13998:14008 | 13999 | Update imports to import portage.os (with unicode wrappers), | | zmedico | and use _unicode_encode() and _unicode_decode() where | | | appropriate. | | 14000 | Filter ACCEPT_PROPERTIES from the ebuild environment, since | | zmedico | unlike ACCEPT_LICENSE (used by check_license in | | | eutils.eclass), ACCEPT_PROPERTIES should never be needed in | | | the ebuild environment. | | 14001 | Wrap portage.data.lchown() for unicode support in | | zmedico | python-2.x. | | 14002 | Update imports to import portage.os (with unicode wrappers), | | zmedico | and use _unicode_encode() and _unicode_decode() where | | | appropriate. | | 14003 | Update imports to import portage.os (with unicode wrappers), | | zmedico | and use _unicode_encode() and _unicode_decode() where | | | appropriate. | | 14004 | Update imports to import portage.os (with unicode wrappers), | | zmedico | and use _unicode_encode() and _unicode_decode() where | | | appropriate. | | 14005 | Show a traceback when spawn() fails. | | zmedico | | | 14006 | Implement ObjectProxy.__int__(). | | zmedico | | | 14007 | Update imports to import portage.os (with unicode wrappers), | | zmedico | and use _unicode_encode() and _unicode_decode() where | | | appropriate. | | 14008 | Use a single sys.hexversion conditional for both | | zmedico | _unicode_func_wrapper() and _unicode_module_wrapper(). | svn path=/main/branches/prefix/; revision=14044 --- pym/portage/__init__.py | 18 ++++++++---- pym/portage/cache/anydbm.py | 10 ++++--- pym/portage/cache/ebuild_xattr.py | 13 +++----- pym/portage/cache/flat_hash.py | 15 ++++++---- pym/portage/cache/flat_list.py | 15 +++++++--- pym/portage/cache/fs_template.py | 8 ++--- pym/portage/cache/metadata.py | 13 ++++---- pym/portage/cache/sqlite.py | 8 ++--- pym/portage/checksum.py | 8 ++--- pym/portage/data.py | 2 ++ pym/portage/dbapi/__init__.py | 2 +- pym/portage/dbapi/bintree.py | 34 ++++++++------------- pym/portage/dbapi/porttree.py | 17 ++++++----- pym/portage/dbapi/vartree.py | 49 +++++++++++++++++-------------- pym/portage/eclass_cache.py | 6 ++-- pym/portage/manifest.py | 31 +++++++++---------- pym/portage/process.py | 2 ++ pym/portage/proxy/objectproxy.py | 3 ++ pym/portage/util.py | 34 ++++++++++----------- 19 files changed, 154 insertions(+), 134 deletions(-) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 4a666cab0..b9418f022 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -61,11 +61,15 @@ try: 'portage.checksum', 'portage.checksum:perform_checksum,perform_md5,prelink_capable', 'portage.cvstree', + 'portage.data', + 'portage.data:lchown,ostype,portage_gid,portage_uid,secpass,' + \ + 'uid,userland,userpriv_groups,wheelgid', 'portage.dep', 'portage.dep:best_match_to_list,dep_getcpv,dep_getkey,' + \ 'get_operator,isjustname,isspecific,isvalidatom,' + \ 'match_from_list,match_to_list', 'portage.eclass_cache', + 'portage.exception', 'portage.getbinpkg', 'portage.locks', 'portage.locks:lockdir,lockfile,unlockdir,unlockfile', @@ -101,10 +105,6 @@ try: INCREMENTALS, EAPI, MISC_SH_BINARY, REPO_NAME_LOC, REPO_NAME_FILE, \ EPREFIX, EPREFIX_LSTRIP, BPREFIX, rootuid, rootgid - from portage.data import ostype, lchown, userland, secpass, uid, wheelgid, \ - portage_uid, portage_gid, userpriv_groups - from portage.manifest import Manifest - import portage.exception from portage.localization import _ except ImportError, e: @@ -179,6 +179,8 @@ class _unicode_module_wrapper(object): return result if sys.hexversion >= 0x3000000: + def _unicode_func_wrapper(func): + return func def _unicode_module_wrapper(mod): return mod @@ -187,6 +189,8 @@ os = _unicode_module_wrapper(os) import shutil shutil = _unicode_module_wrapper(shutil) +# Imports below this point rely on the above unicode wrapper definitions. + try: import portage._selinux as selinux except OSError, e: @@ -195,6 +199,8 @@ except OSError, e: except ImportError: pass +from portage.manifest import Manifest + # =========================================================================== # END OF IMPORTS -- END OF IMPORTS -- END OF IMPORTS -- END OF IMPORTS -- END # =========================================================================== @@ -1140,7 +1146,7 @@ class config(object): # environment in order to prevent sandbox from sourcing /etc/profile # in it's bashrc (causing major leakage). _environ_whitelist += [ - "ACCEPT_LICENSE", "ACCEPT_PROPERTIES", "BASH_ENV", "BUILD_PREFIX", "D", + "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "D", "DISTDIR", "DOC_SYMLINKS_DIR", "EBUILD", "EBUILD_EXIT_STATUS_FILE", "EBUILD_FORCE_TEST", "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "EMERGE_FROM", @@ -1217,7 +1223,7 @@ class config(object): # portage config variables and variables set directly by portage _environ_filter += [ - "ACCEPT_KEYWORDS", "AUTOCLEAN", + "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", diff --git a/pym/portage/cache/anydbm.py b/pym/portage/cache/anydbm.py index 9007a1a56..226c6d7d0 100644 --- a/pym/portage/cache/anydbm.py +++ b/pym/portage/cache/anydbm.py @@ -8,7 +8,8 @@ try: import cPickle as pickle except ImportError: import pickle -import os +from portage import _unicode_encode +from portage import os import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -30,8 +31,8 @@ class database(fs_template.FsBased): self._db_path = os.path.join(self.location, fs_template.gen_label(self.location, self.label)+default_db) self.__db = None try: - self.__db = anydbm_module.open(self._db_path, "w", self._perms) - + self.__db = anydbm_module.open( + _unicode_encode(self._db_path), 'w', self._perms) except anydbm_module.error: # XXX handle this at some point try: @@ -43,7 +44,8 @@ class database(fs_template.FsBased): # try again if failed try: if self.__db == None: - self.__db = anydbm_module.open(self._db_path, "c", self._perms) + self.__db = anydbm_module.open( + _unicode_encode(self._db_path), 'c', self._perms) except anydbm_module.error, e: raise cache_errors.InitializationError(self.__class__, e) self._ensure_access(self._db_path) diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py index 98e848651..baba94321 100644 --- a/pym/portage/cache/ebuild_xattr.py +++ b/pym/portage/cache/ebuild_xattr.py @@ -9,8 +9,8 @@ __all__ = ['database'] from portage.cache import fs_template from portage.versions import catsplit from portage import cpv_getkey -from portage.util import writemsg -import os +from portage import os +from portage import _unicode_decode import xattr from errno import ENODATA,ENOSPC,E2BIG @@ -154,14 +154,9 @@ class database(fs_template.FsBased): def __iter__(self): - portdir = self.portdir - if isinstance(portdir, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - portdir = portdir.encode('utf_8', 'replace') - - for root, dirs, files in os.walk(portdir): + for root, dirs, files in os.walk(self.portdir): for file in files: + file = _unicode_decode(file) if file[-7:] == '.ebuild': cat = os.path.basename(os.path.dirname(root)) pn_pv = file[:-7] diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py index c3c5955fb..2467f6800 100644 --- a/pym/portage/cache/flat_hash.py +++ b/pym/portage/cache/flat_hash.py @@ -6,9 +6,11 @@ import codecs from portage.cache import fs_template from portage.cache import cache_errors -import errno, os, stat -from portage.cache.template import reconstruct_eclasses -# store the current key order *here*. +import errno +import stat +from portage import os +from portage import _unicode_encode + class database(fs_template.FsBased): autocommits = True @@ -27,7 +29,8 @@ class database(fs_template.FsBased): def _getitem(self, cpv): fp = os.path.join(self.location, cpv) try: - myf = codecs.open(fp, mode='r', encoding='utf_8', errors='replace') + myf = codecs.open(_unicode_encode(fp), + mode='r', encoding='utf_8', errors='replace') try: d = self._parse_data(myf.readlines(), cpv) if '_mtime_' not in d: @@ -55,13 +58,13 @@ class database(fs_template.FsBased): s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: - myf = codecs.open(fp, mode='w', + myf = codecs.open(_unicode_encode(fp), mode='w', encoding='utf_8', errors='replace') except (IOError, OSError), e: if errno.ENOENT == e.errno: try: self._ensure_dirs(cpv) - myf = codecs.open(fp, mode='w', + myf = codecs.open(_unicode_encode(fp), mode='w', encoding='utf_8', errors='replace') except (OSError, IOError),e: raise cache_errors.CacheCorruption(cpv, e) diff --git a/pym/portage/cache/flat_list.py b/pym/portage/cache/flat_list.py index 8734e79b0..9906dd22d 100644 --- a/pym/portage/cache/flat_list.py +++ b/pym/portage/cache/flat_list.py @@ -1,6 +1,10 @@ from portage.cache import fs_template from portage.cache import cache_errors -import errno, os, stat +from portage import os +from portage import _unicode_encode +import codecs +import errno +import stat # store the current key order *here*. class database(fs_template.FsBased): @@ -27,7 +31,8 @@ class database(fs_template.FsBased): def _getitem(self, cpv): d = {} try: - myf = open(os.path.join(self._base, cpv),"r") + myf = codecs.open(_unicode_encode(os.path.join(self._base, cpv)), + mode='r', encoding='utf_8', errors='replace') for k,v in zip(self.auxdbkey_order, myf): d[k] = v.rstrip("\n") except (OSError, IOError),e: @@ -48,12 +53,14 @@ class database(fs_template.FsBased): s = cpv.rfind("/") fp=os.path.join(self._base,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: - myf = open(fp, "w") + myf = codecs.open(_unicode_encode(fp), mode='w', + encoding='utf_8', errors='replace') except (OSError, IOError), e: if errno.ENOENT == e.errno: try: self._ensure_dirs(cpv) - myf=open(fp,"w") + myf = codecs.open(_unicode_encode(fp), mode='w', + encoding='utf_8', errors='replace') except (OSError, IOError),e: raise cache_errors.CacheCorruption(cpv, e) else: diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py index d41749186..6d08f345f 100644 --- a/pym/portage/cache/fs_template.py +++ b/pym/portage/cache/fs_template.py @@ -3,16 +3,16 @@ # License: GPL2 # $Id$ -import os from portage.cache import template +from portage import os -import portage.proxy.lazyimport -import portage.proxy as proxy -proxy.lazyimport.lazyimport(globals(), +from portage.proxy.lazyimport import lazyimport +lazyimport(globals(), 'portage.data:portage_gid', 'portage.exception:PortageException', 'portage.util:apply_permissions', ) +del lazyimport class FsBased(template.database): """template wrapping fs needed options, and providing _ensure_access as a way to diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index b51bf980f..e91ed68d2 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -3,7 +3,10 @@ # License: GPL2 # $Id$ -import errno, os, re, sys +import errno +import re +from portage import os +from portage import _unicode_encode from portage.cache import cache_errors, flat_hash import portage.eclass_cache from portage.cache.template import reconstruct_eclasses @@ -82,11 +85,11 @@ class database(flat_hash.database): for i in xrange(magic_line_count - len(self.auxdbkey_order)): new_content.append(u'\n') new_content = u''.join(new_content) - new_content = new_content.encode('utf_8', 'replace') + new_content = _unicode_encode(new_content) new_fp = os.path.join(self.location, cpv) try: - f = open(new_fp, 'rb') + f = open(_unicode_encode(new_fp), 'rb') except EnvironmentError: pass else: @@ -114,12 +117,12 @@ class database(flat_hash.database): fp = os.path.join(self.location,cpv[:s], ".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: - myf = open(fp, 'wb') + myf = open(_unicode_encode(fp), 'wb') except EnvironmentError, e: if errno.ENOENT == e.errno: try: self._ensure_dirs(cpv) - myf = open(fp, 'wb') + myf = open(_unicode_encode(fp), 'wb') except EnvironmentError, e: raise cache_errors.CacheCorruption(cpv, e) else: diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index a0b11165f..0a8424880 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -4,10 +4,10 @@ from portage.cache import fs_template from portage.cache import cache_errors -import os +from portage import os +from portage import _unicode_encode from portage.cache.template import reconstruct_eclasses -from portage.util import writemsg, apply_secpass_permissions -from portage.data import portage_gid +from portage.util import writemsg from portage.localization import _ try: import sqlite3 as db_module # sqlite3 is optional with >=python-2.5 @@ -59,7 +59,7 @@ class database(fs_template.FsBased): try: self._ensure_dirs() self._db_connection = self._db_module.connect( - database=self._dbpath, **connection_kwargs) + database=_unicode_encode(self._dbpath), **connection_kwargs) self._db_cursor = self._db_connection.cursor() self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) if not self._ensure_access(self._dbpath): diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 56719a1df..501ae3e6a 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -3,13 +3,13 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import portage from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE -import os +from portage import os +from portage import _unicode_encode import errno import stat import tempfile -import portage.exception -import portage.process import commands #dict of all available hash functions @@ -25,7 +25,7 @@ def _generate_hash_function(hashtype, hashobject, origin="unknown"): @type filename: String @return: The hash and size of the data """ - f = open(filename, 'rb') + f = open(_unicode_encode(filename), 'rb') blocksize = HASHING_BLOCKSIZE data = f.read(blocksize) size = 0L diff --git a/pym/portage/data.py b/pym/portage/data.py index 4dc902aef..e4011df44 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -39,6 +39,8 @@ if not lchown: " exist. Please rebuild python.\n", noiselevel=-1) lchown() +lchown = portage._unicode_func_wrapper(lchown) + def portage_group_warning(): warn_prefix = colorize("BAD", "*** WARNING *** ") mylines = [ diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 08b7a3b9b..ebcd785a0 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -4,7 +4,6 @@ __all__ = ["dbapi"] -import os import re import portage @@ -16,6 +15,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.versions:catpkgsplit,vercmp', ) +from portage import os from portage import auxdbkeys, dep_expand from portage.localization import _ diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 313a41598..0ba85867a 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -21,11 +21,14 @@ from portage.const import EAPI from portage.localization import _ from portage import dep_expand, listdir, _check_distfile, _movefile +from portage import os +from portage import _unicode_decode +from portage import _unicode_encode import codecs -import os, errno, stat +import errno import re -import sys +import stat from itertools import chain, izip class bindbapi(fakedbapi): @@ -72,8 +75,8 @@ class bindbapi(fakedbapi): tbz2 = portage.xpak.tbz2(tbz2_path) def getitem(k): v = tbz2.getfile(k) - if v is not None and not isinstance(v, unicode): - v = unicode(v, encoding='utf_8', errors='replace') + if v is not None: + v = _unicode_decode(v) return v else: getitem = self.bintree._remotepkgs[mycpv].get @@ -107,21 +110,10 @@ class bindbapi(fakedbapi): mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() - if sys.hexversion < 0x3000000: - for k, v in values.iteritems(): - if isinstance(k, unicode): - k = k.encode('utf_8', 'replace') - if isinstance(v, unicode): - v = v.encode('utf_8', 'replace') - mydata[k] = v - - else: - for k, v in values.iteritems(): - if isinstance(k, str): - k = k.encode('utf_8', 'replace') - if isinstance(v, str): - v = v.encode('utf_8', 'replace') - mydata[k] = v + for k, v in values.iteritems(): + k = _unicode_encode(k) + v = _unicode_encode(v) + mydata[k] = v for k, v in mydata.items(): if not v: @@ -663,7 +655,7 @@ class binarytree(object): urldata[1] + urldata[2], "Packages") pkgindex = self._new_pkgindex() try: - f = codecs.open(pkgindex_file, + f = codecs.open(_unicode_encode(pkgindex_file), encoding='utf_8', errors='replace') try: pkgindex.read(f) @@ -1105,7 +1097,7 @@ class binarytree(object): def _load_pkgindex(self): pkgindex = self._new_pkgindex() try: - f = codecs.open(self._pkgindex_file, + f = codecs.open(_unicode_encode(self._pkgindex_file), encoding='utf_8', errors='replace') except EnvironmentError: pass diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 7ee889a7c..56ed44945 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -26,8 +26,12 @@ from portage.manifest import Manifest from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \ _eapi_is_deprecated +from portage import _unicode_encode +from portage import os -import codecs, logging, os, stat +import codecs +import logging +import stat from itertools import izip def _src_uri_validate(cpv, eapi, src_uri): @@ -167,7 +171,8 @@ class portdbapi(dbapi): continue repo_name_path = os.path.join(path, REPO_NAME_LOC) try: - repo_name = codecs.open(repo_name_path, mode='r', + repo_name = codecs.open( + _unicode_encode(repo_name_path), mode='r', encoding='utf_8', errors='replace').readline().strip() except EnvironmentError: # warn about missing repo_name at some other time, since we @@ -615,8 +620,9 @@ class portdbapi(dbapi): os.path.basename(myebuild)) if eapi is None and \ 'parse-eapi-ebuild-head' in self.doebuild_settings.features: - eapi = portage._parse_eapi_ebuild_head(codecs.open(myebuild, - mode='r', encoding='utf_8', errors='replace')) + eapi = portage._parse_eapi_ebuild_head(codecs.open( + _unicode_encode(myebuild), mode='r', + encoding='utf_8', errors='replace')) if eapi is not None: self.doebuild_settings.configdict['pkg']['EAPI'] = eapi @@ -878,9 +884,6 @@ class portdbapi(dbapi): except OSError: continue for x in file_list: - if not isinstance(x, unicode): - x = unicode(x, encoding='utf_8', errors='replace') - pf = None if glep55: pf, eapi = portage._split_ebuild_name_glep55(x) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 46afe9bda..b2419af69 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -38,6 +38,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ # This is a special version of the os module, wrapped for unicode support. from portage import os +from portage import _unicode_encode from portage.cache.mappings import slot_dict_class @@ -73,7 +74,8 @@ class PreservedLibsRegistry(object): """ Reload the registry data from file """ self._data = None try: - self._data = pickle.load(open(self._filename, 'rb')) + self._data = pickle.load( + open(_unicode_encode(self._filename), 'rb')) except (ValueError, pickle.UnpicklingError), e: writemsg_level(_("!!! Error loading '%s': %s\n") % \ (self._filename, e), level=logging.ERROR, noiselevel=-1) @@ -1777,7 +1779,8 @@ class vardbapi(dbapi): # python-2.x, but buffering makes it much worse. open_kwargs["buffering"] = 0 try: - f = open(self._aux_cache_filename, mode='rb', **open_kwargs) + f = open(_unicode_encode(self._aux_cache_filename), + mode='rb', **open_kwargs) mypickle = pickle.Unpickler(f) try: mypickle.find_global = None @@ -1923,7 +1926,7 @@ class vardbapi(dbapi): results.append(long(st.st_mtime)) continue try: - myf = codecs.open(os.path.join(mydir, x), + myf = codecs.open(_unicode_encode(os.path.join(mydir, x)), mode='r', encoding='utf_8', errors='replace') try: myd = myf.read() @@ -1992,7 +1995,8 @@ class vardbapi(dbapi): new_vdb = False counter = -1 try: - cfile = open(self._counter_path, "r") + cfile = codecs.open(_unicode_encode(self._counter_path), mode='r', + encoding='utf_8', errors='replace') except EnvironmentError, e: new_vdb = not bool(self.cpv_all()) if not new_vdb: @@ -2575,7 +2579,7 @@ class dblink(object): return self.contentscache pkgfiles = {} try: - myc = codecs.open(contents_file, mode='r', + myc = codecs.open(_unicode_encode(contents_file), mode='r', encoding='utf_8', errors='replace') except EnvironmentError, e: if e.errno != errno.ENOENT: @@ -3877,7 +3881,9 @@ class dblink(object): slot = '' for var_name in ('CHOST', 'SLOT'): try: - val = open(os.path.join(inforoot, var_name)).readline().strip() + val = codecs.open(_unicode_encode( + os.path.join(inforoot, var_name)), mode='r', + encoding='utf_8', errors='replace').readline().strip() except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -4167,12 +4173,12 @@ class dblink(object): # write local package counter for recording counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) - lcfile = open(os.path.join(self.dbtmpdir, "COUNTER"),"w") - lcfile.write(str(counter)) - lcfile.close() + open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER')), + 'w').write(str(counter)) # open CONTENTS file (possibly overwriting old one) for recording - outfile = codecs.open(os.path.join(self.dbtmpdir, 'CONTENTS'), + outfile = codecs.open(_unicode_encode( + os.path.join(self.dbtmpdir, 'CONTENTS')), mode='w', encoding='utf_8', errors='replace') self.updateprotect() @@ -4767,9 +4773,8 @@ class dblink(object): "returns contents of a file with whitespace converted to spaces" if not os.path.exists(self.dbdir+"/"+name): return "" - myfile = open(self.dbdir+"/"+name,"r") - mydata = myfile.read().split() - myfile.close() + mydata = codecs.open(_unicode_encode(os.path.join(self.dbdir, name)), + mode='r', encoding='utf_8', errors='replace').read().split() return " ".join(mydata) def copyfile(self,fname): @@ -4778,10 +4783,8 @@ class dblink(object): def getfile(self,fname): if not os.path.exists(self.dbdir+"/"+fname): return "" - myfile = open(self.dbdir+"/"+fname,"r") - mydata = myfile.read() - myfile.close() - return mydata + return codecs.open(_unicode_encode(os.path.join(self.dbdir, fname)), + mode='r', encoding='utf_8', errors='replace').read() def setfile(self,fname,data): mode = 'w' @@ -4792,17 +4795,19 @@ class dblink(object): def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] - myelement = open(self.dbdir+"/"+ename,"r") - mylines = myelement.readlines() + mylines = codecs.open(_unicode_encode( + os.path.join(self.dbdir, ename)), mode='r', + encoding='utf_8', errors='replace').readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): myreturn.append(y) - myelement.close() return myreturn def setelements(self,mylist,ename): - myelement = open(self.dbdir+"/"+ename,"w") + myelement = codecs.open(_unicode_encode( + os.path.join(self.dbdir, ename)), mode='w', + encoding='utf_8', errors='replace') for x in mylist: myelement.write(x+"\n") myelement.close() @@ -4879,7 +4884,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tarinfo.size = 0 tar.addfile(tarinfo) else: - f = open(path, 'rb') + f = open(_unicode_encode(path), 'rb') try: tar.addfile(tarinfo, f) finally: diff --git a/pym/portage/eclass_cache.py b/pym/portage/eclass_cache.py index 8f80ec53e..3f86a6d69 100644 --- a/pym/portage/eclass_cache.py +++ b/pym/portage/eclass_cache.py @@ -6,10 +6,10 @@ __all__ = ["cache"] import warnings -from portage.util import normalize_path, writemsg -import errno, os, sys -from portage.data import portage_gid +from portage.util import normalize_path +import errno from portage.exception import PermissionDenied +from portage import os class cache(object): """ diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index a717d5841..e411be8c1 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -2,7 +2,8 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -import errno, os +import codecs +import errno import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -10,7 +11,12 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:write_atomic', ) -from portage.exception import * +from portage import os +from portage import _unicode_decode +from portage import _unicode_encode +from portage.exception import DigestException, FileNotFound, \ + InvalidDataType, MissingParameter, PermissionDenied, \ + PortageException, PortagePackageException class FileNotInManifestException(PortageException): pass @@ -93,7 +99,7 @@ class Manifest(object): Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of a Manifest (not needed for parsing and checking sums).""" - self.pkgdir = pkgdir.rstrip(os.sep) + os.sep + self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep self.fhashdict = {} self.hashes = set() self.hashes.update(portage.const.MANIFEST2_HASH_FUNCTIONS) @@ -135,7 +141,8 @@ class Manifest(object): """Parse a manifest. If myhashdict is given then data will be added too it. Otherwise, a new dict will be created and returned.""" try: - fd = open(file_path, "r") + fd = codecs.open(_unicode_encode(file_path), mode='r', + encoding='utf_8', errors='replace') if myhashdict is None: myhashdict = {} self._parseDigests(fd, myhashdict=myhashdict, **kwargs) @@ -221,7 +228,8 @@ class Manifest(object): update_manifest = True if not force: try: - f = open(self.getFullname(), "r") + f = codecs.open(_unicode_encode(self.getFullname()), + mode='r', encoding='utf_8', errors='replace') oldentries = list(self._parseManifestLines(f)) f.close() if len(oldentries) == len(myentries): @@ -308,14 +316,11 @@ class Manifest(object): cat = self._pkgdir_category() pkgdir = self.pkgdir - if isinstance(pkgdir, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - pkgdir = pkgdir.encode('utf_8', 'replace') for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir): break for f in pkgdir_files: + f = _unicode_decode(f) if f[:1] == ".": continue pf = None @@ -343,11 +348,6 @@ class Manifest(object): recursive_files = [] pkgdir = self.pkgdir - if isinstance(pkgdir, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - pkgdir = pkgdir.encode('utf_8', 'replace') - cut_len = len(os.path.join(pkgdir, "files") + os.sep) for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")): for f in files: @@ -508,7 +508,8 @@ class Manifest(object): mfname = self.getFullname() if not os.path.exists(mfname): return rVal - myfile = open(mfname, "r") + myfile = codecs.open(_unicode_encode(mfname), + mode='r', encoding='utf_8', errors='replace') lines = myfile.readlines() myfile.close() for l in lines: diff --git a/pym/portage/process.py b/pym/portage/process.py index 051a59f61..5dcf481e2 100644 --- a/pym/portage/process.py +++ b/pym/portage/process.py @@ -8,6 +8,7 @@ import os import atexit import signal import sys +import traceback import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -244,6 +245,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, # propogate out of this function and cause exiting # with anything other than os._exit() sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) + traceback.print_exc() sys.stderr.flush() os._exit(1) diff --git a/pym/portage/proxy/objectproxy.py b/pym/portage/proxy/objectproxy.py index 00d3f6157..00c5076d1 100644 --- a/pym/portage/proxy/objectproxy.py +++ b/pym/portage/proxy/objectproxy.py @@ -64,3 +64,6 @@ class ObjectProxy(object): def __nonzero__(self): return bool(object.__getattribute__(self, '_get_target')()) + + def __int__(self): + return int(object.__getattribute__(self, '_get_target')()) diff --git a/pym/portage/util.py b/pym/portage/util.py index 513e4a79a..3d6e8e6b9 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -22,9 +22,12 @@ import stat import string import sys +import portage +from portage import os +from portage import _unicode_encode +from portage import _unicode_decode from portage.exception import PortageException, FileNotFound, \ OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem -import portage.exception from portage.dep import isvalidatom from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict @@ -57,9 +60,9 @@ def writemsg(mystr,noiselevel=0,fd=None): if fd is None: fd = sys.stderr if noiselevel <= noiselimit: - if sys.hexversion < 0x3000000 and isinstance(mystr, unicode): + if sys.hexversion < 0x3000000: # avoid potential UnicodeEncodeError - mystr = mystr.encode('utf_8', 'replace') + mystr = _unicode_encode(mystr) fd.write(mystr) fd.flush() @@ -322,8 +325,8 @@ def grablines(myfilename,recursive=0): os.path.join(myfilename, f), recursive)) else: try: - myfile = codecs.open(myfilename, mode='r', - encoding='utf_8', errors='replace') + myfile = codecs.open(_unicode_encode(myfilename), + mode='r', encoding='utf_8', errors='replace') mylines = myfile.readlines() myfile.close() except IOError, e: @@ -358,10 +361,10 @@ def shlex_split(s): """ is_unicode = sys.hexversion < 0x3000000 and isinstance(s, unicode) if is_unicode: - s = s.encode('utf_8', 'replace') + s = _unicode_encode(s) rval = shlex.split(s) if is_unicode: - rval = [unicode(x, encoding='utf_8', errors='replace') for x in rval] + rval = [_unicode_decode(x) for x in rval] return rval class _tolerant_shlex(shlex.shlex): @@ -389,9 +392,9 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): # NOTE: shex doesn't seem to supported unicode objects # (produces spurious \0 characters with python-2.6.2) if sys.hexversion < 0x3000000: - content = open(mycfg, 'rb').read() + content = open(_unicode_encode(mycfg), 'rb').read() else: - content = open(mycfg, mode='r', + content = open(_unicode_encode(mycfg), mode='r', encoding='utf_8', errors='replace').read() if content and content[-1] != '\n': content += '\n' @@ -452,10 +455,8 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): raise portage.exception.CorruptionError("ParseError: Unexpected EOF: "+str(mycfg)+": line "+str(lex.lineno)) else: return mykeys - if not isinstance(key, unicode): - key = unicode(key, encoding='utf_8', errors='replace') - if not isinstance(val, unicode): - val = unicode(val, encoding='utf_8', errors='replace') + key = _unicode_decode(key) + val = _unicode_decode(val) if expand: mykeys[key] = varexpand(val, expand_map) expand_map[key] = mykeys[key] @@ -586,7 +587,7 @@ def pickle_read(filename,default=None,debug=0): return default data = None try: - myf = open(filename, 'rb') + myf = open(_unicode_encode(filename), 'rb') mypickle = pickle.Unpickler(myf) data = mypickle.load() myf.close() @@ -802,11 +803,6 @@ def apply_recursive_permissions(top, uid=-1, gid=-1, Returns True if all permissions are applied and False if some are left unapplied.""" - if isinstance(top, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - top = top.encode('utf_8', 'replace') - if onerror is None: # Default behavior is to dump errors to stderr so they won't # go unnoticed. Callers can pass in a quiet instance. -- 2.26.2