sys.stderr.write(" "+str(e)+"\n\n")
raise
-def _unicode_encode(s):
+def _unicode_encode(s, encoding='utf_8', errors='replace'):
if isinstance(s, unicode):
- s = s.encode('utf_8', 'replace')
+ s = s.encode(encoding, errors)
return s
-def _unicode_decode(s):
- if not isinstance(s, unicode) and isinstance(s, basestring):
- s = unicode(s, encoding='utf_8', errors='replace')
+def _unicode_decode(s, encoding='utf_8', errors='replace'):
+ if not isinstance(s, unicode):
+ if sys.hexversion < 0x3000000:
+ if isinstance(s, basestring):
+ s = unicode(s, encoding=encoding, errors=errors)
+ else:
+ if isinstance(s, bytes):
+ s = unicode(s, encoding=encoding, errors=errors)
return s
class _unicode_func_wrapper(object):
Wraps a function, converts arguments from unicode to bytes,
and return values to unicode from bytes.
"""
- __slots__ = ('_func',)
+ __slots__ = ('_func', '_encoding')
- def __init__(self, func):
+ def __init__(self, func, encoding='utf_8'):
self._func = func
+ self._encoding = encoding
def __call__(self, *args, **kwargs):
+ encoding = self._encoding
wrapped_args = [_unicode_encode(x) for x in args]
if kwargs:
- wrapped_kwargs = dict((_unicode_encode(k), _unicode_encode(v)) \
+ wrapped_kwargs = dict((_unicode_encode(k, encoding=encoding),
+ _unicode_encode(v, encoding=encoding)) \
for k, v in kwargs.iteritems())
else:
wrapped_kwargs = {}
if isinstance(rval, (basestring, list, tuple)):
if isinstance(rval, basestring):
- rval = _unicode_decode(rval)
+ rval = _unicode_decode(rval, encoding=encoding)
elif isinstance(rval, list):
- rval = [_unicode_decode(x) for x in rval]
+ rval = [_unicode_decode(x, encoding=encoding) for x in rval]
elif isinstance(rval, tuple):
- rval = tuple(_unicode_decode(x) for x in rval)
+ rval = tuple(_unicode_decode(x, encoding=encoding) \
+ for x in rval)
return rval
"""
Wraps a module and wraps all functions with _unicode_func_wrapper.
"""
- __slots__ = ('_mod',)
+ __slots__ = ('_mod', '_encoding', '_overrides')
- def __init__(self, mod):
+ def __init__(self, mod, encoding='utf_8', overrides=None):
object.__setattr__(self, '_mod', mod)
+ object.__setattr__(self, '_encoding', encoding)
+ object.__setattr__(self, '_overrides', overrides)
def __getattribute__(self, attr):
result = getattr(object.__getattribute__(self, '_mod'), attr)
- if isinstance(result, type):
+ encoding = object.__getattribute__(self, '_encoding')
+ overrides = object.__getattribute__(self, '_overrides')
+ override = None
+ if overrides is not None:
+ override = overrides.get(id(result))
+ if override is not None:
+ result = override
+ elif isinstance(result, type):
pass
elif type(result) is types.ModuleType:
- result = _unicode_module_wrapper(result)
+ result = _unicode_module_wrapper(result,
+ encoding=encoding, overrides=overrides)
elif hasattr(result, '__call__'):
- result = _unicode_func_wrapper(result)
+ result = _unicode_func_wrapper(result, encoding=encoding)
return result
if sys.hexversion >= 0x3000000:
return mod
import os
-os = _unicode_module_wrapper(os)
+os = _unicode_module_wrapper(os, overrides={id(os.read):os.read})
import shutil
shutil = _unicode_module_wrapper(shutil)
try:
import portage._selinux as selinux
except OSError, e:
- writemsg("!!! SELinux not loaded: %s\n" % str(e), noiselevel=-1)
+ sys.stderr.write("!!! SELinux not loaded: %s\n" % str(e))
del e
except ImportError:
pass
# END OF IMPORTS -- END OF IMPORTS -- END OF IMPORTS -- END OF IMPORTS -- END
# ===========================================================================
+def _gen_missing_encodings(missing_encodings):
+
+ encodings = {}
+
+ if 'ascii' in missing_encodings:
+
+ class AsciiIncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return codecs.ascii_encode(input, self.errors)[0]
+
+ class AsciiIncrementalDecoder(codecs.IncrementalDecoder):
+ def decode(self, input, final=False):
+ return codecs.ascii_decode(input, self.errors)[0]
+
+ class AsciiStreamWriter(codecs.StreamWriter):
+ encode = codecs.ascii_encode
+
+ class AsciiStreamReader(codecs.StreamReader):
+ decode = codecs.ascii_decode
+
+ codec_info = codecs.CodecInfo(
+ name='ascii',
+ encode=codecs.ascii_encode,
+ decode=codecs.ascii_decode,
+ incrementalencoder=AsciiIncrementalEncoder,
+ incrementaldecoder=AsciiIncrementalDecoder,
+ streamwriter=AsciiStreamWriter,
+ streamreader=AsciiStreamReader,
+ )
+
+ for alias in ('ascii', '646', 'ansi_x3.4_1968', 'ansi_x3_4_1968',
+ 'ansi_x3.4_1986', 'cp367', 'csascii', 'ibm367', 'iso646_us',
+ 'iso_646.irv_1991', 'iso_ir_6', 'us', 'us_ascii'):
+ encodings[alias] = codec_info
+
+ if 'utf_8' in missing_encodings:
+
+ def utf8decode(input, errors='strict'):
+ return codecs.utf_8_decode(input, errors, True)
+
+ class Utf8IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return codecs.utf_8_encode(input, self.errors)[0]
+
+ class Utf8IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+ _buffer_decode = codecs.utf_8_decode
+
+ class Utf8StreamWriter(codecs.StreamWriter):
+ encode = codecs.utf_8_encode
+
+ class Utf8StreamReader(codecs.StreamReader):
+ decode = codecs.utf_8_decode
+
+ codec_info = codecs.CodecInfo(
+ name='utf-8',
+ encode=codecs.utf_8_encode,
+ decode=utf8decode,
+ incrementalencoder=Utf8IncrementalEncoder,
+ incrementaldecoder=Utf8IncrementalDecoder,
+ streamreader=Utf8StreamReader,
+ streamwriter=Utf8StreamWriter,
+ )
+
+ for alias in ('utf_8', 'u8', 'utf', 'utf8', 'utf8_ucs2', 'utf8_ucs4'):
+ encodings[alias] = codec_info
+
+ return encodings
+
+def _ensure_default_encoding():
+ """
+ The python that's inside stage 1 or 2 is built with a minimal
+ configuration which does not include the /usr/lib/pythonX.Y/encodings
+ directory. This results in error like the following:
+
+ LookupError: no codec search functions registered: can't find encoding
+
+ In order to solve this problem, detect it early and manually register
+ a search function for the ascii and utf_8 codecs. Starting with python-3.0
+ this problem is more noticeable because of stricter handling of encoding
+ and decoding between strings of characters and bytes.
+ """
+
+ default_fallback = 'utf_8'
+ default_encoding = sys.getdefaultencoding().lower().replace('-', '_')
+ required_encodings = set(['ascii', 'utf_8'])
+ required_encodings.add(default_encoding)
+ missing_encodings = set()
+ for codec_name in required_encodings:
+ try:
+ codecs.lookup(codec_name)
+ except LookupError:
+ missing_encodings.add(codec_name)
+
+ if not missing_encodings:
+ return
+
+ encodings = _gen_missing_encodings(missing_encodings)
+
+ if default_encoding in missing_encodings and \
+ default_encoding not in encodings:
+ # Make the fallback codec correspond to whatever name happens
+ # to be returned by sys.getdefaultencoding().
+
+ try:
+ encodings[default_encoding] = codecs.lookup(default_fallback)
+ except LookupError:
+ encodings[default_encoding] = encodings[default_fallback]
+
+ def search_function(name):
+ name = name.lower()
+ name = name.replace('-', '_')
+ codec_info = encodings.get(name)
+ if codec_info is not None:
+ return codecs.CodecInfo(
+ name=codec_info.name,
+ encode=codec_info.encode,
+ decode=codec_info.decode,
+ incrementalencoder=codec_info.incrementalencoder,
+ incrementaldecoder=codec_info.incrementaldecoder,
+ streamreader=codec_info.streamreader,
+ streamwriter=codec_info.streamwriter,
+ )
+ return None
+
+ codecs.register(search_function)
+
+ del codec_name, default_encoding, default_fallback, missing_encodings, \
+ required_encodings, search_function
+
+# Do this ASAP since writemsg() might not work without it.
+_ensure_default_encoding()
+
def _shell_quote(s):
"""
Quote a string in double-quotes and use backslashes to
ldsoconf_path = os.path.join(target_root, EPREFIX_LSTRIP, "etc", "ld.so.conf")
try:
- myld = codecs.open(ldsoconf_path, mode='r',
+ myld = codecs.open(_unicode_encode(ldsoconf_path), mode='r',
encoding='utf_8', errors='replace')
myldlines=myld.readlines()
myld.close()
lines = []
pathname = os.path.join(base_dir, 'Makefile')
try:
- f = open(pathname, 'r')
+ f = codecs.open(_unicode_encode(pathname), mode='r',
+ encoding='utf_8', errors='replace')
except OSError, details:
return (None, str(details))
except IOError, details:
parentsFile = os.path.join(currentPath, "parent")
eapi_file = os.path.join(currentPath, "eapi")
try:
- eapi = open(eapi_file).readline().strip()
+ eapi = codecs.open(_unicode_encode(eapi_file),
+ mode='r', encoding='utf_8', errors='replace'
+ ).readline().strip()
except IOError:
pass
else:
repo_conf_parser = SafeConfigParser()
try:
repo_conf_parser.readfp(
- codecs.open(self._local_repo_conf_path, mode='r',
+ codecs.open(
+ _unicode_encode(self._local_repo_conf_path), mode='r',
encoding='utf_8', errors='replace'))
except EnvironmentError, e:
if e.errno != errno.ENOENT:
return mypids
if logfile:
- log_file = open(logfile, mode='ab')
+ log_file = open(_unicode_encode(logfile), mode='ab')
stdout_file = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb')
master_file = os.fdopen(master_fd, 'rb')
iwtd = [master_file]
# Fetch failed... Try the next one... Kill 404 files though.
if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
- if html404.search(open(mysettings["DISTDIR"]+"/"+myfile).read()):
+ if html404.search(codecs.open(
+ _unicode_encode(myfile_path), mode='r',
+ encoding='utf_8', errors='replace'
+ ).read()):
try:
os.unlink(mysettings["DISTDIR"]+"/"+myfile)
writemsg(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n")
if logfile is None:
return
try:
- f = codecs.open(logfile, mode='r',
+ f = codecs.open(_unicode_encode(logfile), mode='r',
encoding='utf_8', errors='replace')
except EnvironmentError:
return
mode=mystat.st_mode, stat_cached=mystat,
follow_links=False)
- open(os.path.join(mysettings['PORTAGE_BUILDDIR'],
- 'build-info', 'SIZE'), 'w').write(str(size) + '\n')
+ open(_unicode_encode(os.path.join(mysettings['PORTAGE_BUILDDIR'],
+ 'build-info', 'SIZE')), 'w').write(str(size) + '\n')
if bsd_chflags:
# Restore all of the flags saved above.
# From parse-eapi-glep-55 above.
pass
elif 'parse-eapi-ebuild-head' in mysettings.features:
- eapi = _parse_eapi_ebuild_head(codecs.open(ebuild_path,
+ eapi = _parse_eapi_ebuild_head(
+ codecs.open(_unicode_encode(ebuild_path),
mode='r', encoding='utf_8', errors='replace'))
if eapi is not None:
if background and log_path is not None:
try:
- log_file = open(log_path, 'a')
+ log_file = codecs.open(_unicode_encode(log_path), mode='a',
+ encoding='utf_8', errors='replace')
except IOError:
def write(msg):
pass
else:
def write(msg):
- log_file.write(msg)
+ log_file.write(_unicode_decode(msg))
log_file.flush()
try:
os.access(mysettings["PORT_LOGDIR"], os.W_OK):
logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid")
if not os.path.exists(logid_path):
- f = open(logid_path, "w")
- f.close()
- del f
+ open(_unicode_encode(logid_path), 'w')
logid_time = time.strftime("%Y%m%d-%H%M%S",
time.gmtime(os.stat(logid_path).st_mtime))
mysettings["PORTAGE_LOG_FILE"] = os.path.join(
# This is a signal to ebuild.sh, so that it knows to filter
# out things like SANDBOX_{DENY,PREDICT,READ,WRITE} that
# would be preserved between normal phases.
- open(env_file + ".raw", "w")
+ open(_unicode_encode(env_file + '.raw'), 'w')
else:
writemsg(("!!! Error extracting saved " + \
"environment: '%s'\n") % \
xptbz2.unpackinfo(infloc)
mysettings.setcpv(mycat + "/" + mypkg, mydb=mydbapi)
# Store the md5sum in the vdb.
- fp = open(os.path.join(infloc, "BINPKGMD5"), "w")
+ fp = open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5')), 'w')
fp.write(str(portage.checksum.perform_md5(mytbz2))+"\n")
fp.close()
DEPRECATED_PROFILE_FILE.lstrip(os.sep))
if not os.access(deprecated_profile_file, os.R_OK):
return False
- deprecatedfile = open(deprecated_profile_file, "r")
- dcontent = deprecatedfile.readlines()
- deprecatedfile.close()
+ dcontent = codecs.open(_unicode_encode(deprecated_profile_file),
+ mode='r', encoding='utf_8', errors='replace').readlines()
writemsg(colorize("BAD", "\n!!! Your current profile is " + \
"deprecated and not supported anymore.") + "\n", noiselevel=-1)
if not dcontent:
atexit_register(portageexit)
-def _gen_missing_encodings(missing_encodings):
-
- encodings = {}
-
- if 'ascii' in missing_encodings:
-
- class AsciiIncrementalEncoder(codecs.IncrementalEncoder):
- def encode(self, input, final=False):
- return codecs.ascii_encode(input, self.errors)[0]
-
- class AsciiIncrementalDecoder(codecs.IncrementalDecoder):
- def decode(self, input, final=False):
- return codecs.ascii_decode(input, self.errors)[0]
-
- class AsciiStreamWriter(codecs.StreamWriter):
- encode = codecs.ascii_encode
-
- class AsciiStreamReader(codecs.StreamReader):
- decode = codecs.ascii_decode
-
- codec_info = codecs.CodecInfo(
- name='ascii',
- encode=codecs.ascii_encode,
- decode=codecs.ascii_decode,
- incrementalencoder=AsciiIncrementalEncoder,
- incrementaldecoder=AsciiIncrementalDecoder,
- streamwriter=AsciiStreamWriter,
- streamreader=AsciiStreamReader,
- )
-
- for alias in ('ascii', '646', 'ansi_x3.4_1968', 'ansi_x3_4_1968',
- 'ansi_x3.4_1986', 'cp367', 'csascii', 'ibm367', 'iso646_us',
- 'iso_646.irv_1991', 'iso_ir_6', 'us', 'us_ascii'):
- encodings[alias] = codec_info
-
- if 'utf_8' in missing_encodings:
-
- def utf8decode(input, errors='strict'):
- return codecs.utf_8_decode(input, errors, True)
-
- class Utf8IncrementalEncoder(codecs.IncrementalEncoder):
- def encode(self, input, final=False):
- return codecs.utf_8_encode(input, self.errors)[0]
-
- class Utf8IncrementalDecoder(codecs.BufferedIncrementalDecoder):
- _buffer_decode = codecs.utf_8_decode
-
- class Utf8StreamWriter(codecs.StreamWriter):
- encode = codecs.utf_8_encode
-
- class Utf8StreamReader(codecs.StreamReader):
- decode = codecs.utf_8_decode
-
- codec_info = codecs.CodecInfo(
- name='utf-8',
- encode=codecs.utf_8_encode,
- decode=utf8decode,
- incrementalencoder=Utf8IncrementalEncoder,
- incrementaldecoder=Utf8IncrementalDecoder,
- streamreader=Utf8StreamReader,
- streamwriter=Utf8StreamWriter,
- )
-
- for alias in ('utf_8', 'u8', 'utf', 'utf8', 'utf8_ucs2', 'utf8_ucs4'):
- encodings[alias] = codec_info
-
- return encodings
-
-def _ensure_default_encoding():
- """
- The python that's inside stage 1 or 2 is built with a minimal
- configuration which does not include the /usr/lib/pythonX.Y/encodings
- directory. This results in error like the following:
-
- LookupError: no codec search functions registered: can't find encoding
-
- In order to solve this problem, detect it early and manually register
- a search function for the ascii and utf_8 codecs. Starting with python-3.0
- this problem is more noticeable because of stricter handling of encoding
- and decoding between strings of characters and bytes.
- """
-
- default_fallback = 'utf_8'
- default_encoding = sys.getdefaultencoding().lower().replace('-', '_')
- required_encodings = set(['ascii', 'utf_8'])
- required_encodings.add(default_encoding)
- missing_encodings = set()
- for codec_name in required_encodings:
- try:
- codecs.lookup(codec_name)
- except LookupError:
- missing_encodings.add(codec_name)
-
- if not missing_encodings:
- return
-
- encodings = _gen_missing_encodings(missing_encodings)
-
- if default_encoding in missing_encodings and \
- default_encoding not in encodings:
- # Make the fallback codec correspond to whatever name happens
- # to be returned by sys.getdefaultencoding().
-
- try:
- encodings[default_encoding] = codecs.lookup(default_fallback)
- except LookupError:
- encodings[default_encoding] = encodings[default_fallback]
-
- def search_function(name):
- name = name.lower()
- name = name.replace('-', '_')
- codec_info = encodings.get(name)
- if codec_info is not None:
- return codecs.CodecInfo(
- name=codec_info.name,
- encode=codec_info.encode,
- decode=codec_info.decode,
- incrementalencoder=codec_info.incrementalencoder,
- incrementaldecoder=codec_info.incrementaldecoder,
- streamreader=codec_info.streamreader,
- streamwriter=codec_info.streamwriter,
- )
- return None
-
- codecs.register(search_function)
-
- del codec_name, default_encoding, default_fallback, missing_encodings, \
- required_encodings, search_function
-
def _global_updates(trees, prev_mtimes):
"""
Perform new global updates if they exist in $PORTDIR/profiles/updates/.
def _load(self, filename):
try:
- f = open(filename, 'rb')
+ f = open(_unicode_encode(filename), 'rb')
mypickle = pickle.Unpickler(f)
try:
mypickle.find_global = None
"flushmtimedb"):
globals()[k] = _LegacyGlobalProxy(k)
- _ensure_default_encoding()
-
# Clear the cache
dircache={}
projects / portage.git / commitdiff