From: Fabian Groffen Date: Fri, 14 Aug 2009 20:24:14 +0000 (-0000) Subject: Merged from trunk -r13968:13978 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=ec0940f0a253ae57892574c8e681b8647d9434ab;p=portage.git Merged from trunk -r13968:13978 | 13969 | Tweak $ROOT handling inside depgraph._add_pkg_deps() for | | zmedico | removal actions, so --with-bdeps works correctly with | | | --depclean and --prune. | | 13970 | In calc_depclean(), sort packages that are displayed due to | | zmedico | being pulled in by link level dependencies. | | 13971 | Don't trigger the --root-deps code for removal actions such | | zmedico | as --prune and --depclean. | | 13972 | Fix PreservedLibsRegistry.pruneNonExisting() to work with | | zmedico | $ROOT. | | 13973 | Use writemsg_stdout() for safe unicode output of | | zmedico | --list-sets. | | 13974 | Bug #280962 - Fix broken path normalization inside | | zmedico | StaticFileSet.multiBuilder(). | | 13975 | Open new items in text mode (unicode), and safely handle | | zmedico | unicode in news item names. | | 13977 | Bug #280998 - Misc selinux fixes. Thanks to Chris PeBenito | | zmedico | for this patch. | | 13978 | Bug #280460 - Wrap os and os.path modules with unicode | | zmedico | encode/decode wrappers for python-2.x. | svn path=/main/branches/prefix/; revision=14039 --- diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 6a38cc25a..d8a1f93c0 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -1030,7 +1030,8 @@ def calc_depclean(settings, trees, ldpath_mtimes, line in wrap(msg, 70)), level=logging.WARNING, noiselevel=-1) msg = [] - for pkg, consumers in consumer_map.iteritems(): + for pkg in sorted(consumer_map, key=cmp_sort_key(cmp_pkg_cpv)): + consumers = consumer_map[pkg] unique_consumers = set(chain(*consumers.values())) unique_consumers = sorted(consumer.mycpv \ for consumer in unique_consumers) diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index cac48b456..1af872562 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -1016,13 +1016,16 @@ class depgraph(object): if removal_action and self._frozen_config.myopts.get("--with-bdeps", "y") == "n": edepend["DEPEND"] = "" - bdeps_root = "/" - root_deps = self._frozen_config.myopts.get("--root-deps") - if root_deps is not None: - if root_deps is True: - bdeps_root = myroot - elif root_deps == "rdeps": - edepend["DEPEND"] = "" + if removal_action: + bdeps_root = myroot + else: + bdeps_root = "/" + root_deps = self._frozen_config.myopts.get("--root-deps") + if root_deps is not None: + if root_deps is True: + bdeps_root = myroot + elif root_deps == "rdeps": + edepend["DEPEND"] = "" deps = ( (bdeps_root, edepend["DEPEND"], diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 05eb91080..4565422be 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -44,7 +44,7 @@ import portage.exception from portage.const import EPREFIX, BPREFIX, EPREFIX_LSTRIP from portage.data import secpass from portage.util import normalize_path as normpath -from portage.util import writemsg, writemsg_level +from portage.util import writemsg, writemsg_level, writemsg_stdout from portage.sets import SETPREFIX from _emerge.actions import action_config, action_sync, action_metadata, \ @@ -1121,8 +1121,7 @@ def emerge_main(): root_config = trees[settings["ROOT"]]["root_config"] if myaction == "list-sets": - sys.stdout.write("".join("%s\n" % s for s in sorted(root_config.sets))) - sys.stdout.flush() + writemsg_stdout("".join("%s\n" % s for s in sorted(root_config.sets))) return os.EX_OK # only expand sets for actions taking package arguments diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 4f849722f..ad91ac6a6 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -23,6 +23,7 @@ try: import re import shutil import time + import types try: import cPickle as pickle except ImportError: @@ -124,6 +125,70 @@ except ImportError: # END OF IMPORTS -- END OF IMPORTS -- END OF IMPORTS -- END OF IMPORTS -- END # =========================================================================== +def _unicode_encode(s): + if isinstance(s, unicode): + s = s.encode('utf_8', 'replace') + return s + +def _unicode_decode(s): + if not isinstance(s, unicode) and isinstance(s, basestring): + s = unicode(s, encoding='utf_8', errors='replace') + return s + +class _unicode_func_wrapper(object): + """ + Wraps a function, converts arguments from unicode to bytes, + and return values to unicode from bytes. + """ + __slots__ = ('_func',) + + def __init__(self, func): + self._func = func + + def __call__(self, *args, **kwargs): + + wrapped_args = [_unicode_encode(x) for x in args] + if kwargs: + wrapped_kwargs = dict((_unicode_encode(k), _unicode_encode(v)) \ + for k, v in kwargs.iteritems()) + else: + wrapped_kwargs = {} + + rval = self._func(*wrapped_args, **wrapped_kwargs) + + if isinstance(rval, (basestring, list, tuple)): + if isinstance(rval, basestring): + rval = _unicode_decode(rval) + elif isinstance(rval, list): + rval = [_unicode_decode(x) for x in rval] + elif isinstance(rval, tuple): + rval = tuple(_unicode_decode(x) for x in rval) + + return rval + +class _unicode_module_wrapper(object): + """ + Wraps a module and wraps all functions with _unicode_func_wrapper. + """ + __slots__ = ('_mod',) + + def __init__(self, mod): + object.__setattr__(self, '_mod', mod) + + def __getattribute__(self, attr): + result = getattr(object.__getattribute__(self, '_mod'), attr) + if isinstance(result, type): + pass + elif type(result) is types.ModuleType: + result = _unicode_module_wrapper(result) + elif hasattr(result, '__call__'): + result = _unicode_func_wrapper(result) + return result + +if sys.hexversion >= 0x3000000: + def _unicode_module_wrapper(mod): + return mod + def _shell_quote(s): """ Quote a string in double-quotes and use backslashes to @@ -3505,9 +3570,7 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero spawn_func = portage.process.spawn_sandbox if sesandbox: - con = selinux.getcontext() - con = con.replace(mysettings["PORTAGE_T"], - mysettings["PORTAGE_SANDBOX_T"]) + con = selinux.settype(mysettings["PORTAGE_SANDBOX_T"]) selinux.setexec(con) returnpid = keywords.get("returnpid") @@ -3518,7 +3581,7 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero if logfile: os.close(slave_fd) if sesandbox: - selinux.setexec(None) + selinux.setexec() if returnpid: return mypids @@ -3596,8 +3659,7 @@ def _spawn_fetch(settings, args, **kwargs): try: if settings.selinux_enabled(): - con = selinux.getcontext() - con = con.replace(settings["PORTAGE_T"], settings["PORTAGE_FETCH_T"]) + con = selinux.settype(settings["PORTAGE_FETCH_T"]) selinux.setexec(con) # bash is an allowed entrypoint, while most binaries are not if args[0] != BASH_BINARY: @@ -3608,7 +3670,7 @@ def _spawn_fetch(settings, args, **kwargs): finally: if settings.selinux_enabled(): - selinux.setexec(None) + selinux.setexec() return rval diff --git a/pym/portage/_selinux.py b/pym/portage/_selinux.py index 2a50f7434..1b5f530ce 100644 --- a/pym/portage/_selinux.py +++ b/pym/portage/_selinux.py @@ -61,19 +61,27 @@ def rename(src, dest): finally: setfscreate() +def settype(newtype): + ret = getcontext().split(":") + ret[2] = newtype + return ":".join(ret) + def setexec(ctx="\n"): + if isinstance(ctx, unicode): + ctx = ctx.encode('utf_8', 'replace') if selinux.setexeccon(ctx) < 0: raise OSError("setexec: Failed setting exec() context \"%s\"." % ctx) def setfscreate(ctx="\n"): + if isinstance(ctx, unicode): + ctx = ctx.encode('utf_8', 'replace') if selinux.setfscreatecon(ctx) < 0: raise OSError( "setfscreate: Failed setting fs create context \"%s\"." % ctx) def spawn(selinux_type, spawn_func, mycommand, opt_name=None, **keywords): - con = getcontext().split(":") - con[2] = selinux_type - setexec(":".join(con)) + con = settype(selinux_type) + setexec(con) try: return spawn_func(mycommand, opt_name=opt_name, **keywords) finally: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index a5958db69..3ada2e1a5 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -34,7 +34,8 @@ from portage.localization import _ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ doebuild_environment, doebuild, env_update, prepare_build_dirs, \ - abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey + abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey, \ + _unicode_module_wrapper from portage.cache.mappings import slot_dict_class @@ -44,6 +45,8 @@ import logging import sys from itertools import izip +os = _unicode_module_wrapper(os) + try: import cPickle as pickle except ImportError: @@ -51,12 +54,16 @@ except ImportError: class PreservedLibsRegistry(object): """ This class handles the tracking of preserved library objects """ - def __init__(self, filename, autocommit=True): - """ @param filename: absolute path for saving the preserved libs records + def __init__(self, root, filename, autocommit=True): + """ + @param root: root used to check existence of paths in pruneNonExisting + @type root: String + @param filename: absolute path for saving the preserved libs records @type filename: String @param autocommit: determines if the file is written after every update @type autocommit: Boolean """ + self._root = root self._filename = filename self._autocommit = autocommit self.load() @@ -133,7 +140,8 @@ class PreservedLibsRegistry(object): """ Remove all records for objects that no longer exist on the filesystem. """ for cps in self._data.keys(): cpv, counter, paths = self._data[cps] - paths = [f for f in paths if os.path.exists(f)] + paths = [f for f in paths \ + if os.path.exists(os.path.join(self._root, f.lstrip(os.sep)))] if len(paths) > 0: self._data[cps] = (cpv, counter, paths) else: @@ -1453,7 +1461,7 @@ class vardbapi(dbapi): CACHE_PATH.lstrip(os.path.sep), "counter") try: - self.plib_registry = PreservedLibsRegistry( + self.plib_registry = PreservedLibsRegistry(self.root, os.path.join(self.root, EPREFIX_LSTRIP, PRIVATE_PATH, "preserved_libs_registry")) except PermissionDenied: # apparently this user isn't allowed to access PRIVATE_PATH diff --git a/pym/portage/news.py b/pym/portage/news.py index 750df3dd6..c6bd9eafa 100644 --- a/pym/portage/news.py +++ b/pym/portage/news.py @@ -7,6 +7,7 @@ __all__ = ["NewsManager", "NewsItem", "DisplayRestriction", "DisplayProfileRestriction", "DisplayKeywordRestriction", "DisplayInstalledRestriction"] +import codecs import logging import os import re @@ -116,6 +117,8 @@ class NewsManager(object): itemid + "." + self.language_id + ".txt") if not os.path.isfile(filename): continue + if not isinstance(itemid, unicode): + itemid = unicode(itemid, encoding='utf_8', errors='replace') item = NewsItem(filename, itemid) if not item.isValid(): continue @@ -224,7 +227,8 @@ class NewsItem(object): return self._valid def parse(self): - lines = open(self.path).readlines() + lines = codecs.open(self.path, mode='r', + encoding='utf_8', errors='replace').readlines() self.restrictions = [] invalids = [] for i, line in enumerate(lines): diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 17d2f0bc5..800c187f2 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -125,17 +125,18 @@ class StaticFileSet(EditablePackageSet): except KeyError: raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"]) - if isinstance(directory, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - directory_unicode = directory - directory = directory.encode('utf_8', 'replace') - else: - directory_unicode = unicode(directory, - encoding='utf_8', errors='replace') - if os.path.isdir(directory): directory = normalize_path(directory) + + if isinstance(directory, unicode): + # Avoid UnicodeDecodeError raised from + # os.path.join when called by os.walk. + directory_unicode = directory + directory = directory.encode('utf_8', 'replace') + else: + directory_unicode = unicode(directory, + encoding='utf_8', errors='replace') + for parent, dirs, files in os.walk(directory): if not isinstance(parent, unicode): parent = unicode(parent,