From cb7a46ab788faa386d4d67f0a6d26c5cb132a56c Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 31 Jul 2008 13:30:22 +0000 Subject: [PATCH] Merged from trunk 11287:11300 | 11288 | Bug #233421 - Fix grammar, missing "be" in --update | | zmedico | description. Thanks to Mikael Magnusson for this patch. | | 11289 | Split out a _spawn_fetch() function that will be useful for | | zmedico | implementing a userpriv testcase for bug #233303. | | 11290 | Fixes in portage.fetch() for bugs #233303 and #94133: * | | zmedico | Totally skip $DISTDIR creation if the fetch_to_ro feature is | | | enabled. * Don't touch $DISTDIR permissions unless unless | | | usepriv and/or userfetch are enabled. * When usepriv and/or | | | userfetch are enabled, test whether or not a process that | | | has dropped privileges is able to create a file in the | | | directory, and only adjust permissions if the test fails. * | | | Completely | | 11291 | disable default IONICE command as it breaks for non-root, | | genone | and ionice might not always be available | | 11292 | Implement a new @live-ebuilds which is generated from | | zmedico | installed packages that inherit from know live eclasses such | | | as cvs, darcs, git, mercurial, and subversion. The list of | | | eclasses is controlled by an "inherits" attribute that is | | | configure in sets.conf for and instance of InheritSet. This | | | set serves a purpose similar to the -scm ebuild suffix that | | | has been proposed in GLEP 54. | | 11293 | Add a new @module-rebuild set which emulates the behavior of | | zmedico | the module-rebuild tool. The /lib/modules path is set in | | | sets.conf via a "files" attribute of an OwnerSet instance. | | | This can be easily used to define similar sets based on | | | paths on installed files. | | 11294 | Remove quotes since the seem to cause incorrect results. | | zmedico | | | 11295 | * Rename @live-ebuilds to @live-rebuild, for consistency | | zmedico | with the other *-rebuild sets. * Document the new sets. | | 11296 | Describe InheritSet and OwnerSet. | | zmedico | | | 11297 | Bug #233458 - Fix AsynchronousTask exit listener handling so | | zmedico | that an exit listener will never get called after it's been | | | passed into removeExitListener(), since the caller of | | | removeExitListener() needs to be able to be able to trust | | | that the given exit listener will not be called under any | | | circumstances. | | 11298 | Always invalidate results from | | zmedico | _userpriv_test_write_file_cache when adjusting permissions | | | on a given directory. | | 11299 | Bug #233253 - Implement a @downgrade set which selects | | zmedico | packages for which the highest visible ebuild version is | | | lower than the currently installed version. This is useful | | | if you have installed packages from an overlay and you want | | | to downgrade to the highest visible after removing the | | | overlay, even though the packages that will be dowgraded are | | | not necessarily masked in any way. | | 11300 | Fix DowngradeSet so it's safe for cases when no ebuild is | | zmedico | available. | svn path=/main/branches/prefix/; revision=11302 --- cnf/make.globals | 3 +- cnf/sets.conf | 18 ++++++ doc/config/sets.docbook | 38 ++++++++++++ pym/_emerge/__init__.py | 22 +++++-- pym/_emerge/help.py | 4 +- pym/portage/__init__.py | 126 +++++++++++++++++++++++++++++--------- pym/portage/sets/dbapi.py | 117 ++++++++++++++++++++++++++++++++++- 7 files changed, 288 insertions(+), 40 deletions(-) diff --git a/cnf/make.globals b/cnf/make.globals index 016810a9f..30932b054 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -63,7 +63,8 @@ PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" PORTAGE_FETCH_RESUME_MIN_SIZE="350K" # Command called to adjust the io priority of portage and it's subprocesses. -PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}" +# Note: should be wrapped inside a uid check +#PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}" # Number of times 'emerge --sync' will run before giving up. PORTAGE_RSYNC_RETRIES="3" diff --git a/cnf/sets.conf b/cnf/sets.conf index 241fe63c5..a0ab06858 100644 --- a/cnf/sets.conf +++ b/cnf/sets.conf @@ -41,3 +41,21 @@ directory = @DOMAIN_PREFIX@/etc/portage/sets [preserved-rebuild] class = portage.sets.libs.PreservedLibraryConsumerSet world-candidate = False + +# Installed ebuilds that inherit from known live eclasses. +[live-rebuild] +class = portage.sets.dbapi.InheritSet +world-candidate = False +inherits = cvs darcs git mercurial subversion + +# Installed packages that own files inside /lib/modules. +[module-rebuild] +class = portage.sets.dbapi.OwnerSet +world-candidate = False +files = /lib/modules + +# Installed packages for which the highest visible ebuild +# version is lower than the currently installed version. +[downgrade] +class = portage.sets.dbapi.DowngradeSet +world-candidate = False diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook index b284f5e2a..7cccbbc93 100644 --- a/doc/config/sets.docbook +++ b/doc/config/sets.docbook @@ -453,6 +453,41 @@ + + portage.sets.dbapi.InheritSet + + Package set which contains all packages + that inherit one or more specific eclasses. + This class supports the following options: + + inherits: Required. A list of eclass names + which should be used to create the package set. + + + + + + portage.sets.dbapi.OwnerSet + + Package set which contains all packages + that own one or more files. + This class supports the following options: + + files: Required. A list of file paths + that should be used to create the package set. + + + + + + portage.sets.dbapi.DowngradeSet + + Package set which contains all packages + for which the highest visible ebuild version is lower than + the currently installed version. + This class doesn't support any extra options. + + portage.sets.libs.PreservedLibraryConsumerSet @@ -490,6 +525,9 @@ security: uses NewAffectedSet with default options everything: uses EverythingSet preserved-rebuild: uses PreservedLibraryConsumerSet + live-rebuild: uses InheritSet + module-rebuild: uses OwnerSet + downgrade: uses DowngradeSet Additionally the default configuration includes a multi set section based on the StaticFileSet defaults that creates a set for each diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index c7877e06a..42d19117c 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1626,7 +1626,7 @@ class AsynchronousTask(SlotObject): """ __slots__ = ("background", "cancelled", "returncode") + \ - ("_exit_listeners", "_start_listeners") + ("_exit_listeners", "_exit_listener_stack", "_start_listeners") def start(self): """ @@ -1692,6 +1692,8 @@ class AsynchronousTask(SlotObject): def removeExitListener(self, f): if self._exit_listeners is None: + if self._exit_listener_stack is not None: + self._exit_listener_stack.remove(f) return self._exit_listeners.remove(f) @@ -1707,12 +1709,22 @@ class AsynchronousTask(SlotObject): # This prevents recursion, in case one of the # exit handlers triggers this method again by - # calling wait(). - exit_listeners = self._exit_listeners + # calling wait(). Use a stack that gives + # removeExitListener() an opportunity to consume + # listeners from the stack, before they can get + # called below. This is necessary because a call + # to one exit listener may result in a call to + # removeExitListener() for another listener on + # the stack. That listener needs to be removed + # from the stack since it would be inconsistent + # to call it after it has been been passed into + # removeExitListener(). + self._exit_listener_stack = self._exit_listeners self._exit_listeners = None - for f in exit_listeners: - f(self) + self._exit_listener_stack.reverse() + while self._exit_listener_stack: + self._exit_listener_stack.pop()(self) class PipeReader(AsynchronousTask): diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index bb4077aa5..e0b93480c 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -189,8 +189,8 @@ def help(myaction,myopts,havecolor=1): print " Updates packages to the best version available, which may not" print " always be the highest version number due to masking for testing" print " and development. This will also update direct dependencies which" - print " may not what you want. Package atoms specified on the command line" - print " are greedy, meaning that unspecific atoms may match multiple" + print " may not be what you want. Package atoms specified on the command" + print " line are greedy, meaning that unspecific atoms may match multiple" print " installed versions of slotted packages." print print " "+green("--version")+" ("+green("-V")+" short option)" diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 264fb47f2..ff50264e5 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -3167,6 +3167,84 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero return retval >> 8 return retval +_userpriv_spawn_kwargs = ( + ("uid", portage_uid), + ("gid", portage_gid), + ("groups", userpriv_groups), + ("umask", 002), +) + +def _spawn_fetch(settings, args, **kwargs): + """ + Spawn a process with appropriate settings for fetching, including + userfetch and selinux support. + """ + + global _userpriv_spawn_kwargs + + # Redirect all output to stdout since some fetchers like + # wget pollute stderr (if portage detects a problem then it + # can send it's own message to stderr). + if "fd_pipes" not in kwargs: + + kwargs["fd_pipes"] = { + 0 : sys.stdin.fileno(), + 1 : sys.stdout.fileno(), + 2 : sys.stdout.fileno(), + } + + if "userfetch" in settings.features and \ + os.getuid() == 0 and portage_gid and portage_uid: + kwargs.update(_userpriv_spawn_kwargs) + + try: + + if settings.selinux_enabled(): + con = selinux.getcontext() + con = con.replace(settings["PORTAGE_T"], settings["PORTAGE_FETCH_T"]) + selinux.setexec(con) + # bash is an allowed entrypoint, while most binaries are not + if args[0] != BASH_BINARY: + args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args + + rval = portage.process.spawn(args, + env=dict(settings.iteritems()), **kwargs) + + finally: + if settings.selinux_enabled(): + selinux.setexec(None) + + return rval + +_userpriv_test_write_file_cache = {} +_userpriv_test_write_cmd_script = "> %(file_path)s ; rval=$? ; " + \ + "rm -f %(file_path)s ; exit $rval" + +def _userpriv_test_write_file(settings, file_path): + """ + Drop privileges and try to open a file for writing. The file may or + may not exist, and the parent directory is assumed to exist. The file + is removed before returning. + + @param settings: A config instance which is passed to _spawn_fetch() + @param file_path: A file path to open and write. + @return: True if write succeeds, False otherwise. + """ + + global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script + rval = _userpriv_test_write_file_cache.get(file_path) + if rval is not None: + return rval + + args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \ + {"file_path" : _shell_quote(file_path)}] + + returncode = _spawn_fetch(settings, args) + + rval = returncode == os.EX_OK + _userpriv_test_write_file_cache[file_path] = rval + return rval + def _checksum_failure_temp_file(distdir, basename): """ First try to find a duplicate temp file with the same checksum and return @@ -3274,6 +3352,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", features = mysettings.features restrict = mysettings.get("PORTAGE_RESTRICT","").split() + + from portage.data import secpass + userfetch = secpass >= 2 and "userfetch" in features + userpriv = secpass >= 2 and "userpriv" in features + # 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring. if "mirror" in restrict or \ "nomirror" in restrict: @@ -3473,7 +3556,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if not mysettings.get(var_name, None): can_fetch = False - if can_fetch: + if can_fetch and not fetch_to_ro: + global _userpriv_test_write_file_cache dirmode = 02070 filemode = 060 modemask = 02 @@ -3491,6 +3575,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", for x in distdir_dirs: mydir = os.path.join(mysettings["DISTDIR"], x) + write_test_file = os.path.join( + mydir, ".__portage_test_write__") + + if os.path.isdir(mydir): + if not (userfetch or userpriv): + continue + if _userpriv_test_write_file(mysettings, write_test_file): + continue + + _userpriv_test_write_file_cache.pop(write_test_file, None) if portage.util.ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask): writemsg("Adjusting permissions recursively: '%s'\n" % mydir, noiselevel=-1) @@ -3840,38 +3934,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", lexer = shlex.shlex(StringIO.StringIO(locfetch), posix=True) lexer.whitespace_split = True myfetch = [varexpand(x, mydict=variables) for x in lexer] - - spawn_keywords = {} - # Redirect all output to stdout since some fetchers like - # wget pollute stderr (if portage detects a problem then it - # can send it's own message to stderr). - spawn_keywords["fd_pipes"] = { - 0:sys.stdin.fileno(), - 1:sys.stdout.fileno(), - 2:sys.stdout.fileno() - } - if "userfetch" in mysettings.features and \ - os.getuid() == 0 and portage_gid and portage_uid: - spawn_keywords.update({ - "uid" : portage_uid, - "gid" : portage_gid, - "groups" : userpriv_groups, - "umask" : 002}) myret = -1 try: - if mysettings.selinux_enabled(): - con = selinux.getcontext() - con = con.replace(mysettings["PORTAGE_T"], mysettings["PORTAGE_FETCH_T"]) - selinux.setexec(con) - # bash is an allowed entrypoint, while most binaries are not - myfetch = ["bash", "-c", "exec \"$@\"", myfetch[0]] + myfetch - - myret = portage.process.spawn(myfetch, - env=dict(mysettings.iteritems()), **spawn_keywords) - - if mysettings.selinux_enabled(): - selinux.setexec(None) + myret = _spawn_fetch(mysettings, myfetch) finally: try: diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index 5352a32f3..798b322e3 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -2,11 +2,11 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -from portage.versions import catsplit +from portage.versions import catpkgsplit, catsplit, pkgcmp from portage.sets.base import PackageSet from portage.sets import SetConfigError, get_boolean -__all__ = ["CategorySet", "EverythingSet"] +__all__ = ["CategorySet", "EverythingSet", "InheritSet"] class EverythingSet(PackageSet): _operations = ["merge", "unmerge"] @@ -32,6 +32,119 @@ class EverythingSet(PackageSet): return EverythingSet(trees["vartree"].dbapi) singleBuilder = classmethod(singleBuilder) +class OwnerSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all packages " + \ + "that own one or more files." + + def __init__(self, vardb=None, files=None): + super(OwnerSet, self).__init__() + self._db = vardb + self._files = files + + def mapPathsToAtoms(self, paths): + rValue = set() + vardb = self._db + aux_get = vardb.aux_get + aux_keys = ["SLOT"] + for link, p in vardb._owners.iter_owners(paths): + cat, pn = catpkgsplit(link.mycpv)[:2] + slot, = aux_get(link.mycpv, aux_keys) + rValue.add("%s/%s:%s" % (cat, pn, slot)) + return rValue + + def load(self): + self._setAtoms(self.mapPathsToAtoms(self._files)) + + def singleBuilder(cls, options, settings, trees): + if not "files" in options: + raise SetConfigError("no files given") + + import shlex + return cls(vardb=trees["vartree"].dbapi, + files=frozenset(shlex.split(options["files"]))) + + singleBuilder = classmethod(singleBuilder) + +class InheritSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all packages " + \ + "that inherit one or more specific eclasses." + + def __init__(self, vardb=None, inherits=None): + super(InheritSet, self).__init__() + self._db = vardb + self._inherits = inherits + + def load(self): + atoms = [] + inherits = self._inherits + cp_list = self._db.cp_list + aux_get = self._db.aux_get + aux_keys = ["INHERITED", "SLOT"] + for cp in self._db.cp_all(): + for cpv in cp_list(cp): + inherited, slot = aux_get(cpv, aux_keys) + inherited = inherited.split() + if inherits.intersection(inherited): + atoms.append("%s:%s" % (cp, slot)) + + self._setAtoms(atoms) + + def singleBuilder(cls, options, settings, trees): + if not "inherits" in options: + raise SetConfigError("no inherits given") + + inherits = options["inherits"] + return cls(vardb=trees["vartree"].dbapi, + inherits=frozenset(inherits.split())) + + singleBuilder = classmethod(singleBuilder) + +class DowngradeSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all packages " + \ + "for which the highest visible ebuild version is lower than " + \ + "the currently installed version." + + def __init__(self, portdb=None, vardb=None): + super(DowngradeSet, self).__init__() + self._portdb = portdb + self._vardb = vardb + + def load(self): + atoms = [] + xmatch = self._portdb.xmatch + xmatch_level = "bestmatch-visible" + cp_list = self._vardb.cp_list + aux_get = self._vardb.aux_get + aux_keys = ["SLOT"] + for cp in self._vardb.cp_all(): + for cpv in cp_list(cp): + slot, = aux_get(cpv, aux_keys) + slot_atom = "%s:%s" % (cp, slot) + ebuild = xmatch(xmatch_level, slot_atom) + if not ebuild: + continue + ebuild_split = catpkgsplit(ebuild)[1:] + installed_split = catpkgsplit(cpv)[1:] + if pkgcmp(installed_split, ebuild_split) > 0: + atoms.append(slot_atom) + + self._setAtoms(atoms) + + def singleBuilder(cls, options, settings, trees): + return cls(portdb=trees["porttree"].dbapi, + vardb=trees["vartree"].dbapi) + + singleBuilder = classmethod(singleBuilder) + class CategorySet(PackageSet): _operations = ["merge", "unmerge"] -- 2.26.2