From 572c8f8c82dd2dccc40bec9a46f849f4329dc1e0 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 24 Jun 2009 12:45:17 +0000 Subject: [PATCH] Greetings from LinuxTag 2009 too :) Merged from trunk -r13654:13662 | 13655 | Bug #264434 - Delay evaluation of all disjunctive (virtual | | zmedico | and ||) dependencies. Evaluting disjuctions as late as | | | possible allows better decisions since the graph is more | | | complete when the decisions are made. Thanks to Sebastian | | | Mingramm (few) for the initial patch. | | 13656 | Don't set KV for EAPI >=3. | | arfrever | | | 13657 | Factor duplicate code out of depgraph._add_pkg and | | zmedico | _pop_disjunction. | | 13658 | In doebuild_environment, ensure that cached $KV from the | | zmedico | backupenv does not leak into the environment for EAPI >= 3. | | 13659 | Bug #274742 - Exempt $dir from quoting checks since it's a | | zmedico | likely candidate for false positives. | | 13660 | Tweak depcache permission handling so egencache can be run | | zmedico | by a user who's not in the portage group, allowing for | | | better privilege isolation. | | 13661 | Reorganize $KV logic in doebuild_environment. | | zmedico | | | 13662 | Use portage.util.apply_permissions() inside | | zmedico | _ensure_access(). | svn path=/main/branches/prefix/; revision=13684 --- pym/_emerge/__init__.py | 183 +++++++++++++++++++++++++------ pym/portage/__init__.py | 6 +- pym/portage/cache/fs_template.py | 31 ++++-- pym/portage/cache/sqlite.py | 2 +- pym/portage/dbapi/porttree.py | 15 ++- pym/repoman/checks.py | 2 +- 6 files changed, 189 insertions(+), 50 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d30174470..7af91cd2b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -4781,6 +4781,7 @@ class depgraph(object): self._unsatisfied_blockers_for_display = None self._circular_deps_for_display = None self._dep_stack = [] + self._dep_disjunctive_stack = [] self._unsatisfied_deps = [] self._initially_unsatisfied_deps = [] self._ignored_deps = [] @@ -5062,16 +5063,21 @@ class depgraph(object): def _create_graph(self, allow_unsatisfied=False): dep_stack = self._dep_stack - while dep_stack: + dep_disjunctive_stack = self._dep_disjunctive_stack + while dep_stack or dep_disjunctive_stack: self.spinner.update() - dep = dep_stack.pop() - if isinstance(dep, Package): - if not self._add_pkg_deps(dep, - allow_unsatisfied=allow_unsatisfied): + while dep_stack: + dep = dep_stack.pop() + if isinstance(dep, Package): + if not self._add_pkg_deps(dep, + allow_unsatisfied=allow_unsatisfied): + return 0 + continue + if not self._add_dep(dep, allow_unsatisfied=allow_unsatisfied): + return 0 + if dep_disjunctive_stack: + if not self._pop_disjunction(allow_unsatisfied): return 0 - continue - if not self._add_dep(dep, allow_unsatisfied=allow_unsatisfied): - return 0 return 1 def _add_dep(self, dep, allow_unsatisfied=False): @@ -5381,6 +5387,9 @@ class depgraph(object): debug = "--debug" in self.myopts strict = mytype != "installed" try: + if not strict: + portage.dep._dep_check_strict = False + for dep_root, dep_string, dep_priority in deps: if not dep_string: continue @@ -5389,41 +5398,34 @@ class depgraph(object): print "Parent: ", jbigkey print "Depstring:", dep_string print "Priority:", dep_priority - vardb = self.roots[dep_root].trees["vartree"].dbapi + try: - selected_atoms = self._select_atoms(dep_root, - dep_string, myuse=myuse, parent=pkg, strict=strict, - priority=dep_priority) - except portage.exception.InvalidDependString, e: - show_invalid_depstring_notice(jbigkey, dep_string, str(e)) - return 0 - if debug: - print "Candidates:", selected_atoms - for atom in selected_atoms: - try: + dep_string = portage.dep.paren_normalize( + portage.dep.use_reduce( + portage.dep.paren_reduce(dep_string), + uselist=pkg.use.enabled)) - atom = portage.dep.Atom(atom) + dep_string = list(self._queue_disjunctive_deps( + pkg, dep_root, dep_priority, dep_string)) - mypriority = dep_priority.copy() - if not atom.blocker and vardb.match(atom): - mypriority.satisfied = True + except portage.exception.InvalidDependString, e: + if pkg.installed: + del e + continue + show_invalid_depstring_notice(pkg, dep_string, str(e)) + return 0 - if not self._add_dep(Dependency(atom=atom, - blocker=atom.blocker, depth=depth, parent=pkg, - priority=mypriority, root=dep_root), - allow_unsatisfied=allow_unsatisfied): - return 0 + if not dep_string: + continue - except portage.exception.InvalidAtom, e: - show_invalid_depstring_notice( - pkg, dep_string, str(e)) - del e - if not pkg.installed: - return 0 + dep_string = portage.dep.paren_enclose(dep_string) + + if not self._add_pkg_dep_string( + pkg, dep_root, dep_priority, dep_string, + allow_unsatisfied): + return 0 - if debug: - print "Exiting...", jbigkey except portage.exception.AmbiguousPackageName, e: pkgs = e.args[0] portage.writemsg("\n\n!!! An atom in the dependencies " + \ @@ -5443,6 +5445,115 @@ class depgraph(object): portage.writemsg("!!! Please notify the package maintainer " + \ "that atoms must be fully-qualified.\n", noiselevel=-1) return 0 + finally: + portage.dep._dep_check_strict = True + return 1 + + def _add_pkg_dep_string(self, pkg, dep_root, dep_priority, dep_string, + allow_unsatisfied): + depth = pkg.depth + 1 + debug = "--debug" in self.myopts + strict = pkg.type_name != "installed" + + if debug: + print + print "Parent: ", pkg + print "Depstring:", dep_string + print "Priority:", dep_priority + + try: + selected_atoms = self._select_atoms(dep_root, + dep_string, myuse=pkg.use.enabled, parent=pkg, + strict=strict, priority=dep_priority) + except portage.exception.InvalidDependString, e: + show_invalid_depstring_notice(pkg, dep_string, str(e)) + del e + if pkg.installed: + return 1 + return 0 + + if debug: + print "Candidates:", selected_atoms + + vardb = self.roots[dep_root].trees["vartree"].dbapi + + for atom in selected_atoms: + try: + + atom = portage.dep.Atom(atom) + + mypriority = dep_priority.copy() + if not atom.blocker and vardb.match(atom): + mypriority.satisfied = True + + if not self._add_dep(Dependency(atom=atom, + blocker=atom.blocker, depth=depth, parent=pkg, + priority=mypriority, root=dep_root), + allow_unsatisfied=allow_unsatisfied): + return 0 + + except portage.exception.InvalidAtom, e: + show_invalid_depstring_notice( + pkg, dep_string, str(e)) + del e + if not pkg.installed: + return 0 + + if debug: + print "Exiting...", pkg + + return 1 + + def _queue_disjunctive_deps(self, pkg, dep_root, dep_priority, dep_struct): + """ + Queue disjunctive (virtual and ||) deps in self._dep_disjunctive_stack. + Yields non-disjunctive deps. Raises InvalidDependString when + necessary. + """ + i = 0 + while i < len(dep_struct): + x = dep_struct[i] + if isinstance(x, list): + for y in self._queue_disjunctive_deps( + pkg, dep_root, dep_priority, x): + yield y + elif x == "||": + self._queue_disjunction(pkg, dep_root, dep_priority, + [ x, dep_struct[ i + 1 ] ] ) + i += 1 + else: + try: + x = portage.dep.Atom(x) + except portage.exception.InvalidAtom: + if not pkg.installed: + raise portage.exception.InvalidDependString( + "invalid atom: '%s'" % x) + else: + # Note: Eventually this will check for PROPERTIES=virtual + # or whatever other metadata gets implemented for this + # purpose. + if x.cp.startswith('virtual/'): + self._queue_disjunction( pkg, dep_root, + dep_priority, [ str(x) ] ) + else: + yield str(x) + i += 1 + + def _queue_disjunction(self, pkg, dep_root, dep_priority, dep_struct): + self._dep_disjunctive_stack.append( + (pkg, dep_root, dep_priority, dep_struct)) + + def _pop_disjunction(self, allow_unsatisfied): + """ + Pop one disjunctive dep from self._dep_disjunctive_stack, and use it to + populate self._dep_stack. + """ + pkg, dep_root, dep_priority, dep_struct = \ + self._dep_disjunctive_stack.pop() + dep_string = portage.dep.paren_enclose(dep_struct) + if not self._add_pkg_dep_string( + pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): + return 0 return 1 def _priority(self, **kwargs): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index b5c2abf05..8196b8f5c 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5442,7 +5442,11 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m mysettings["PORTAGE_BUILDDIR"], ".exit_status") #set up KV variable -- DEP SPEEDUP :: Don't waste time. Keep var persistent. - if mydo != 'depend' and 'KV' not in mysettings and \ + if eapi not in ('0', '1', '2'): + # Discard KV for EAPIs that don't support it. Cache KV is restored + # from the backupenv whenever config.reset() is called. + mysettings.pop('KV', None) + elif mydo != 'depend' and 'KV' not in mysettings and \ mydo in ('compile', 'config', 'configure', 'info', 'install', 'nofetch', 'postinst', 'postrm', 'preinst', 'prepare', 'prerm', 'setup', 'test', 'unpack'): diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py index f3dc0ce44..d41749186 100644 --- a/pym/portage/cache/fs_template.py +++ b/pym/portage/cache/fs_template.py @@ -5,7 +5,14 @@ import os from portage.cache import template -from portage.data import portage_gid + +import portage.proxy.lazyimport +import portage.proxy as proxy +proxy.lazyimport.lazyimport(globals(), + 'portage.data:portage_gid', + 'portage.exception:PortageException', + 'portage.util:apply_permissions', +) class FsBased(template.database): """template wrapping fs needed options, and providing _ensure_access as a way to @@ -17,7 +24,7 @@ class FsBased(template.database): gid=portage_gid perms=0665""" - for x,y in (("gid",portage_gid),("perms",0664)): + for x, y in (("gid", -1), ("perms", -1)): if x in config: setattr(self, "_"+x, config[x]) del config[x] @@ -34,12 +41,11 @@ class FsBased(template.database): """returns true or false if it's able to ensure that path is properly chmod'd and chowned. if mtime is specified, attempts to ensure that's correct also""" try: - os.chown(path, -1, self._gid) - os.chmod(path, self._perms) + apply_permissions(path, gid=self._gid, mode=self._perms) if mtime != -1: mtime=long(mtime) os.utime(path, (mtime, mtime)) - except (OSError, IOError): + except (PortageException, EnvironmentError): return False return True @@ -55,12 +61,19 @@ class FsBased(template.database): for dir in path.lstrip(os.path.sep).rstrip(os.path.sep).split(os.path.sep): base = os.path.join(base,dir) if not os.path.exists(base): - um=os.umask(0) + if self._perms != -1: + um = os.umask(0) try: - os.mkdir(base, self._perms | 0111) - os.chown(base, -1, self._gid) + perms = self._perms + if perms == -1: + perms = 0 + perms |= 0755 + os.mkdir(base, perms) + if self._gid != -1: + os.chown(base, -1, self._gid) finally: - os.umask(um) + if self._perms != -1: + os.umask(um) def gen_label(base, label): diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index ea77391c7..5657617b3 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -62,7 +62,7 @@ class database(fs_template.FsBased): database=self._dbpath, **connection_kwargs) self._db_cursor = self._db_connection.cursor() self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) - if not apply_secpass_permissions(self._dbpath, gid=portage_gid, mode=070, mask=02): + if not self._ensure_access(self._dbpath): raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath) self._db_init_cache_size(config["cache_bytes"]) self._db_init_synchronous(config["synchronous"]) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 0cdb36591..ff16a9441 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -295,12 +295,23 @@ class portdbapi(dbapi): self.auxdb = {} self._pregen_auxdb = {} self._init_cache_dirs() + depcachedir_w_ok = os.access(self.depcachedir, os.W_OK) + cache_kwargs = { + 'gid' : portage_gid, + 'perms' : 0664 + } + + if secpass < 1: + # portage_gid is irrelevant, so just obey umask + cache_kwargs['gid'] = -1 + cache_kwargs['perms'] = -1 + # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys # ~harring filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys) filtered_auxdbkeys.sort() from portage.cache import metadata_overlay, volatile - if secpass < 1: + if not depcachedir_w_ok: for x in self.porttrees: db_ro = self.auxdbmodule(self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid, readonly=True) @@ -314,7 +325,7 @@ class portdbapi(dbapi): continue # location, label, auxdbkeys self.auxdb[x] = self.auxdbmodule( - self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid) + self.depcachedir, x, filtered_auxdbkeys, **cache_kwargs) if self.auxdbmodule is metadata_overlay.database: self.auxdb[x].db_ro.ec = self._repo_info[x].eclass_db if "metadata-transfer" not in self.mysettings.features: diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py index 7dfbc8124..4343ab119 100644 --- a/pym/repoman/checks.py +++ b/pym/repoman/checks.py @@ -92,7 +92,7 @@ class EbuildQuote(LineCheck): var_names = ["D", "DISTDIR", "FILESDIR", "S", "T", "ROOT", "WORKDIR"] # variables for games.eclass - var_names += ["Ddir", "dir", "GAMES_PREFIX_OPT", "GAMES_DATADIR", + var_names += ["Ddir", "GAMES_PREFIX_OPT", "GAMES_DATADIR", "GAMES_DATADIR_BASE", "GAMES_SYSCONFDIR", "GAMES_STATEDIR", "GAMES_LOGDIR", "GAMES_BINDIR"] -- 2.26.2