From: Fabian Groffen Date: Thu, 10 Apr 2008 16:44:23 +0000 (-0000) Subject: Merged from trunk 9697:9725 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=171a081c11da3ac384c30081260522297d4426f6;p=portage.git Merged from trunk 9697:9725 | 9699 | When the user has given --depclean a list of atoms to clean, | | zmedico | pull all installed packages into the graph (not just those | | | traversable from system or world) to ensure that packages | | | can not be uninstalled unless nothing requires them. | | 9700 | Fix a typo. | | zmedico | | | 9702 | Fix a typo in the depclean USE flag handling when populating | | zmedico | the digraph for bug #201045. | | 9705 | Display the note referencing "masked packages" docs when | | zmedico | there are installed masked packages to display. | | 9706 | Adjust newlines whitespace masked packages display. | | zmedico | | | 9708 | * In the event that a previously installed package has since | | zmedico | been masked, emerge will no longer perform an automatic | | | downgrade as part of a "world" update. You should either | | | unmask such packages or else explicitly re-merge them in | | | order to have them dowgraded to an unmasked version. | | 9710 | Add missing "the" to fix grammar. | | zmedico | | | 9712 | In dbapi.move_ent(), never assume that any of the expected | | zmedico | metadata entries exist (PF, CATEGORY, or *.ebuild). | | 9713 | Replace the depgraph's fakedbapi of visible packages with a | | zmedico | new CompositeDbapi that is implemented on top of the | | | depgraph's existing package selection logic. | | 9714 | Remove "selective" depgraph logic from dep_zapdeps() since | | zmedico | it's now handled by CompositeDbapi. | | 9715 | Memoize package selection results in the depgraph for better | | zmedico | performance. | | 9716 | Fix a typo. | | zmedico | | | 9717 | Fix the depgraph package selection cache to update the | | zmedico | "existing node" state of packages if they've been added to | | | the graph since the last call. | | 9718 | Bug #216328 - Fix broken --newuse reinstallation logic. | | zmedico | | | 9720 | Handle a potential InvalidDependString exception inside | | zmedico | CompositeDbapi.match(). | | 9723 | When necessary, make quickpkg add missing metadata to the | | zmedico | vdb entry before creating the binary package. | | 9725 | Clarify --with-bdeps behavior with example. Thanks to | | zmedico | Griffon26 for this patch. | svn path=/main/branches/prefix/; revision=9804 --- diff --git a/RELEASE-NOTES b/RELEASE-NOTES index c107bbce3..788f9118d 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -32,6 +32,13 @@ portage-2.2 * "world" does no longer include "system" unconditionally, but you can add "@system" to the worldfile to restore the old state. +portage-2.1.5 +================================== +* In the event that a previously installed package has since been masked, + emerge will no longer perform an automatic downgrade as part of a "world" + update. You should either unmask such packages or else explicitly re-merge + them in order to have them dowgraded to an unmasked version. + portage-2.1.4.1 ================================== diff --git a/bin/quickpkg b/bin/quickpkg index 08332cc4d..419e40f96 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -4,6 +4,7 @@ # $Id$ import errno, signal, sys, os +from itertools import izip # for an explanation on this logic, see pym/_emerge/__init__.py import os @@ -39,6 +40,7 @@ def quickpkg_main(options, args, eout): config_files_excluded = 0 include_config = options.include_config == "y" include_unmodified_config = options.include_unmodified_config == "y" + fix_metadata_keys = ["PF", "CATEGORY"] for arg in args: try: atom = dep_expand(arg, mydb=vardb, settings=vartree.settings) @@ -110,6 +112,18 @@ def quickpkg_main(options, args, eout): return False excluded_config_files.append(filename) return True + existing_metadata = dict(izip(fix_metadata_keys, + vardb.aux_get(cpv, fix_metadata_keys))) + category, pf = portage.catsplit(cpv) + required_metadata = {} + required_metadata["CATEGORY"] = category + required_metadata["PF"] = pf + update_metadata = {} + for k, v in required_metadata.iteritems(): + if v != existing_metadata[k]: + update_metadata[k] = v + if update_metadata: + vardb.aux_update(cpv, update_metadata) xpdata = xpak.xpak(dblnk.dbdir) binpkg_tmpfile = os.path.join(bintree.pkgdir, cpv + ".tbz2." + str(os.getpid())) diff --git a/man/emerge.1 b/man/emerge.1 index d019eb54a..6df5af075 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -412,9 +412,11 @@ in order to indicate their status: \fI() circumfix\fR = forced, masked, or removed .TP .BR "\-\-with\-bdeps < y | n >" -In dependency calculations, pull in build time dependencies that are not -strictly required. This defaults to \'n\' for installation actions and -\'y\' for the \fB\-\-depclean\fR action. This setting can be added to +In dependency calculations, pull in build time dependencies +that are not strictly required. This defaults to \'n\' for +installation actions, meaning they will not be installed, and +\'y\' for the \fB\-\-depclean\fR action, meaning they will not be removed. +This setting can be added to \fBEMERGE_DEFAULT_OPTS\fR (see make.conf(5)) and later overridden via the command line. .SH "ENVIRONMENT OPTIONS" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index b8aad59aa..c50cf24c9 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1447,6 +1447,41 @@ def show_invalid_depstring_notice(parent_node, depstring, error_msg): f.add_flowing_data(x) f.end_paragraph(1) +class CompositeDbapi(object): + def __init__(self, depgraph, root): + self._depgraph = depgraph + self._root = root + self._match_cache = {} + self._cpv_tree_map = {} + + def match(self, atom): + ret = self._match_cache.get(atom) + if ret is not None: + return ret[:] + pkg, existing = self._depgraph._select_package(self._root, atom) + if not pkg: + ret = [] + else: + if pkg.installed and "selective" not in self._depgraph.myparams: + try: + self._depgraph._iter_args_for_pkg(pkg).next() + except StopIteration: + pass + except portage.exception.InvalidDependString: + pass + else: + ret = [] + if ret is None: + self._cpv_tree_map[pkg.cpv] = \ + self._depgraph.pkg_tree_map[pkg.type_name] + ret = [pkg.cpv] + self._match_cache[atom] = ret + return ret[:] + + def aux_get(self, cpv, wants): + return self._depgraph.trees[self._root][ + self._cpv_tree_map[cpv]].dbapi.aux_get(cpv, wants) + class depgraph(object): pkg_tree_map = { @@ -1525,10 +1560,8 @@ class depgraph(object): self._filtered_trees[myroot]["vartree"] = self.trees[myroot]["vartree"] def filtered_tree(): pass - filtered_tree.dbapi = portage.fakedbapi( - settings=self.pkgsettings[myroot], exclusive_slots=False) + filtered_tree.dbapi = CompositeDbapi(self, myroot) self._filtered_trees[myroot]["porttree"] = filtered_tree - self._filtered_trees[myroot]["atoms"] = set() dbs = [] portdb = self.trees[myroot]["porttree"].dbapi bindb = self.trees[myroot]["bintree"].dbapi @@ -1578,6 +1611,8 @@ class depgraph(object): self._required_set_names = set(["system", "world"]) self._select_atoms = self._select_atoms_highest_available self._select_package = self._select_pkg_highest_available + self._highest_pkg_cache = {} + self._installed_pkg_cache = {} def _show_slot_collision_notice(self): """Show an informational message advising the user to mask one of the @@ -2009,7 +2044,6 @@ class depgraph(object): """Generate SLOT atoms for the highest available match and any matching installed SLOTs that are also available.""" vardb = self.roots[root].trees["vartree"].dbapi - filtered_db = self._filtered_trees[root]["porttree"].dbapi mykey = portage.dep_getkey(atom) myslots = set() for cpv in vardb.match(mykey): @@ -2067,7 +2101,6 @@ class depgraph(object): myfavorites=[] myroot = self.target_root dbs = self._filtered_trees[myroot]["dbs"] - filtered_db = self._filtered_trees[myroot]["porttree"].dbapi vardb = self.trees[myroot]["vartree"].dbapi portdb = self.trees[myroot]["porttree"].dbapi bindb = self.trees[myroot]["bintree"].dbapi @@ -2386,129 +2419,6 @@ class depgraph(object): # We're true here unless we are missing binaries. return (not missing,myfavorites) - def _populate_filtered_repo(self, myroot, depstring, - strict=True, myuse=None, exclude_installed=False): - """Extract all of the atoms from the depstring, select preferred - packages from appropriate repositories, and use them to populate - the filtered repository. This will raise InvalidDependString when - necessary.""" - - filtered_db = self._filtered_trees[myroot]["porttree"].dbapi - pkgsettings = self.pkgsettings[myroot] - usepkgonly = "--usepkgonly" in self.myopts - - from portage.dep import paren_reduce, use_reduce - try: - portage.dep._dep_check_strict = strict - atoms = paren_reduce(depstring) - atoms = use_reduce(atoms, uselist=myuse) - atoms = list(iter_atoms(atoms)) - for x in atoms: - if portage.dep._dep_check_strict and \ - not portage.isvalidatom(x, allow_blockers=True): - raise portage.exception.InvalidDependString( - "Invalid atom: %s" % x) - finally: - portage.dep._dep_check_strict = True - - filtered_atoms = self._filtered_trees[myroot]["atoms"] - dbs = self._filtered_trees[myroot]["dbs"] - old_virts = pkgsettings.getvirtuals() - while atoms: - x = atoms.pop() - if x.startswith("!"): - continue - if x in filtered_atoms: - continue - filtered_atoms.add(x) - cp = portage.dep_getkey(x) - cat = portage.catsplit(cp)[0] - slot = portage.dep.dep_getslot(x) - is_virt = cp.startswith("virtual/") - atom_populated = False - for db, pkg_type, built, installed, db_keys in dbs: - if installed and \ - (exclude_installed or not usepkgonly): - continue - cpv_list = db.cp_list(cp) - if not cpv_list: - if is_virt: - # old-style virtual - # Create a transformed atom for each choice - # and add it to the stack for processing. - for choice in old_virts.get(cp, []): - atoms.append(x.replace(cp, choice)) - # Maybe a new-style virtual exists in another db, so - # we have to try all of them to prevent the old-style - # virtuals from overriding available new-styles. - continue - # descending order - cpv_list.reverse() - for cpv in cpv_list: - if filtered_db.cpv_exists(cpv): - continue - if not portage.match_from_list(x, [cpv]): - continue - if is_virt: - mykeys = db_keys[:] - mykeys.extend(self._dep_keys) - else: - mykeys = db_keys - try: - metadata = dict(izip(mykeys, - db.aux_get(cpv, mykeys))) - except KeyError: - # masked by corruption - continue - if slot is not None: - if slot != metadata["SLOT"]: - continue - if not built: - if (is_virt or "?" in metadata["LICENSE"]): - pkgsettings.setcpv(cpv, mydb=metadata) - metadata["USE"] = pkgsettings["PORTAGE_USE"] - else: - metadata["USE"] = "" - - if not visible(pkgsettings, Package(built=built, - cpv=cpv, root=myroot, type_name=pkg_type, - installed=installed, metadata=metadata)): - continue - - filtered_db.cpv_inject(cpv, metadata=metadata) - if not is_virt: - # break here since we only want the best version - # for now (eventually will be configurable). - atom_populated = True - break - # For new-style virtuals, we explore all available - # versions and recurse on their deps. This is a - # preparation for the lookahead that happens when - # new-style virtuals are expanded by dep_check(). - virtual_deps = " ".join(metadata[k] \ - for k in self._dep_keys) - try: - if installed: - portage.dep._dep_check_strict = False - try: - deps = paren_reduce(virtual_deps) - deps = use_reduce(deps, - uselist=metadata["USE"].split()) - for y in iter_atoms(deps): - if portage.dep._dep_check_strict and \ - not portage.isvalidatom(y, - allow_blockers=True): - raise portage.exception.InvalidDependString( - "Invalid atom: %s" % y) - atoms.append(y) - except portage.exception.InvalidDependString, e: - # Masked by corruption - filtered_db.cpv_remove(cpv) - finally: - portage.dep._dep_check_strict = True - if atom_populated: - break - def _select_atoms_from_graph(self, *pargs, **kwargs): """ Prefer atoms matching packages that have already been @@ -2525,18 +2435,14 @@ class depgraph(object): pkgsettings = self.pkgsettings[root] if trees is None: trees = self._filtered_trees - self._populate_filtered_repo(root, depstring, - myuse=myuse, strict=strict) if True: try: - self.trees[root]["selective"] = "selective" in self.myparams if not strict: portage.dep._dep_check_strict = False mycheck = portage.dep_check(depstring, None, pkgsettings, myuse=myuse, myroot=root, trees=trees) finally: - self.trees[root]["selective"] = False portage.dep._dep_check_strict = True if not mycheck[0]: raise portage.exception.InvalidDependString(mycheck[1]) @@ -2594,8 +2500,7 @@ class depgraph(object): for line in wrap(msg, 75): print line print - print "For more information, see MASKED PACKAGES section in the emerge man page or " - print "refer to the Gentoo Handbook." + show_mask_docs() else: print "\nemerge: there are no ebuilds to satisfy "+green(xinfo)+"." if myparent: @@ -2603,6 +2508,23 @@ class depgraph(object): print def _select_pkg_highest_available(self, root, atom, onlydeps=False): + cache_key = (root, atom, onlydeps) + ret = self._highest_pkg_cache.get(cache_key) + if ret is not None: + pkg, existing = ret + if pkg and not existing: + existing = self._slot_pkg_map[root].get(pkg.slot_atom) + if existing and existing == pkg: + # Update the cache to reflect that the + # package has been added to the graph. + ret = pkg, pkg + self._highest_pkg_cache[cache_key] = ret + return ret + ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) + self._highest_pkg_cache[cache_key] = ret + return ret + + def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False): pkgsettings = self.pkgsettings[root] dbs = self._filtered_trees[root]["dbs"] vardb = self.roots[root].trees["vartree"].dbapi @@ -2616,6 +2538,7 @@ class depgraph(object): usepkgonly = "--usepkgonly" in self.myopts empty = "empty" in self.myparams selective = "selective" in self.myparams + reinstall = False noreplace = "--noreplace" in self.myopts # Behavior of the "selective" parameter depends on # whether or not a package matches an argument atom. @@ -2636,7 +2559,7 @@ class depgraph(object): if existing_node: break if installed and not find_existing_node: - want_reinstall = empty or \ + want_reinstall = reinstall or empty or \ (found_available_arg and not selective) if want_reinstall and matched_packages: continue @@ -2761,6 +2684,8 @@ class depgraph(object): self._reinstall_for_flags( forced_flags, old_use, old_iuse, cur_use, cur_iuse) + if reinstall_for_flags: + reinstall = True if not installed: must_reinstall = empty or \ (myarg and not selective) @@ -2836,12 +2761,18 @@ class depgraph(object): e_pkg = self._slot_pkg_map[root].get(slot_atom) if e_pkg: return e_pkg, e_pkg + cache_key = (root, atom, onlydeps) + ret = self._installed_pkg_cache.get(cache_key) + if ret is not None: + return ret metadata = dict(izip(self._mydbapi_keys, graph_db.aux_get(cpv, self._mydbapi_keys))) pkg = Package(cpv=cpv, built=True, installed=True, type_name="installed", metadata=metadata, root=root) - return pkg, None + ret = (pkg, None) + self._installed_pkg_cache[cache_key] = ret + return ret def _complete_graph(self): """ @@ -4134,17 +4065,6 @@ class depgraph(object): # TODO: Add generic support for "set problem" handlers so that # the below warnings aren't special cases for world only. - masked_packages = [] - for pkg, pkgsettings in self._masked_installed: - root_config = self.roots[pkg.root] - mreasons = get_masking_status(pkg, pkgsettings, root_config) - masked_packages.append((root_config, pkgsettings, - pkg.cpv, pkg.metadata, mreasons)) - if masked_packages: - sys.stderr.write("\n" + colorize("BAD", "!!!") + \ - " The following installed packages are masked:\n") - show_masked_packages(masked_packages) - if self._missing_args: world_problems = False if "world" in self._sets: @@ -4207,6 +4127,19 @@ class depgraph(object): msg.append("package.provided entry exists.\n\n") sys.stderr.write("".join(msg)) + masked_packages = [] + for pkg, pkgsettings in self._masked_installed: + root_config = self.roots[pkg.root] + mreasons = get_masking_status(pkg, pkgsettings, root_config) + masked_packages.append((root_config, pkgsettings, + pkg.cpv, pkg.metadata, mreasons)) + if masked_packages: + sys.stderr.write("\n" + colorize("BAD", "!!!") + \ + " The following installed packages are masked:\n") + show_masked_packages(masked_packages) + show_mask_docs() + print + for pargs, kwargs in self._unsatisfied_deps_for_display: self._show_unsatisfied_dep(*pargs, **kwargs) @@ -5632,6 +5565,10 @@ def show_blocker_docs_link(): print "http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?full=1#blocked" print +def show_mask_docs(): + print "For more information, see the MASKED PACKAGES section in the emerge" + print "man page or refer to the Gentoo Handbook." + def action_sync(settings, trees, mtimedb, myopts, myaction): xterm_titles = "notitles" not in settings.features emergelog(xterm_titles, " === sync") @@ -6631,12 +6568,18 @@ def action_depclean(settings, trees, ldpath_mtimes, remaining_atoms = [] if action == "depclean": - for atom in worldlist: - if vardb.match(atom): - remaining_atoms.append((atom, 'world', runtime)) for atom in syslist: if vardb.match(atom): remaining_atoms.append((atom, 'system', runtime)) + if myfiles: + # Pull in everything that's installed since we don't want + # to clean any package if something depends on it. + remaining_atoms.extend( + ("="+cpv, 'world', runtime) for cpv in vardb.cpv_all()) + else: + for atom in worldlist: + if vardb.match(atom): + remaining_atoms.append((atom, 'world', runtime)) elif action == "prune": for atom in syslist: if vardb.match(atom): @@ -6842,7 +6785,7 @@ def action_depclean(settings, trees, ldpath_mtimes, graph.add(node, None) myaux = dict(izip(aux_keys, vardb.aux_get(node, aux_keys))) mydeps = [] - usedef = vardb.aux_get(pkg, ["USE"])[0].split() + usedef = vardb.aux_get(node, ["USE"])[0].split() for dep_type, depstr in myaux.iteritems(): if not depstr: continue diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 3cecd9ef7..94729054f 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5430,7 +5430,6 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): if trees is None: global db trees = db - selective = trees[myroot].get("selective", False) writemsg("ZapDeps -- %s\n" % (use_binaries), 2) if not reduced or unreduced == ["||"] or dep_eval(reduced): return [] @@ -5491,13 +5490,6 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg = avail_pkg[-1] # highest (ascending order) avail_slot = "%s:%s" % (dep_getkey(atom), mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) - elif not avail_pkg: - if selective: - avail_pkg = vardb.match(atom) - if avail_pkg: - avail_pkg = avail_pkg[-1] # highest (ascending order) - avail_slot = "%s:%s" % (dep_getkey(atom), - vardb.aux_get(avail_pkg, ["SLOT"])[0]) if not avail_pkg: all_available = False break diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index bd5e6b7fd..9efca10a3 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -209,11 +209,13 @@ class binarytree(object): mydata = mytbz2.get_data() updated_items = update_dbentries([mylist], mydata) mydata.update(updated_items) + mydata["PF"] = mynewpkg + "\n" mydata["CATEGORY"] = mynewcat+"\n" if mynewpkg != myoldpkg: - mydata[mynewpkg+".ebuild"] = mydata[myoldpkg+".ebuild"] - del mydata[myoldpkg+".ebuild"] - mydata["PF"] = mynewpkg + "\n" + ebuild_data = mydata.get(myoldpkg+".ebuild") + if ebuild_data is not None: + mydata[mynewpkg+".ebuild"] = ebuild_data + del mydata[myoldpkg+".ebuild"] mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) self.dbapi.cpv_remove(mycpv) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 0155cea6d..cc1db12fb 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -345,8 +345,7 @@ class vardbapi(dbapi): if e.errno != errno.ENOENT: raise del e - write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") - + write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") fixdbentries([mylist], newpath) return moves