Merged from trunk 9741:9774
authorFabian Groffen <grobian@gentoo.org>
Thu, 10 Apr 2008 16:52:27 +0000 (16:52 -0000)
committerFabian Groffen <grobian@gentoo.org>
Thu, 10 Apr 2008 16:52:27 +0000 (16:52 -0000)
   | 9742    | When reading and writing the Packages index file, translate  |
   | zmedico | metadata keys on the fly. Current translations: DESC         |
   |         | DESCRIPTION REPO repository                                  |

   | 9743    | For repoman, only cache the metadata necessary for           |
   | zmedico | visibility checks in order to reduce memory consumption.     |

   | 9744    | Remove unused iter_atoms() function.                         |
   | zmedico |                                                              |

   | 9745    | Remove the depgraph._get_arg_for_pkg() method since          |
   | zmedico | _iter_atoms_for_pkg() does the job.                          |

   | 9747    | Fix a little issue in the code from bug #197905 that causes  |
   | zmedico | some elog messages to get lost.                              |

   | 9755    | Fix --update greedy atoms logic so that it only pulls in     |
   | zmedico | packages that actually match the given atom.                 |

   | 9756    | Bug #197810 - Don't automatically downgrade packages for     |
   | zmedico | which available matching versions are masked by corruption,  |
   |         | but do automatically downgrade installed packages that       |
   |         | themselves have a valid mask such as from package.mask.      |

   | 9759    | Bug #162542 - When checking permissions inside CCACHE_DIR    |
   | zmedico | for compatibility with FEATURES=userpriv, check the          |
   |         | permissions on files inside the directory to decide whether  |
   |         | to fix the permissions.                                      |

   | 9761    | Split out a depgraph method to check if a specific new-style |
   | zmedico | virtual package exists.                                      |

   | 9762    | Make depgraph.select_files() detect when a package name      |
   | zmedico | given as an argument is ambiguous due to the existence of a  |
   |         | new-style virtual with the same name (cpv_expand() will not  |
   |         | always raise a ValueError in cases like this).               |

   | 9763    | Handle issues with newlines in elog messages that can        |
   | zmedico | trigger an unhandled ValueError to be raised from a split()  |
   |         | call inside collect_ebuild_messages(): * Use \0 to delimit   |
   |         | messages, so that that elog messages containing newlines are |
   |         | handled correctly. * Handle a potential ValueError when      |
   |         | splitting the message type.                                  |

   | 9765    | Create a new depgraph._dep_expand() method for               |
   | zmedico | select_files() to expand atoms that have no category.        |

   | 9769    | When depgraph.select_files() expands an atom category, in    |
   | zmedico | case of an old-style virtual, defer category expansion so    |
   |         | that the depgraph has more choices.                          |

   | 9771    | Make DepcheckCompositeDB cache Package instances for         |
   | zmedico | aux_get() calls.                                             |

   | 9774    | When pruning the list of parents for the slot collision      |
   | zmedico | display, always show parents that themselves have been       |
   |         | pulled into collision slots, since those are more relevant   |
   |         | that others. (branches/2.1.2 r9773)                          |

svn path=/main/branches/prefix/; revision=9806

RELEASE-NOTES
bin/isolated-functions.sh
bin/repoman
pym/_emerge/__init__.py
pym/portage/__init__.py
pym/portage/dbapi/bintree.py
pym/portage/elog/messages.py
pym/portage/getbinpkg.py

index 788f9118d6a42791fb746b35265f095942635502..c107bbce3dc0da24613b44730a9972360857d461 100644 (file)
@@ -32,13 +32,6 @@ portage-2.2
 * "world" does no longer include "system" unconditionally, but you can add
   "@system" to the worldfile to restore the old state.
 
-portage-2.1.5
-==================================
-* In the event that a previously installed package has since been masked,
-  emerge will no longer perform an automatic downgrade as part of a "world"
-  update. You should either unmask such packages or else explicitly re-merge
-  them in order to have them dowgraded to an unmasked version.
-
 portage-2.1.4.1
 ==================================
 
index aa5d73d720e3185cfe2ec590cd7ccea5ae4dbe32..343d6265700e9a9f222e9af8d5277e3178d751c3 100755 (executable)
@@ -177,7 +177,7 @@ elog_base() {
                        return 1
                        ;;
        esac
-       echo -e "${messagetype} $*" >> "${T}/logging/${EBUILD_PHASE:-other}"
+       echo -ne "${messagetype} $*\n\0" >> "${T}/logging/${EBUILD_PHASE:-other}"
        return 0
 }
 
index 74adcca7540b45ff41e2ad4233d6be4b75e20129..2954a15dab5e62cba5640b42782c8abb4da8f5a1 100755 (executable)
@@ -472,6 +472,11 @@ trees = portage.create_trees()
 trees["/"]["porttree"].settings = repoman_settings
 portdb = trees["/"]["porttree"].dbapi
 portdb.mysettings = repoman_settings
+# We really only need to cache the metadata that's necessary for visibility
+# filtering. Anything else can be discarded to reduce memory consumption.
+for k in ("DEPEND", "IUSE", "LICENCE", "PDEPEND",
+       "PROVIDE", "RDEPEND", "RESTRICT", "repository"):
+       portdb._aux_cache_keys.discard(k)
 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 del trees["/"]["vartree"]
 
index 7a8a802c88e15ea14d07c491164076e59e87343a..95454799743f791b2c026bd9fabbc069616265fc 100644 (file)
@@ -1193,21 +1193,6 @@ def show_masked_packages(masked_packages):
                        shown_licenses.add(l)
        return have_eapi_mask
 
-def iter_atoms(deps):
-       """Take a dependency structure as returned by paren_reduce or use_reduce
-       and iterate over all the atoms."""
-       i = iter(deps)
-       for x in i:
-               if isinstance(x, basestring):
-                       if x == '||' or x.endswith('?'):
-                               for x in iter_atoms(i.next()):
-                                       yield x
-                       else:
-                               yield x
-               else:
-                       for x in iter_atoms(x):
-                               yield x
-
 class Package(object):
        __slots__ = ("__weakref__", "built", "cpv", "depth",
                "installed", "metadata", "root", "onlydeps", "type_name",
@@ -1452,7 +1437,7 @@ class DepcheckCompositeDB(object):
                self._depgraph = depgraph
                self._root = root
                self._match_cache = {}
-               self._cpv_tree_map = {}
+               self._cpv_pkg_map = {}
 
        def match(self, atom):
                ret = self._match_cache.get(atom)
@@ -1469,16 +1454,20 @@ class DepcheckCompositeDB(object):
                                        arg = None
                                if arg:
                                        ret = []
+                       if ret is None and pkg.installed and \
+                               not visible(self._depgraph.pkgsettings[pkg.root], pkg):
+                               # For disjunctive || deps, this will cause alternative
+                               # atoms or packages to be selected if available.
+                               ret = []
                        if ret is None:
-                               self._cpv_tree_map[pkg.cpv] = \
-                                       self._depgraph.pkg_tree_map[pkg.type_name]
+                               self._cpv_pkg_map[pkg.cpv] = pkg
                                ret = [pkg.cpv]
                self._match_cache[atom] = ret
                return ret[:]
 
        def aux_get(self, cpv, wants):
-               return self._depgraph.trees[self._root][
-                       self._cpv_tree_map[cpv]].dbapi.aux_get(cpv, wants)
+               metadata = self._cpv_pkg_map[cpv].metadata
+               return [metadata.get(x, "") for x in wants]
 
 class depgraph(object):
 
@@ -1653,16 +1642,25 @@ class depgraph(object):
                                                for parent in parents:
                                                        if isinstance(parent, DependencyArg):
                                                                pruned_list.append(parent)
-                                                               if len(pruned_list) == max_parents:
-                                                                       break
+                                               # Prefer Packages instances that themselves have been
+                                               # pulled into collision slots.
+                                               for parent in parents:
+                                                       if isinstance(parent, Package) and \
+                                                               (parent.slot_atom, parent.root) \
+                                                               in self._slot_collision_info:
+                                                               pruned_list.append(parent)
                                                for parent in parents:
                                                        if not isinstance(parent, DependencyArg):
                                                                pruned_list.append(parent)
-                                                               if len(pruned_list) == max_parents:
+                                                               if len(pruned_list) >= max_parents:
                                                                        break
                                                parents = pruned_list
                                        msg.append(" pulled in by\n")
+                                       shown_parents = set()
                                        for parent in parents:
+                                               if parent in shown_parents:
+                                                       continue
+                                               shown_parents.add(parent)
                                                msg.append(2*indent)
                                                msg.append(str(parent))
                                                msg.append("\n")
@@ -1770,7 +1768,9 @@ class depgraph(object):
                                myarg = None
                                if dep.root == self.target_root:
                                        try:
-                                               myarg = self._get_arg_for_pkg(dep_pkg)
+                                               myarg = self._iter_atoms_for_pkg(dep_pkg).next()
+                                       except StopIteration:
+                                               pass
                                        except portage.exception.InvalidDependString:
                                                if not dep_pkg.installed:
                                                        # This shouldn't happen since the package
@@ -2040,6 +2040,40 @@ class depgraph(object):
                        return 0
                return 1
 
+       def _dep_expand(self, root_config, atom_without_category):
+               """
+               @param root_config: a root config instance
+               @type root_config: RootConfig
+               @returns: a list of cat/pn for any matching packages
+               """
+               null_cp = portage.dep_getkey(insert_category_into_atom(
+                       atom_without_category, "null"))
+               cat, atom_pn = portage.catsplit(null_cp)
+
+               cp_set = set()
+               for db, pkg_type, built, installed, db_keys in \
+                       self._filtered_trees[root_config.root]["dbs"]:
+                       cp_set.update(db.cp_all())
+               for cp in list(cp_set):
+                       cat, pn = portage.catsplit(cp)
+                       if pn != atom_pn:
+                               cp_set.discard(cp)
+               deps = []
+               for cp in cp_set:
+                       cat, pn = portage.catsplit(cp)
+                       deps.append(insert_category_into_atom(
+                               atom_without_category, cat))
+               return deps
+
+       def _have_new_virt(self, root, atom_cp):
+               ret = False
+               for db, pkg_type, built, installed, db_keys in \
+                       self._filtered_trees[root]["dbs"]:
+                       if db.cp_list(atom_cp):
+                               ret = True
+                               break
+               return ret
+
        def _iter_atoms_for_pkg(self, pkg):
                # TODO: add multiple $ROOT support
                if pkg.root != self.target_root:
@@ -2047,36 +2081,15 @@ class depgraph(object):
                atom_arg_map = self._atom_arg_map
                for atom in self._set_atoms.iterAtomsForPackage(pkg):
                        atom_cp = portage.dep_getkey(atom)
-                       if atom_cp != pkg.cp:
-                               have_new_virt = False
-                               for db, pkg_type, built, installed, db_keys in \
-                                       self._filtered_trees[pkg.root]["dbs"]:
-                                       if db.cp_list(atom_cp):
-                                               have_new_virt = True
-                                               break
-                               if have_new_virt:
-                                       continue
+                       if atom_cp != pkg.cp and \
+                               self._have_new_virt(pkg.root, atom_cp):
+                               continue
                        for arg in atom_arg_map[(atom, pkg.root)]:
                                if isinstance(arg, PackageArg) and \
                                        arg.package != pkg:
                                        continue
                                yield arg, atom
 
-       def _get_arg_for_pkg(self, pkg):
-               """
-               Return a matching DependencyArg instance for the given Package if
-               any exist, otherwise None. An attempt will be made to return the most
-               specific match (PackageArg type is the most specific).
-
-               This will raise an InvalidDependString exception if PROVIDE is invalid.
-               """
-               any_arg = None
-               for arg, atom in self._iter_atoms_for_pkg(pkg):
-                       if isinstance(arg, PackageArg):
-                               return arg
-                       any_arg = arg
-               return any_arg
-
        def select_files(self, myfiles):
                """Given a list of .tbz2s, .ebuilds sets, and deps, create the
                appropriate depgraph and return a favorite list."""
@@ -2226,37 +2239,40 @@ class depgraph(object):
                                        args.append(AtomArg(arg=x, atom=x,
                                                root_config=root_config))
                                        continue
-                               try:
-                                       try:
-                                               for db, pkg_type, built, installed, db_keys in dbs:
-                                                       mykey = portage.dep_expand(x,
-                                                               mydb=db, settings=pkgsettings)
-                                                       if portage.dep_getkey(mykey).startswith("null/"):
-                                                               continue
-                                                       break
-                                       except ValueError, e:
-                                               if not e.args or not isinstance(e.args[0], list) or \
-                                                       len(e.args[0]) < 2:
-                                                       raise
-                                               mykey = portage.dep_expand(x,
-                                                       mydb=vardb, settings=pkgsettings)
-                                               cp = portage.dep_getkey(mykey)
-                                               if cp.startswith("null/") or \
-                                                       cp not in e[0]:
-                                                       raise
-                                               del e
-                                       args.append(AtomArg(arg=x, atom=mykey,
-                                               root_config=root_config))
-                               except ValueError, e:
-                                       if not e.args or not isinstance(e.args[0], list) or \
-                                               len(e.args[0]) < 2:
-                                               raise
+                               expanded_atoms = self._dep_expand(root_config, x)
+                               installed_cp_set = set()
+                               for atom in expanded_atoms:
+                                       if vardb.match(atom):
+                                               installed_cp_set.add(portage.dep_getkey(atom))
+                               if len(expanded_atoms) > 1 and len(installed_cp_set) == 1:
+                                       installed_cp = iter(installed_cp_set).next()
+                                       expanded_atoms = [atom for atom in expanded_atoms \
+                                               if portage.dep_getkey(atom) == installed_cp]
+
+                               if len(expanded_atoms) > 1:
                                        print "\n\n!!! The short ebuild name \"" + x + "\" is ambiguous.  Please specify"
                                        print "!!! one of the following fully-qualified ebuild names instead:\n"
-                                       for i in e.args[0]:
+                                       expanded_atoms = set(portage.dep_getkey(atom) \
+                                               for atom in expanded_atoms)
+                                       for i in sorted(expanded_atoms):
                                                print "    " + green(i)
                                        print
                                        return False, myfavorites
+                               if expanded_atoms:
+                                       atom = expanded_atoms[0]
+                               else:
+                                       null_atom = insert_category_into_atom(x, "null")
+                                       null_cp = portage.dep_getkey(null_atom)
+                                       cat, atom_pn = portage.catsplit(null_cp)
+                                       virts_p = root_config.settings.get_virts_p().get(atom_pn)
+                                       if virts_p:
+                                               # Allow the depgraph to choose which virtual.
+                                               atom = insert_category_into_atom(x, "virtual")
+                                       else:
+                                               atom = insert_category_into_atom(x, "null")
+
+                               args.append(AtomArg(arg=x, atom=atom,
+                                       root_config=root_config))
 
                if "--update" in self.myopts:
                        # Enable greedy SLOT atoms for atoms given as arguments.
@@ -2271,7 +2287,7 @@ class depgraph(object):
                                        continue
                                atom_cp = portage.dep_getkey(arg.atom)
                                slots = set()
-                               for cpv in vardb.match(atom_cp):
+                               for cpv in vardb.match(arg.atom):
                                        slots.add(vardb.aux_get(cpv, ["SLOT"])[0])
                                for slot in slots:
                                        greedy_atoms.append(
@@ -2603,9 +2619,15 @@ class depgraph(object):
                                                        if not installed:
                                                                # masked by corruption
                                                                continue
-                                       if not installed:
-                                               if myarg:
-                                                       found_available_arg = True
+                                       if not installed and myarg:
+                                               found_available_arg = True
+                                       if not installed or (installed and matched_packages):
+                                               # Only enforce visibility on installed packages
+                                               # if there is at least one other visible package
+                                               # available. By filtering installed masked packages
+                                               # here, packages that have been masked since they
+                                               # were installed can be automatically downgraded
+                                               # to an unmasked version.
                                                if not visible(pkgsettings, pkg):
                                                        continue
                                        if not built and not calculated_use:
@@ -5542,6 +5564,26 @@ def checkUpdatedNewsItems(portdb, vardb, NEWS_PATH, UNREAD_PATH, repo_id):
        manager = NewsManager(portdb, vardb, NEWS_PATH, UNREAD_PATH)
        return manager.getUnreadItems( repo_id, update=True )
 
+def expand_virtual_atom(x):
+       """
+       Take an atom without a category and insert virtual/ for the
+       category. This works correctly with atoms that have operators.
+
+       @param x: an atom without a category
+       @type x: String
+       @returns: the atom with virtual/ inserted for the category, or None
+       """
+       return insert_category_into_atom(atom, "virtual")
+
+def insert_category_into_atom(atom, category):
+       alphanum = re.search(r'\w', atom)
+       if alphanum:
+               ret = atom[:alphanum.start()] + "%s/" % category + \
+                       atom[alphanum.start():]
+       else:
+               ret = None
+       return ret
+
 def is_valid_package_atom(x):
        if "/" not in x:
                alphanum = re.search(r'\w', x)
index 94729054f4f63d7c12b33840c24a5535f6696dab..2e81a3b61f095abea3b331e9af6954c6d1b860b3 100644 (file)
@@ -2538,10 +2538,10 @@ class config(object):
 
                self.already_in_regenerate = 0
 
-       def get_virts_p(self, myroot):
+       def get_virts_p(self, myroot=None):
                if self.virts_p:
                        return self.virts_p
-               virts = self.getvirtuals(myroot)
+               virts = self.getvirtuals()
                if virts:
                        for x in virts:
                                vkeysplit = x.split("/")
@@ -4291,12 +4291,34 @@ def prepare_build_dirs(myroot, mysettings, cleanup):
                                        if droppriv:
                                                st = os.stat(mydir)
                                                if st.st_gid != portage_gid or \
-                                                       not stat.S_IMODE(st.st_mode) & dirmode:
+                                                       not dirmode == (stat.S_IMODE(st.st_mode) & dirmode):
                                                        droppriv_fix = True
+                                               if not droppriv_fix:
+                                                       # Check permissions of files in the directory.
+                                                       for filename in os.listdir(mydir):
+                                                               try:
+                                                                       subdir_st = os.lstat(
+                                                                               os.path.join(mydir, filename))
+                                                               except OSError:
+                                                                       continue
+                                                               if subdir_st.st_gid != portage_gid or \
+                                                                       ((stat.S_ISDIR(subdir_st.st_mode) and \
+                                                                       not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode)) or \
+                                                                       (not stat.S_ISDIR(subdir_st.st_mode) and \
+                                                                       not filemode == (stat.S_IMODE(subdir_st.st_mode) & filemode))):
+                                                                       droppriv_fix = True
+                                                                       break
+                                       if droppriv_fix:
+                                               writemsg(colorize("WARN", " * ") + \
+                                                        "Adjusting permissions " + \
+                                                        "for FEATURES=userpriv: '%s'\n" % mydir,
+                                                       noiselevel=-1)
+                                       elif modified:
+                                               writemsg(colorize("WARN", " * ") + \
+                                                        "Adjusting permissions " + \
+                                                        "for FEATURES=%s: '%s'\n" % (myfeature, mydir),
+                                                       noiselevel=-1)
                                        if modified or kwargs["always_recurse"] or droppriv_fix:
-                                               if modified:
-                                                       writemsg("Adjusting permissions recursively: '%s'\n" % mydir,
-                                                               noiselevel=-1)
                                                def onerror(e):
                                                        raise   # The feature is disabled if a single error
                                                                        # occurs during permissions adjustment.
index b2421c98df810389654c8c9cd2004f6fd7efe4a5..541de39ccf1fd9818d0ae1c506d5de9c080be167 100644 (file)
@@ -181,6 +181,10 @@ class binarytree(object):
                        self._pkgindex_default_header_data = {
                                "repository":""
                        }
+                       self._pkgindex_translated_keys = (
+                               ("DESCRIPTION"   ,   "DESC"),
+                               ("repository"    ,   "REPO"),
+                       )
 
        def move_ent(self, mylist):
                if not self.populated:
@@ -821,7 +825,8 @@ class binarytree(object):
                return portage.getbinpkg.PackageIndex(
                        default_header_data=self._pkgindex_default_header_data,
                        default_pkg_data=self._pkgindex_default_pkg_data,
-                       inherited_keys=self._pkgindex_inherited_keys)
+                       inherited_keys=self._pkgindex_inherited_keys,
+                       translated_keys=self._pkgindex_translated_keys)
 
        def _update_pkgindex_header(self, header):
                portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"]))
@@ -856,8 +861,6 @@ class binarytree(object):
                return False
 
        def _eval_use_flags(self, cpv, metadata):
-               metadata["DESC"] = metadata["DESCRIPTION"]
-               del metadata["DESCRIPTION"]
                use = metadata["USE"].split()
                raw_use = use
                iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split())
index 63f9a69c53a53902244991a83aa414fff4304eab..3bf1fb1bbdb58827a388b82f2110758202f6ae9a 100644 (file)
@@ -25,16 +25,24 @@ def collect_ebuild_messages(path):
        mylogfiles.reverse()
        logentries = {}
        for msgfunction in mylogfiles:
+               filename = os.path.join(path, msgfunction)
                if msgfunction not in EBUILD_PHASES:
-                       writemsg("!!! can't process invalid log file: %s\n" % f,
+                       writemsg("!!! can't process invalid log file: %s\n" % filename,
                                noiselevel=-1)
                        continue
                if not msgfunction in logentries:
                        logentries[msgfunction] = []
                lastmsgtype = None
                msgcontent = []
-               for l in open(os.path.join(path, msgfunction), "r").readlines():
-                       msgtype, msg = l.split(" ", 1)
+               for l in open(filename, "r").read().split("\0"):
+                       if not l:
+                               continue
+                       try:
+                               msgtype, msg = l.split(" ", 1)
+                       except ValueError:
+                               writemsg("!!! malformed entry in " + \
+                                       "log file: '%s'\n" % filename, noiselevel=-1)
+                               continue
                        if lastmsgtype is None:
                                lastmsgtype = msgtype
                        if msgtype == lastmsgtype:
@@ -44,6 +52,8 @@ def collect_ebuild_messages(path):
                                        logentries[msgfunction].append((lastmsgtype, msgcontent))
                                msgcontent = [msg]
                        lastmsgtype = msgtype
+               if msgcontent:
+                       logentries[msgfunction].append((lastmsgtype, msgcontent))
 
        # clean logfiles to avoid repetitions
        for f in mylogfiles:
index 64cd74699bb1c311fd8d760ef776f052cb99e8ac..635fcdfa8c7d523aed4a0e0b8ae83f505dac6062 100644 (file)
@@ -669,10 +669,15 @@ def _cmp_cpv(d1, d2):
 class PackageIndex(object):
 
        def __init__(self, default_header_data=None, default_pkg_data=None,
-               inherited_keys=None):
+               inherited_keys=None, translated_keys=None):
                self._default_header_data = default_header_data
                self._default_pkg_data = default_pkg_data
                self._inherited_keys = inherited_keys
+               self._write_translation_map = {}
+               self._read_translation_map = {}
+               if translated_keys:
+                       self._write_translation_map.update(translated_keys)
+                       self._read_translation_map.update(((y, x) for (x, y) in translated_keys))
                self.header = {}
                if self._default_header_data:
                        self.header.update(self._default_header_data)
@@ -691,12 +696,13 @@ class PackageIndex(object):
                        k, v = line
                        if v:
                                v = v[1:]
-                       d[k] = v
+                       d[self._read_translation_map.get(k, k)] = v
                return d
-       
+
        def _writepkgindex(self, pkgfile, items):
                for k, v in items:
-                       pkgfile.write("%s: %s\n" % (k, v))
+                       pkgfile.write("%s: %s\n" % \
+                               (self._write_translation_map.get(k, k), v))
                pkgfile.write("\n")
 
        def read(self, pkgfile):