From: Fabian Groffen Date: Thu, 10 Apr 2008 16:52:27 +0000 (-0000) Subject: Merged from trunk 9741:9774 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=7f388b545d728b1828376df54a940919d357b8a0;p=portage.git Merged from trunk 9741:9774 | 9742 | When reading and writing the Packages index file, translate | | zmedico | metadata keys on the fly. Current translations: DESC | | | DESCRIPTION REPO repository | | 9743 | For repoman, only cache the metadata necessary for | | zmedico | visibility checks in order to reduce memory consumption. | | 9744 | Remove unused iter_atoms() function. | | zmedico | | | 9745 | Remove the depgraph._get_arg_for_pkg() method since | | zmedico | _iter_atoms_for_pkg() does the job. | | 9747 | Fix a little issue in the code from bug #197905 that causes | | zmedico | some elog messages to get lost. | | 9755 | Fix --update greedy atoms logic so that it only pulls in | | zmedico | packages that actually match the given atom. | | 9756 | Bug #197810 - Don't automatically downgrade packages for | | zmedico | which available matching versions are masked by corruption, | | | but do automatically downgrade installed packages that | | | themselves have a valid mask such as from package.mask. | | 9759 | Bug #162542 - When checking permissions inside CCACHE_DIR | | zmedico | for compatibility with FEATURES=userpriv, check the | | | permissions on files inside the directory to decide whether | | | to fix the permissions. | | 9761 | Split out a depgraph method to check if a specific new-style | | zmedico | virtual package exists. | | 9762 | Make depgraph.select_files() detect when a package name | | zmedico | given as an argument is ambiguous due to the existence of a | | | new-style virtual with the same name (cpv_expand() will not | | | always raise a ValueError in cases like this). | | 9763 | Handle issues with newlines in elog messages that can | | zmedico | trigger an unhandled ValueError to be raised from a split() | | | call inside collect_ebuild_messages(): * Use \0 to delimit | | | messages, so that that elog messages containing newlines are | | | handled correctly. * Handle a potential ValueError when | | | splitting the message type. | | 9765 | Create a new depgraph._dep_expand() method for | | zmedico | select_files() to expand atoms that have no category. | | 9769 | When depgraph.select_files() expands an atom category, in | | zmedico | case of an old-style virtual, defer category expansion so | | | that the depgraph has more choices. | | 9771 | Make DepcheckCompositeDB cache Package instances for | | zmedico | aux_get() calls. | | 9774 | When pruning the list of parents for the slot collision | | zmedico | display, always show parents that themselves have been | | | pulled into collision slots, since those are more relevant | | | that others. (branches/2.1.2 r9773) | svn path=/main/branches/prefix/; revision=9806 --- diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 788f9118d..c107bbce3 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -32,13 +32,6 @@ portage-2.2 * "world" does no longer include "system" unconditionally, but you can add "@system" to the worldfile to restore the old state. -portage-2.1.5 -================================== -* In the event that a previously installed package has since been masked, - emerge will no longer perform an automatic downgrade as part of a "world" - update. You should either unmask such packages or else explicitly re-merge - them in order to have them dowgraded to an unmasked version. - portage-2.1.4.1 ================================== diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index aa5d73d72..343d62657 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -177,7 +177,7 @@ elog_base() { return 1 ;; esac - echo -e "${messagetype} $*" >> "${T}/logging/${EBUILD_PHASE:-other}" + echo -ne "${messagetype} $*\n\0" >> "${T}/logging/${EBUILD_PHASE:-other}" return 0 } diff --git a/bin/repoman b/bin/repoman index 74adcca75..2954a15da 100755 --- a/bin/repoman +++ b/bin/repoman @@ -472,6 +472,11 @@ trees = portage.create_trees() trees["/"]["porttree"].settings = repoman_settings portdb = trees["/"]["porttree"].dbapi portdb.mysettings = repoman_settings +# We really only need to cache the metadata that's necessary for visibility +# filtering. Anything else can be discarded to reduce memory consumption. +for k in ("DEPEND", "IUSE", "LICENCE", "PDEPEND", + "PROVIDE", "RDEPEND", "RESTRICT", "repository"): + portdb._aux_cache_keys.discard(k) # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman. del trees["/"]["vartree"] diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 7a8a802c8..954547997 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1193,21 +1193,6 @@ def show_masked_packages(masked_packages): shown_licenses.add(l) return have_eapi_mask -def iter_atoms(deps): - """Take a dependency structure as returned by paren_reduce or use_reduce - and iterate over all the atoms.""" - i = iter(deps) - for x in i: - if isinstance(x, basestring): - if x == '||' or x.endswith('?'): - for x in iter_atoms(i.next()): - yield x - else: - yield x - else: - for x in iter_atoms(x): - yield x - class Package(object): __slots__ = ("__weakref__", "built", "cpv", "depth", "installed", "metadata", "root", "onlydeps", "type_name", @@ -1452,7 +1437,7 @@ class DepcheckCompositeDB(object): self._depgraph = depgraph self._root = root self._match_cache = {} - self._cpv_tree_map = {} + self._cpv_pkg_map = {} def match(self, atom): ret = self._match_cache.get(atom) @@ -1469,16 +1454,20 @@ class DepcheckCompositeDB(object): arg = None if arg: ret = [] + if ret is None and pkg.installed and \ + not visible(self._depgraph.pkgsettings[pkg.root], pkg): + # For disjunctive || deps, this will cause alternative + # atoms or packages to be selected if available. + ret = [] if ret is None: - self._cpv_tree_map[pkg.cpv] = \ - self._depgraph.pkg_tree_map[pkg.type_name] + self._cpv_pkg_map[pkg.cpv] = pkg ret = [pkg.cpv] self._match_cache[atom] = ret return ret[:] def aux_get(self, cpv, wants): - return self._depgraph.trees[self._root][ - self._cpv_tree_map[cpv]].dbapi.aux_get(cpv, wants) + metadata = self._cpv_pkg_map[cpv].metadata + return [metadata.get(x, "") for x in wants] class depgraph(object): @@ -1653,16 +1642,25 @@ class depgraph(object): for parent in parents: if isinstance(parent, DependencyArg): pruned_list.append(parent) - if len(pruned_list) == max_parents: - break + # Prefer Packages instances that themselves have been + # pulled into collision slots. + for parent in parents: + if isinstance(parent, Package) and \ + (parent.slot_atom, parent.root) \ + in self._slot_collision_info: + pruned_list.append(parent) for parent in parents: if not isinstance(parent, DependencyArg): pruned_list.append(parent) - if len(pruned_list) == max_parents: + if len(pruned_list) >= max_parents: break parents = pruned_list msg.append(" pulled in by\n") + shown_parents = set() for parent in parents: + if parent in shown_parents: + continue + shown_parents.add(parent) msg.append(2*indent) msg.append(str(parent)) msg.append("\n") @@ -1770,7 +1768,9 @@ class depgraph(object): myarg = None if dep.root == self.target_root: try: - myarg = self._get_arg_for_pkg(dep_pkg) + myarg = self._iter_atoms_for_pkg(dep_pkg).next() + except StopIteration: + pass except portage.exception.InvalidDependString: if not dep_pkg.installed: # This shouldn't happen since the package @@ -2040,6 +2040,40 @@ class depgraph(object): return 0 return 1 + def _dep_expand(self, root_config, atom_without_category): + """ + @param root_config: a root config instance + @type root_config: RootConfig + @returns: a list of cat/pn for any matching packages + """ + null_cp = portage.dep_getkey(insert_category_into_atom( + atom_without_category, "null")) + cat, atom_pn = portage.catsplit(null_cp) + + cp_set = set() + for db, pkg_type, built, installed, db_keys in \ + self._filtered_trees[root_config.root]["dbs"]: + cp_set.update(db.cp_all()) + for cp in list(cp_set): + cat, pn = portage.catsplit(cp) + if pn != atom_pn: + cp_set.discard(cp) + deps = [] + for cp in cp_set: + cat, pn = portage.catsplit(cp) + deps.append(insert_category_into_atom( + atom_without_category, cat)) + return deps + + def _have_new_virt(self, root, atom_cp): + ret = False + for db, pkg_type, built, installed, db_keys in \ + self._filtered_trees[root]["dbs"]: + if db.cp_list(atom_cp): + ret = True + break + return ret + def _iter_atoms_for_pkg(self, pkg): # TODO: add multiple $ROOT support if pkg.root != self.target_root: @@ -2047,36 +2081,15 @@ class depgraph(object): atom_arg_map = self._atom_arg_map for atom in self._set_atoms.iterAtomsForPackage(pkg): atom_cp = portage.dep_getkey(atom) - if atom_cp != pkg.cp: - have_new_virt = False - for db, pkg_type, built, installed, db_keys in \ - self._filtered_trees[pkg.root]["dbs"]: - if db.cp_list(atom_cp): - have_new_virt = True - break - if have_new_virt: - continue + if atom_cp != pkg.cp and \ + self._have_new_virt(pkg.root, atom_cp): + continue for arg in atom_arg_map[(atom, pkg.root)]: if isinstance(arg, PackageArg) and \ arg.package != pkg: continue yield arg, atom - def _get_arg_for_pkg(self, pkg): - """ - Return a matching DependencyArg instance for the given Package if - any exist, otherwise None. An attempt will be made to return the most - specific match (PackageArg type is the most specific). - - This will raise an InvalidDependString exception if PROVIDE is invalid. - """ - any_arg = None - for arg, atom in self._iter_atoms_for_pkg(pkg): - if isinstance(arg, PackageArg): - return arg - any_arg = arg - return any_arg - def select_files(self, myfiles): """Given a list of .tbz2s, .ebuilds sets, and deps, create the appropriate depgraph and return a favorite list.""" @@ -2226,37 +2239,40 @@ class depgraph(object): args.append(AtomArg(arg=x, atom=x, root_config=root_config)) continue - try: - try: - for db, pkg_type, built, installed, db_keys in dbs: - mykey = portage.dep_expand(x, - mydb=db, settings=pkgsettings) - if portage.dep_getkey(mykey).startswith("null/"): - continue - break - except ValueError, e: - if not e.args or not isinstance(e.args[0], list) or \ - len(e.args[0]) < 2: - raise - mykey = portage.dep_expand(x, - mydb=vardb, settings=pkgsettings) - cp = portage.dep_getkey(mykey) - if cp.startswith("null/") or \ - cp not in e[0]: - raise - del e - args.append(AtomArg(arg=x, atom=mykey, - root_config=root_config)) - except ValueError, e: - if not e.args or not isinstance(e.args[0], list) or \ - len(e.args[0]) < 2: - raise + expanded_atoms = self._dep_expand(root_config, x) + installed_cp_set = set() + for atom in expanded_atoms: + if vardb.match(atom): + installed_cp_set.add(portage.dep_getkey(atom)) + if len(expanded_atoms) > 1 and len(installed_cp_set) == 1: + installed_cp = iter(installed_cp_set).next() + expanded_atoms = [atom for atom in expanded_atoms \ + if portage.dep_getkey(atom) == installed_cp] + + if len(expanded_atoms) > 1: print "\n\n!!! The short ebuild name \"" + x + "\" is ambiguous. Please specify" print "!!! one of the following fully-qualified ebuild names instead:\n" - for i in e.args[0]: + expanded_atoms = set(portage.dep_getkey(atom) \ + for atom in expanded_atoms) + for i in sorted(expanded_atoms): print " " + green(i) print return False, myfavorites + if expanded_atoms: + atom = expanded_atoms[0] + else: + null_atom = insert_category_into_atom(x, "null") + null_cp = portage.dep_getkey(null_atom) + cat, atom_pn = portage.catsplit(null_cp) + virts_p = root_config.settings.get_virts_p().get(atom_pn) + if virts_p: + # Allow the depgraph to choose which virtual. + atom = insert_category_into_atom(x, "virtual") + else: + atom = insert_category_into_atom(x, "null") + + args.append(AtomArg(arg=x, atom=atom, + root_config=root_config)) if "--update" in self.myopts: # Enable greedy SLOT atoms for atoms given as arguments. @@ -2271,7 +2287,7 @@ class depgraph(object): continue atom_cp = portage.dep_getkey(arg.atom) slots = set() - for cpv in vardb.match(atom_cp): + for cpv in vardb.match(arg.atom): slots.add(vardb.aux_get(cpv, ["SLOT"])[0]) for slot in slots: greedy_atoms.append( @@ -2603,9 +2619,15 @@ class depgraph(object): if not installed: # masked by corruption continue - if not installed: - if myarg: - found_available_arg = True + if not installed and myarg: + found_available_arg = True + if not installed or (installed and matched_packages): + # Only enforce visibility on installed packages + # if there is at least one other visible package + # available. By filtering installed masked packages + # here, packages that have been masked since they + # were installed can be automatically downgraded + # to an unmasked version. if not visible(pkgsettings, pkg): continue if not built and not calculated_use: @@ -5542,6 +5564,26 @@ def checkUpdatedNewsItems(portdb, vardb, NEWS_PATH, UNREAD_PATH, repo_id): manager = NewsManager(portdb, vardb, NEWS_PATH, UNREAD_PATH) return manager.getUnreadItems( repo_id, update=True ) +def expand_virtual_atom(x): + """ + Take an atom without a category and insert virtual/ for the + category. This works correctly with atoms that have operators. + + @param x: an atom without a category + @type x: String + @returns: the atom with virtual/ inserted for the category, or None + """ + return insert_category_into_atom(atom, "virtual") + +def insert_category_into_atom(atom, category): + alphanum = re.search(r'\w', atom) + if alphanum: + ret = atom[:alphanum.start()] + "%s/" % category + \ + atom[alphanum.start():] + else: + ret = None + return ret + def is_valid_package_atom(x): if "/" not in x: alphanum = re.search(r'\w', x) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 94729054f..2e81a3b61 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2538,10 +2538,10 @@ class config(object): self.already_in_regenerate = 0 - def get_virts_p(self, myroot): + def get_virts_p(self, myroot=None): if self.virts_p: return self.virts_p - virts = self.getvirtuals(myroot) + virts = self.getvirtuals() if virts: for x in virts: vkeysplit = x.split("/") @@ -4291,12 +4291,34 @@ def prepare_build_dirs(myroot, mysettings, cleanup): if droppriv: st = os.stat(mydir) if st.st_gid != portage_gid or \ - not stat.S_IMODE(st.st_mode) & dirmode: + not dirmode == (stat.S_IMODE(st.st_mode) & dirmode): droppriv_fix = True + if not droppriv_fix: + # Check permissions of files in the directory. + for filename in os.listdir(mydir): + try: + subdir_st = os.lstat( + os.path.join(mydir, filename)) + except OSError: + continue + if subdir_st.st_gid != portage_gid or \ + ((stat.S_ISDIR(subdir_st.st_mode) and \ + not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode)) or \ + (not stat.S_ISDIR(subdir_st.st_mode) and \ + not filemode == (stat.S_IMODE(subdir_st.st_mode) & filemode))): + droppriv_fix = True + break + if droppriv_fix: + writemsg(colorize("WARN", " * ") + \ + "Adjusting permissions " + \ + "for FEATURES=userpriv: '%s'\n" % mydir, + noiselevel=-1) + elif modified: + writemsg(colorize("WARN", " * ") + \ + "Adjusting permissions " + \ + "for FEATURES=%s: '%s'\n" % (myfeature, mydir), + noiselevel=-1) if modified or kwargs["always_recurse"] or droppriv_fix: - if modified: - writemsg("Adjusting permissions recursively: '%s'\n" % mydir, - noiselevel=-1) def onerror(e): raise # The feature is disabled if a single error # occurs during permissions adjustment. diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index b2421c98d..541de39cc 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -181,6 +181,10 @@ class binarytree(object): self._pkgindex_default_header_data = { "repository":"" } + self._pkgindex_translated_keys = ( + ("DESCRIPTION" , "DESC"), + ("repository" , "REPO"), + ) def move_ent(self, mylist): if not self.populated: @@ -821,7 +825,8 @@ class binarytree(object): return portage.getbinpkg.PackageIndex( default_header_data=self._pkgindex_default_header_data, default_pkg_data=self._pkgindex_default_pkg_data, - inherited_keys=self._pkgindex_inherited_keys) + inherited_keys=self._pkgindex_inherited_keys, + translated_keys=self._pkgindex_translated_keys) def _update_pkgindex_header(self, header): portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"])) @@ -856,8 +861,6 @@ class binarytree(object): return False def _eval_use_flags(self, cpv, metadata): - metadata["DESC"] = metadata["DESCRIPTION"] - del metadata["DESCRIPTION"] use = metadata["USE"].split() raw_use = use iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split()) diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index 63f9a69c5..3bf1fb1bb 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -25,16 +25,24 @@ def collect_ebuild_messages(path): mylogfiles.reverse() logentries = {} for msgfunction in mylogfiles: + filename = os.path.join(path, msgfunction) if msgfunction not in EBUILD_PHASES: - writemsg("!!! can't process invalid log file: %s\n" % f, + writemsg("!!! can't process invalid log file: %s\n" % filename, noiselevel=-1) continue if not msgfunction in logentries: logentries[msgfunction] = [] lastmsgtype = None msgcontent = [] - for l in open(os.path.join(path, msgfunction), "r").readlines(): - msgtype, msg = l.split(" ", 1) + for l in open(filename, "r").read().split("\0"): + if not l: + continue + try: + msgtype, msg = l.split(" ", 1) + except ValueError: + writemsg("!!! malformed entry in " + \ + "log file: '%s'\n" % filename, noiselevel=-1) + continue if lastmsgtype is None: lastmsgtype = msgtype if msgtype == lastmsgtype: @@ -44,6 +52,8 @@ def collect_ebuild_messages(path): logentries[msgfunction].append((lastmsgtype, msgcontent)) msgcontent = [msg] lastmsgtype = msgtype + if msgcontent: + logentries[msgfunction].append((lastmsgtype, msgcontent)) # clean logfiles to avoid repetitions for f in mylogfiles: diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 64cd74699..635fcdfa8 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -669,10 +669,15 @@ def _cmp_cpv(d1, d2): class PackageIndex(object): def __init__(self, default_header_data=None, default_pkg_data=None, - inherited_keys=None): + inherited_keys=None, translated_keys=None): self._default_header_data = default_header_data self._default_pkg_data = default_pkg_data self._inherited_keys = inherited_keys + self._write_translation_map = {} + self._read_translation_map = {} + if translated_keys: + self._write_translation_map.update(translated_keys) + self._read_translation_map.update(((y, x) for (x, y) in translated_keys)) self.header = {} if self._default_header_data: self.header.update(self._default_header_data) @@ -691,12 +696,13 @@ class PackageIndex(object): k, v = line if v: v = v[1:] - d[k] = v + d[self._read_translation_map.get(k, k)] = v return d - + def _writepkgindex(self, pkgfile, items): for k, v in items: - pkgfile.write("%s: %s\n" % (k, v)) + pkgfile.write("%s: %s\n" % \ + (self._write_translation_map.get(k, k), v)) pkgfile.write("\n") def read(self, pkgfile):