From: Fabian Groffen Date: Mon, 14 Apr 2008 17:49:59 +0000 (-0000) Subject: Merged from trunk 9869:9891 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=bfc1d537438301bde889fd9801a1dbcb9545c37f;p=portage.git Merged from trunk 9869:9891 | 9870 | Make _dep_check_composite_db match() return multiple slots | | zmedico | so that it behaves more like other dbapi instances would, | | | and also make it inherit from dbapi. | | 9871 | Fix _dep_check_composite_db.match() so that it doesn't | | zmedico | exempt the highest slot from visibility checks. | | 9872 | Fix the _dep_check_composite_db visibility filtering some | | zmedico | more. | | 9873 | Improve lookahead for new-style virtuals: * Substitue the | | zmedico | graph tree for the vartree in dep_check() since we want atom | | | selections to be consistent with package selections have | | | already been made. * Add the selected package to the graph | | | as soon as possible so that later dep_check() calls can use | | | it as feedback for making more consistent atom selections. * | | | Expand all slots of new-style virtuals inside | | | _dep_check_composite_db so that lookahead examines a matched | | | slots. This solves some cases of bug #1343, and extends the | | | fix for bug #141118 to work in cases when a virtual is not | | | yet installed but it has been pulled into the graph. | | 9874 | To optimize consistency of atom selections, add all of the | | zmedico | root packages to the graph before processing any | | | dependencies. | | 9879 | * For optimal atom selection, use PackageVirtualDbapi | | zmedico | instances in place of the vardb for all dep_check() calls. * | | | Replace dep_check_trees in depgraph.validate_blockers() with | | | the equivalent self._graph_trees reference. | | 9881 | Fix buggy logic in WorldSet.load() that causes it to discard | | zmedico | all of it's atoms in some cases (leading to loss of the | | | whole world file). | | 9882 | Fix some more broken logic inside WorldSet.load(). | | zmedico | | | 9883 | s/timetasmp/timestamp/ | | zmedico | | | 9884 | Split out a depgraph._add_dep() method. | | zmedico | | | 9885 | Call depgraph._add_dep() to add package dependencies to the | | zmedico | graph as soon as possible rather than pushing them onto the | | | stack. | | 9886 | Fix some breakage in the allow_unsatisfied handling. | | zmedico | | | 9890 | When regenerating USE_EXPAND in config.setcpv(), improve | | zmedico | performance by using regular expression instead of | | | str.startswith(). | | 9891 | Revert back to using startwith because I screwed up my | | zmedico | benchmark and it turns out starswith is faster than using | | | regular expressions. | svn path=/main/branches/prefix/; revision=9893 --- diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index 6b08f8b40..d53ee4536 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -260,7 +260,7 @@ do_snapshot() { # if [ ${have_files} -eq 1 ]; then - vecho "Getting snapshot timetasmp ..." + vecho "Getting snapshot timestamp ..." local snapshot_timestamp=$(get_snapshot_timestamp "${file}") if [ ${ignore_timestamp} == 0 ]; then diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index b228ce28d..522115ddf 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1620,11 +1620,13 @@ class depgraph(object): pass graph_tree.dbapi = fakedb self._graph_trees[myroot] = {} - self._graph_trees[myroot]["porttree"] = graph_tree - self._graph_trees[myroot]["vartree"] = self.trees[myroot]["vartree"] - del vardb, fakedb self._filtered_trees[myroot] = {} - self._filtered_trees[myroot]["vartree"] = self.trees[myroot]["vartree"] + # Substitute the graph tree for the vartree in dep_check() since we + # want atom selections to be consistent with package selections + # have already been made. + self._graph_trees[myroot]["porttree"] = graph_tree + self._graph_trees[myroot]["vartree"] = graph_tree + self._filtered_trees[myroot]["vartree"] = graph_tree def filtered_tree(): pass filtered_tree.dbapi = self._dep_check_composite_db(self, myroot) @@ -1795,69 +1797,75 @@ class depgraph(object): return None def _create_graph(self, allow_unsatisfied=False): - debug = "--debug" in self.myopts - buildpkgonly = "--buildpkgonly" in self.myopts - nodeps = "--nodeps" in self.myopts - empty = "empty" in self.myparams - deep = "deep" in self.myparams - consistent = "consistent" in self.myparams dep_stack = self._dep_stack while dep_stack: dep = dep_stack.pop() if isinstance(dep, Package): - if not self._add_pkg_deps(dep): + if not self._add_pkg_deps(dep, + allow_unsatisfied=allow_unsatisfied): return 0 continue - update = "--update" in self.myopts and dep.depth <= 1 - if dep.blocker: - if not buildpkgonly and \ - not nodeps and \ - dep.parent not in self._slot_collision_nodes: - if dep.parent.onlydeps: - # It's safe to ignore blockers if the - # parent is an --onlydeps node. - continue - # The blocker applies to the root where - # the parent is or will be installed. - self.blocker_parents.setdefault( - ("blocks", dep.parent.root, dep.atom), set()).add( - dep.parent) - continue - dep_pkg, existing_node = self._select_package(dep.root, dep.atom, - onlydeps=dep.onlydeps) - if not dep_pkg: - if allow_unsatisfied: - self._unsatisfied_deps.append(dep) - continue - self._unsatisfied_deps_for_display.append( - ((dep.root, dep.atom), {"myparent":dep.parent})) + if not self._add_dep(dep, allow_unsatisfied=allow_unsatisfied): return 0 - # In some cases, dep_check will return deps that shouldn't - # be proccessed any further, so they are identified and - # discarded here. Try to discard as few as possible since - # discarded dependencies reduce the amount of information - # available for optimization of merge order. - if dep.priority.satisfied and \ - not (existing_node or empty or deep or update): - myarg = None - if dep.root == self.target_root: - try: - myarg = self._iter_atoms_for_pkg(dep_pkg).next() - except StopIteration: - pass - except portage.exception.InvalidDependString: - if not dep_pkg.installed: - # This shouldn't happen since the package - # should have been masked. - raise - if not myarg: - if consistent: - self._ignored_deps.append(dep) - continue + return 1 - if not self._add_pkg(dep_pkg, dep.parent, - priority=dep.priority, depth=dep.depth): - return 0 + def _add_dep(self, dep, allow_unsatisfied=False): + debug = "--debug" in self.myopts + buildpkgonly = "--buildpkgonly" in self.myopts + nodeps = "--nodeps" in self.myopts + empty = "empty" in self.myparams + deep = "deep" in self.myparams + consistent = "consistent" in self.myparams + update = "--update" in self.myopts and dep.depth <= 1 + if dep.blocker: + if not buildpkgonly and \ + not nodeps and \ + dep.parent not in self._slot_collision_nodes: + if dep.parent.onlydeps: + # It's safe to ignore blockers if the + # parent is an --onlydeps node. + return 1 + # The blocker applies to the root where + # the parent is or will be installed. + self.blocker_parents.setdefault( + ("blocks", dep.parent.root, dep.atom), set()).add( + dep.parent) + return 1 + dep_pkg, existing_node = self._select_package(dep.root, dep.atom, + onlydeps=dep.onlydeps) + if not dep_pkg: + if allow_unsatisfied: + self._unsatisfied_deps.append(dep) + return 1 + self._unsatisfied_deps_for_display.append( + ((dep.root, dep.atom), {"myparent":dep.parent})) + return 0 + # In some cases, dep_check will return deps that shouldn't + # be proccessed any further, so they are identified and + # discarded here. Try to discard as few as possible since + # discarded dependencies reduce the amount of information + # available for optimization of merge order. + if dep.priority.satisfied and \ + not (existing_node or empty or deep or update): + myarg = None + if dep.root == self.target_root: + try: + myarg = self._iter_atoms_for_pkg(dep_pkg).next() + except StopIteration: + pass + except portage.exception.InvalidDependString: + if not dep_pkg.installed: + # This shouldn't happen since the package + # should have been masked. + raise + if not myarg: + if consistent: + self._ignored_deps.append(dep) + return 1 + + if not self._add_pkg(dep_pkg, dep.parent, + priority=dep.priority, depth=dep.depth): + return 0 return 1 def _add_pkg(self, pkg, myparent, priority=None, depth=0): @@ -2011,7 +2019,7 @@ class depgraph(object): dep_stack.append(pkg) return 1 - def _add_pkg_deps(self, pkg): + def _add_pkg_deps(self, pkg, allow_unsatisfied=False): mytype = pkg.type_name myroot = pkg.root @@ -2085,10 +2093,11 @@ class depgraph(object): mypriority = dep_priority.copy() if not blocker and vardb.match(atom): mypriority.satisfied = True - self._dep_stack.append( - Dependency(atom=atom, - blocker=blocker, depth=depth, parent=pkg, - priority=mypriority, root=dep_root)) + if not self._add_dep(Dependency(atom=atom, + blocker=blocker, depth=depth, parent=pkg, + priority=mypriority, root=dep_root), + allow_unsatisfied=allow_unsatisfied): + return 0 if debug: print "Exiting...", jbigkey except ValueError, e: @@ -2405,6 +2414,7 @@ class depgraph(object): while args: arg = args.pop() for atom in arg.set: + self.spinner.update() atom_cp = portage.dep_getkey(atom) try: pprovided = pprovideddict.get(portage.dep_getkey(atom)) @@ -2453,9 +2463,14 @@ class depgraph(object): arg.name in ("system", "world")): return 0, myfavorites - self._dep_stack.append( - Dependency(atom=atom, onlydeps=onlydeps, root=myroot, parent=arg)) - if not self._create_graph(): + dep = Dependency(atom=atom, onlydeps=onlydeps, + root=myroot, parent=arg) + + # Add the selected package to the graph as soon as possible + # so that later dep_check() calls can use it as feedback + # for making more consistent atom selections. + if not self._add_pkg(pkg, dep.parent, + priority=dep.priority, depth=dep.depth): if isinstance(arg, SetArg): sys.stderr.write(("\n\n!!! Problem resolving " + \ "dependencies for %s from %s\n") % \ @@ -2464,6 +2479,7 @@ class depgraph(object): sys.stderr.write(("\n\n!!! Problem resolving " + \ "dependencies for %s\n") % atom) return 0, myfavorites + except portage.exception.MissingSignature, e: portage.writemsg("\n\n!!! A missing gpg signature is preventing portage from calculating the\n") portage.writemsg("!!! required dependencies. This is a security feature enabled by the admin\n") @@ -2485,6 +2501,11 @@ class depgraph(object): print >> sys.stderr, "!!!", str(e), getattr(e, "__module__", None) raise + # Now that the root packages have been added to the graph, + # process the dependencies. + if not self._create_graph(): + return 0, myfavorites + missing=0 if "--usepkgonly" in self.myopts: for xs in self.digraph.all_nodes(): @@ -2978,18 +2999,6 @@ class depgraph(object): # due to the performance penalty that is incurred by all the # additional dep_check calls that are required. - # Optimization hack for dep_check calls that minimizes the - # available matches by replacing the portdb with a fakedbapi - # instance. - class FakePortageTree(object): - def __init__(self, mydb): - self.dbapi = mydb - dep_check_trees = {} - for myroot in self.trees: - dep_check_trees[myroot] = self.trees[myroot].copy() - dep_check_trees[myroot]["porttree"] = \ - FakePortageTree(self.mydbapi[myroot]) - dep_keys = ["DEPEND","RDEPEND","PDEPEND"] for myroot in self.trees: vardb = self.trees[myroot]["vartree"].dbapi @@ -3025,7 +3034,7 @@ class depgraph(object): try: success, atoms = portage.dep_check(depstr, final_db, pkgsettings, myuse=myuse, - trees=dep_check_trees, myroot=myroot) + trees=self._graph_trees, myroot=myroot) except Exception, e: if isinstance(e, SystemExit): raise @@ -4358,7 +4367,7 @@ class depgraph(object): fakedb[myroot].cpv_inject(pkg) self.spinner.update() - class _dep_check_composite_db(object): + class _dep_check_composite_db(portage.dbapi): """ A dbapi-like interface that is optimized for use in dep_check() calls. This is built on top of the existing depgraph package selection logic. @@ -4367,6 +4376,7 @@ class depgraph(object): via dep_check(). """ def __init__(self, depgraph, root): + portage.dbapi.__init__(self) self._depgraph = depgraph self._root = root self._match_cache = {} @@ -4383,24 +4393,58 @@ class depgraph(object): if not pkg: ret = [] else: - if pkg.installed and "selective" not in self._depgraph.myparams: - try: - arg = self._depgraph._iter_atoms_for_pkg(pkg).next() - except (StopIteration, portage.exception.InvalidDependString): - arg = None - if arg: - ret = [] - if ret is None and pkg.installed and \ - not visible(self._depgraph.pkgsettings[pkg.root], pkg): - # For disjunctive || deps, this will cause alternative - # atoms or packages to be selected if available. - ret = [] - if ret is None: + # Return the highest available from select_package() as well as + # any matching slots in the graph db. + slots = set() + slots.add(pkg.metadata["SLOT"]) + atom_cp = portage.dep_getkey(atom) + if pkg.cp.startswith("virtual/"): + # For new-style virtual lookahead that occurs inside + # dep_check(), examine all slots. This is needed + # so that newer slots will not unnecessarily be pulled in + # when a satisfying lower slot is already installed. For + # example, if virtual/jdk-1.4 is satisfied via kaffe then + # there's no need to pull in a newer slot to satisfy a + # virtual/jdk dependency. + for db, pkg_type, built, installed, db_keys in \ + self._depgraph._filtered_trees[self._root]["dbs"]: + for cpv in db.match(atom): + if portage.cpv_getkey(cpv) != pkg.cp: + continue + slots.add(db.aux_get(cpv, ["SLOT"])[0]) + ret = [] + if self._visible(pkg): + self._cpv_pkg_map[pkg.cpv] = pkg + ret.append(pkg.cpv) + slots.remove(pkg.metadata["SLOT"]) + while slots: + slot_atom = "%s:%s" % (atom_cp, slots.pop()) + pkg, existing = self._depgraph._select_package( + self._root, slot_atom) + if not pkg: + continue + if not self._visible(pkg): + continue self._cpv_pkg_map[pkg.cpv] = pkg - ret = [pkg.cpv] + ret.append(pkg.cpv) + if ret: + self._cpv_sort_ascending(ret) self._match_cache[orig_atom] = ret return ret[:] + def _visible(self, pkg): + if pkg.installed and "selective" not in self._depgraph.myparams: + try: + arg = self._depgraph._iter_atoms_for_pkg(pkg).next() + except (StopIteration, portage.exception.InvalidDependString): + arg = None + if arg: + return False + if pkg.installed and \ + not visible(self._depgraph.pkgsettings[pkg.root], pkg): + return False + return True + def _dep_expand(self, atom): """ This is only needed for old installed packages that may diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 8458faa58..cfc594ffb 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -184,6 +184,7 @@ class WorldSet(EditablePackageSet): def load(self): atoms = [] nonatoms = [] + atoms_changed = False # load atoms and non-atoms from different files so the worldfile is # backwards-compatible with older versions and other PMs, even though # it's supposed to be private state data :/ @@ -204,6 +205,9 @@ class WorldSet(EditablePackageSet): data = {} atoms = data.keys() self._mtime = mtime + atoms_changed = True + else: + atoms.extend(self._atoms) try: mtime = os.stat(self._filename2).st_mtime except (OSError, IOError): @@ -221,7 +225,10 @@ class WorldSet(EditablePackageSet): data = {} nonatoms = data.keys() self._mtime2 = mtime - if self._atoms != atoms or self._nonatoms != nonatoms: + atoms_changed = True + else: + nonatoms.extend(self._nonatoms) + if atoms_changed: self._setAtoms(atoms+nonatoms) def _ensure_dirs(self):