From dd3c45e02bdc488972c6bb7b799818cb32bba24b Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 6 Jul 2009 18:29:28 +0000 Subject: [PATCH] Merged from trunk -r13754:13764 | 13756 | Bug #275947 - Enable build.log when FEATURES=sesandbox is | | zmedico | enabled, since it works when a pty is available (but not | | | through a normal pipe, due to bug #162404). | | 13757 | Fix reversal in --nodeps merge order. Thanks to Michael | | zmedico | Haubenwallner for reporting. | | 13758 | Move RootConfig.visible_pkgs to | | zmedico | dynamic_config._visible_pkgs, so RootConfig can be | | | considered entirely "frozen". | | 13759 | Use depgraph._pkg() whenever possible. | | zmedico | | | 13760 | Add a frozen_config parameter to the depgraph constructor, | | zmedico | so that multiple depgraph instances created for backtracking | | | purposes can reuse the same frozen_config instance. | | 13761 | Use the new frozen_config depgraph constructor parameter to | | zmedico | optimize resume_depgraph(). | | 13762 | Fix broken reference to self.myopts in the dynamic_config | | zmedico | constructor. | | 13763 | Fix broken reference to _graph_trees. | | zmedico | | | 13764 | Skip PROVIDE conditional evaluation in | | zmedico | PackageSet.iterAtomsForPackage(), since Package.metadata | | | handles that automatically now. | svn path=/main/branches/prefix/; revision=13795 --- pym/_emerge/RootConfig.py | 2 - pym/_emerge/actions.py | 2 +- pym/_emerge/depgraph.py | 254 +++++++++++++++++++------------------- pym/portage/__init__.py | 13 +- pym/portage/sets/base.py | 6 +- 5 files changed, 134 insertions(+), 143 deletions(-) diff --git a/pym/_emerge/RootConfig.py b/pym/_emerge/RootConfig.py index 8d1a744c6..48af99120 100644 --- a/pym/_emerge/RootConfig.py +++ b/pym/_emerge/RootConfig.py @@ -28,5 +28,3 @@ class RootConfig(object): self.sets = {} else: self.sets = self.setconfig.getSets() - self.visible_pkgs = PackageVirtualDbapi(self.settings) - diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 60019e5a7..c70183ba5 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -1119,7 +1119,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, portage.dep._dep_check_strict = False success, atoms = portage.dep_check(depstr, None, settings, myuse=node_use, - trees=resolver._frozen_config._graph_trees, + trees=resolver._dynamic_config._graph_trees, myroot=myroot) finally: portage.dep._dep_check_strict = True diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 12011b5d1..0e4f8510b 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -56,7 +56,7 @@ from _emerge.visible import visible class _frozen_depgraph_config(object): - def __init__(self, settings, trees, myopts, spinner, dynamic_config, depgraph): + def __init__(self, settings, trees, myopts, spinner): self.settings = settings self.target_root = settings["ROOT"] self.myopts = myopts @@ -71,9 +71,8 @@ class _frozen_depgraph_config(object): self.trees = {} self._trees_orig = trees self.roots = {} - # Contains installed packages and new packages that have been added - # to the graph. - self._graph_trees = {} + # All Package instances + self._pkg_cache = {} for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -86,90 +85,15 @@ class _frozen_depgraph_config(object): self.trees[myroot][tree] = trees[myroot][tree] self.trees[myroot]["vartree"] = \ FakeVartree(trees[myroot]["root_config"], - pkg_cache=dynamic_config._pkg_cache) + pkg_cache=self._pkg_cache) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) - dynamic_config._slot_pkg_map[myroot] = {} - vardb = self.trees[myroot]["vartree"].dbapi - preload_installed_pkgs = "--nodeps" not in self.myopts and \ - "--buildpkgonly" not in self.myopts - # This fakedbapi instance will model the state that the vdb will - # have after new packages have been installed. - fakedb = PackageVirtualDbapi(vardb.settings) - if preload_installed_pkgs: - for pkg in vardb: - self.spinner.update() - # This triggers metadata updates via FakeVartree. - vardb.aux_get(pkg.cpv, []) - fakedb.cpv_inject(pkg) - - # Now that the vardb state is cached in our FakeVartree, - # we won't be needing the real vartree cache for awhile. - # To make some room on the heap, clear the vardbapi - # caches. - trees[myroot]["vartree"].dbapi._clear_cache() - gc.collect() - - dynamic_config.mydbapi[myroot] = fakedb - def graph_tree(): - pass - graph_tree.dbapi = fakedb - self._graph_trees[myroot] = {} - dynamic_config._filtered_trees[myroot] = {} - # Substitute the graph tree for the vartree in dep_check() since we - # want atom selections to be consistent with package selections - # have already been made. - self._graph_trees[myroot]["porttree"] = graph_tree - self._graph_trees[myroot]["vartree"] = graph_tree - def filtered_tree(): - pass - filtered_tree.dbapi = _dep_check_composite_db(depgraph, myroot) - dynamic_config._filtered_trees[myroot]["porttree"] = filtered_tree - - # Passing in graph_tree as the vartree here could lead to better - # atom selections in some cases by causing atoms for packages that - # have been added to the graph to be preferred over other choices. - # However, it can trigger atom selections that result in - # unresolvable direct circular dependencies. For example, this - # happens with gwydion-dylan which depends on either itself or - # gwydion-dylan-bin. In case gwydion-dylan is not yet installed, - # gwydion-dylan-bin needs to be selected in order to avoid a - # an unresolvable direct circular dependency. - # - # To solve the problem described above, pass in "graph_db" so that - # packages that have been added to the graph are distinguishable - # from other available packages and installed packages. Also, pass - # the parent package into self._select_atoms() calls so that - # unresolvable direct circular dependencies can be detected and - # avoided when possible. - dynamic_config._filtered_trees[myroot]["graph_db"] = graph_tree.dbapi - dynamic_config._filtered_trees[myroot]["vartree"] = self.trees[myroot]["vartree"] - - dbs = [] - portdb = self.trees[myroot]["porttree"].dbapi - bindb = self.trees[myroot]["bintree"].dbapi - vardb = self.trees[myroot]["vartree"].dbapi - # (db, pkg_type, built, installed, db_keys) - if "--usepkgonly" not in self.myopts: - db_keys = list(portdb._aux_cache_keys) - dbs.append((portdb, "ebuild", False, False, db_keys)) - if "--usepkg" in self.myopts: - db_keys = list(bindb._aux_cache_keys) - dbs.append((bindb, "binary", True, False, db_keys)) - db_keys = list(trees[myroot]["vartree"].dbapi._aux_cache_keys) - dbs.append((vardb, "installed", True, True, db_keys)) - dynamic_config._filtered_trees[myroot]["dbs"] = dbs - if "--usepkg" in self.myopts: - self.trees[myroot]["bintree"].populate( - "--getbinpkg" in self.myopts, - "--getbinpkgonly" in self.myopts) - del trees self._required_set_names = set(["system", "world"]) class _dynamic_depgraph_config(object): - def __init__(self, myparams): + def __init__(self, depgraph, myparams): self.myparams = myparams # Maps slot atom to package for each Package added to the graph. self._slot_pkg_map = {} @@ -179,8 +103,12 @@ class _dynamic_depgraph_config(object): # Contains a filtered view of preferred packages that are selected # from available repositories. self._filtered_trees = {} - # All Package instances - self._pkg_cache = {} + # Contains installed packages and new packages that have been added + # to the graph. + self._graph_trees = {} + # Caches visible packages returned from _select_package, for use in + # depgraph._iter_atoms_for_pkg() SLOT logic. + self._visible_pkgs = {} #contains the args created by select_files self._initial_arg_list = [] self.digraph = portage.digraph() @@ -230,6 +158,87 @@ class _dynamic_depgraph_config(object): self._ignored_deps = [] self._highest_pkg_cache = {} + for myroot in depgraph._frozen_config.trees: + self._slot_pkg_map[myroot] = {} + vardb = depgraph._frozen_config.trees[myroot]["vartree"].dbapi + preload_installed_pkgs = \ + "--nodeps" not in depgraph._frozen_config.myopts and \ + "--buildpkgonly" not in depgraph._frozen_config.myopts + # This fakedbapi instance will model the state that the vdb will + # have after new packages have been installed. + fakedb = PackageVirtualDbapi(vardb.settings) + if preload_installed_pkgs: + for pkg in vardb: + depgraph._frozen_config.spinner.update() + # This triggers metadata updates via FakeVartree. + vardb.aux_get(pkg.cpv, []) + fakedb.cpv_inject(pkg) + + # Now that the vardb state is cached in our FakeVartree, + # we won't be needing the real vartree cache for awhile. + # To make some room on the heap, clear the vardbapi + # caches. + depgraph._frozen_config._trees_orig[myroot + ]["vartree"].dbapi._clear_cache() + gc.collect() + + self.mydbapi[myroot] = fakedb + def graph_tree(): + pass + graph_tree.dbapi = fakedb + self._graph_trees[myroot] = {} + self._filtered_trees[myroot] = {} + # Substitute the graph tree for the vartree in dep_check() since we + # want atom selections to be consistent with package selections + # have already been made. + self._graph_trees[myroot]["porttree"] = graph_tree + self._graph_trees[myroot]["vartree"] = graph_tree + def filtered_tree(): + pass + filtered_tree.dbapi = _dep_check_composite_db(depgraph, myroot) + self._filtered_trees[myroot]["porttree"] = filtered_tree + self._visible_pkgs[myroot] = PackageVirtualDbapi(vardb.settings) + + # Passing in graph_tree as the vartree here could lead to better + # atom selections in some cases by causing atoms for packages that + # have been added to the graph to be preferred over other choices. + # However, it can trigger atom selections that result in + # unresolvable direct circular dependencies. For example, this + # happens with gwydion-dylan which depends on either itself or + # gwydion-dylan-bin. In case gwydion-dylan is not yet installed, + # gwydion-dylan-bin needs to be selected in order to avoid a + # an unresolvable direct circular dependency. + # + # To solve the problem described above, pass in "graph_db" so that + # packages that have been added to the graph are distinguishable + # from other available packages and installed packages. Also, pass + # the parent package into self._select_atoms() calls so that + # unresolvable direct circular dependencies can be detected and + # avoided when possible. + self._filtered_trees[myroot]["graph_db"] = graph_tree.dbapi + self._filtered_trees[myroot]["vartree"] = \ + depgraph._frozen_config.trees[myroot]["vartree"] + + dbs = [] + portdb = depgraph._frozen_config.trees[myroot]["porttree"].dbapi + bindb = depgraph._frozen_config.trees[myroot]["bintree"].dbapi + vardb = depgraph._frozen_config.trees[myroot]["vartree"].dbapi + # (db, pkg_type, built, installed, db_keys) + if "--usepkgonly" not in depgraph._frozen_config.myopts: + db_keys = list(portdb._aux_cache_keys) + dbs.append((portdb, "ebuild", False, False, db_keys)) + if "--usepkg" in depgraph._frozen_config.myopts: + db_keys = list(bindb._aux_cache_keys) + dbs.append((bindb, "binary", True, False, db_keys)) + db_keys = list(depgraph._frozen_config._trees_orig[myroot + ]["vartree"].dbapi._aux_cache_keys) + dbs.append((vardb, "installed", True, True, db_keys)) + self._filtered_trees[myroot]["dbs"] = dbs + if "--usepkg" in depgraph._frozen_config.myopts: + depgraph._frozen_config._trees_orig[myroot + ]["bintree"].populate( + "--getbinpkg" in depgraph._frozen_config.myopts, + "--getbinpkgonly" in depgraph._frozen_config.myopts) class depgraph(object): @@ -237,11 +246,14 @@ class depgraph(object): _dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] - def __init__(self, settings, trees, myopts, myparams, spinner): - self._dynamic_config = _dynamic_depgraph_config(myparams) - self._frozen_config = _frozen_depgraph_config(settings, trees, \ - myopts, spinner, self._dynamic_config, self) - + def __init__(self, settings, trees, myopts, myparams, spinner, + frozen_config=None): + if frozen_config is None: + frozen_config = _frozen_depgraph_config(settings, trees, + myopts, spinner) + self._frozen_config = frozen_config + self._dynamic_config = _dynamic_depgraph_config(self, myparams) + self._select_atoms = self._select_atoms_highest_available self._select_package = self._select_pkg_highest_available @@ -1064,7 +1076,8 @@ class depgraph(object): if atom_cp != pkg.cp and \ self._have_new_virt(pkg.root, atom_cp): continue - visible_pkgs = root_config.visible_pkgs.match_pkgs(atom) + visible_pkgs = \ + self._dynamic_config._visible_pkgs[pkg.root].match_pkgs(atom) visible_pkgs.reverse() # descending order higher_slot = None for visible_pkg in visible_pkgs: @@ -1124,12 +1137,9 @@ class depgraph(object): os.path.realpath(self._frozen_config.trees[myroot]["bintree"].getname(mykey)): print colorize("BAD", "\n*** You need to adjust PKGDIR to emerge this package.\n") return 0, myfavorites - db_keys = list(bindb._aux_cache_keys) - metadata = izip(db_keys, bindb.aux_get(mykey, db_keys)) - pkg = Package(type_name="binary", root_config=root_config, - cpv=mykey, built=True, metadata=metadata, + + pkg = self._pkg(mykey, "binary", root_config, onlydeps=onlydeps) - self._dynamic_config._pkg_cache[pkg] = pkg args.append(PackageArg(arg=x, package=pkg, root_config=root_config)) elif ext==".ebuild": @@ -1162,11 +1172,8 @@ class depgraph(object): else: raise portage.exception.PackageNotFound( "%s is not in a valid portage tree hierarchy or does not exist" % x) - db_keys = list(portdb._aux_cache_keys) - metadata = izip(db_keys, portdb.aux_get(mykey, db_keys)) - pkg = Package(type_name="ebuild", root_config=root_config, - cpv=mykey, metadata=metadata, onlydeps=onlydeps) - self._dynamic_config._pkg_cache[pkg] = pkg + pkg = self._pkg(mykey, "ebuild", root_config, + onlydeps=onlydeps) args.append(PackageArg(arg=x, package=pkg, root_config=root_config)) elif x.startswith(os.path.sep): @@ -1346,7 +1353,6 @@ class depgraph(object): portage.writemsg("\n", noiselevel=-1) # Order needs to be preserved since a feature of --nodeps # is to allow the user to force a specific merge order. - args.reverse() self._dynamic_config._initial_arg_list = args[:] return self._resolve(myfavorites) @@ -1615,7 +1621,7 @@ class depgraph(object): added to the graph or those that are installed and have not been scheduled for replacement. """ - kwargs["trees"] = self._frozen_config._graph_trees + kwargs["trees"] = self._dynamic_config._graph_trees return self._select_atoms_highest_available(*pargs, **kwargs) def _select_atoms_highest_available(self, root, depstring, @@ -1892,7 +1898,7 @@ class depgraph(object): settings = pkg.root_config.settings if visible(settings, pkg) and not (pkg.installed and \ settings._getMissingKeywords(pkg.cpv, pkg.metadata)): - pkg.root_config.visible_pkgs.cpv_inject(pkg) + self._dynamic_config._visible_pkgs[pkg.root].cpv_inject(pkg) return ret def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False): @@ -2145,7 +2151,7 @@ class depgraph(object): those that are installed and have not been scheduled for replacement. """ - graph_db = self._frozen_config._graph_trees[root]["porttree"].dbapi + graph_db = self._dynamic_config._graph_trees[root]["porttree"].dbapi matches = graph_db.match_pkgs(atom) if not matches: return None, None @@ -2253,7 +2259,7 @@ class depgraph(object): operation = "merge" if installed or onlydeps: operation = "nomerge" - pkg = self._dynamic_config._pkg_cache.get( + pkg = self._frozen_config._pkg_cache.get( (type_name, root_config.root, cpv, operation)) if pkg is None and onlydeps and not installed: # Maybe it already got pulled in as a "merge" node. @@ -2272,7 +2278,7 @@ class depgraph(object): pkg = Package(built=(type_name != "ebuild"), cpv=cpv, installed=installed, metadata=metadata, root_config=root_config, type_name=type_name) - self._dynamic_config._pkg_cache[pkg] = pkg + self._frozen_config._pkg_cache[pkg] = pkg return pkg def _validate_blockers(self): @@ -2374,7 +2380,7 @@ class depgraph(object): try: success, atoms = portage.dep_check(depstr, final_db, pkgsettings, myuse=pkg.use.enabled, - trees=self._frozen_config._graph_trees, myroot=myroot) + trees=self._dynamic_config._graph_trees, myroot=myroot) except Exception, e: if isinstance(e, SystemExit): raise @@ -2553,7 +2559,6 @@ class depgraph(object): operation="uninstall", root_config=inst_pkg.root_config, type_name=inst_pkg.type_name) - self._dynamic_config._pkg_cache[uninst_task] = uninst_task # Enforce correct merge order with a hard dep. self._dynamic_config.digraph.addnode(uninst_task, inst_task, priority=BlockerDepPriority.instance) @@ -2995,8 +3000,8 @@ class depgraph(object): continue root_config = self._frozen_config.roots[task.root] - inst_pkg = self._dynamic_config._pkg_cache[ - ("installed", task.root, task.cpv, "nomerge")] + inst_pkg = self._pkg(task.cpv, "installed", root_config, + installed=True) if self._dynamic_config.digraph.contains(inst_pkg): continue @@ -3547,8 +3552,9 @@ class depgraph(object): # Remove the corresponding "nomerge" node and substitute # the Uninstall node. - inst_pkg = self._dynamic_config._pkg_cache[ - ("installed", uninstall.root, uninstall.cpv, "nomerge")] + inst_pkg = self._pkg(uninstall.cpv, "installed", + uninstall.root_config, installed=True) + try: mygraph.remove(inst_pkg) except KeyError: @@ -4376,30 +4382,16 @@ class depgraph(object): continue if action != "merge": continue - tree_type = self.pkg_tree_map[pkg_type] - mydb = trees[myroot][tree_type].dbapi - db_keys = list(self._frozen_config._trees_orig[myroot][ - tree_type].dbapi._aux_cache_keys) + root_config = self._frozen_config.roots[myroot] try: - metadata = izip(db_keys, mydb.aux_get(pkg_key, db_keys)) - except KeyError: + pkg = self._pkg(pkg_key, pkg_type, root_config) + except portage.exception.PackageNotFound: # It does no exist or it is corrupt. - if action == "uninstall": - continue if skip_missing: # TODO: log these somewhere continue - raise portage.exception.PackageNotFound(pkg_key) - installed = action == "uninstall" - built = pkg_type != "ebuild" - root_config = self._frozen_config.roots[myroot] - pkg = Package(built=built, cpv=pkg_key, - installed=installed, metadata=metadata, - operation=action, root_config=root_config, - type_name=pkg_type) - self._dynamic_config._pkg_cache[pkg] = pkg + raise - root_config = self._frozen_config.roots[pkg.root] if "merge" == pkg.operation and \ not visible(root_config.settings, pkg): if skip_masked: @@ -4767,9 +4759,11 @@ def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): skip_unsatisfied = True mergelist = mtimedb["resume"]["mergelist"] dropped_tasks = set() + frozen_config = _frozen_depgraph_config(settings, trees, + myopts, spinner) while True: mydepgraph = depgraph(settings, trees, - myopts, myparams, spinner) + myopts, myparams, spinner, frozen_config=frozen_config) try: success = mydepgraph._loadResumeCommand(mtimedb["resume"], skip_masked=skip_masked) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index ed480c60a..762c3c1aa 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5749,13 +5749,12 @@ def _prepare_workdir(mysettings): (mysettings["CATEGORY"], mysettings["PF"], logid_time)) del logid_path, logid_time else: - # When sesandbox is enabled, only log if PORT_LOGDIR is explicitly - # enabled since it is possible that local SELinux security policies - # do not allow ouput to be piped out of the sesandbox domain. - if not (mysettings.selinux_enabled() and \ - "sesandbox" in mysettings.features): - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["T"], "build.log") + # NOTE: When sesandbox is enabled, the local SELinux security policies + # may not allow output to be piped out of the sesandbox domain. The + # current policy will allow it to work when a pty is available, but + # not through a normal pipe. See bug #162404. + mysettings["PORTAGE_LOG_FILE"] = os.path.join( + mysettings["T"], "build.log") def _doebuild_exit_status_check(mydo, settings): """ diff --git a/pym/portage/sets/base.py b/pym/portage/sets/base.py index 370b03d45..87a0d1db7 100644 --- a/pym/portage/sets/base.py +++ b/pym/portage/sets/base.py @@ -142,10 +142,10 @@ class PackageSet(object): for atom in atoms: if match_from_list(atom, cpv_slot_list): yield atom - if not pkg.metadata["PROVIDE"]: + provides = pkg.metadata['PROVIDE'] + if not provides: return - provides = flatten(use_reduce(paren_reduce(pkg.metadata["PROVIDE"]), - uselist=pkg.metadata["USE"].split())) + provides = provides.split() for provide in provides: provided_cp = dep_getkey(provide) atoms = self._atommap.get(provided_cp) -- 2.26.2