From 44395db8d2d11f9e2433368553b4a4e680945382 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 17 Oct 2006 19:44:37 +0000 Subject: [PATCH] Merged with trunk 4727:4738 Remove blocker nodes from the digraph prior to the merge order calculation so that they don't trigger false circular dependencies as reported in comment #17 of bug #151331. Blockers that depend on merge order must be validated by depgraph.altlist(reversed=False) before the reverse tree is generated, since the validation won't work in reverse mode. Fix altlist blocker validation logic so that the correct nodes are added to the list when called in reverse mode. Always add blockers with priority=digraph.SOFT so that blockers with multiple parents can't trigger a circular deps panick. Make --depclean treat DEPEND as soft deps instead of forcing the user to have all build time deps installed. Use setdefault to avoid an extra dict lookup. Merge package.use and USE from make.defaults together at each level of the profile stack. This is analogous to how use.mask and package.use.mask are planned to be merged for bug #151586. Stack use.mask/package.use.mask together at each level of the profile for bug #151586, and do the same with use.force/package.use.force. Add back the -v option for tar (removed for bug #151146) in dyn_package(), but don't enable -v when --quiet is enabled. Encapsulate altlist calls (for blockers that depend on merge order) in validate_blockers(). svn path=/main/branches/prefix/; revision=4739 --- bin/emerge | 81 ++++++++++++++---------- bin/misc-functions.sh | 5 +- cnf/make.globals | 2 +- pym/portage.py | 143 +++++++++++++++++++++++------------------- pym/portage_util.py | 10 ++- 5 files changed, 138 insertions(+), 103 deletions(-) diff --git a/bin/emerge b/bin/emerge index 4cf92f5bf..9cbeec6f9 100755 --- a/bin/emerge +++ b/bin/emerge @@ -735,7 +735,7 @@ class depgraph: if addme and "--buildpkgonly" not in self.myopts and myparent: mybigkey[1] = myparent.split()[1] self.digraph.addnode(" ".join(mybigkey), myparent, - priority=priority) + priority=digraph.SOFT) return 1 if not arg: arg = portage.best_match_to_list(mykey, self.args_keys) @@ -772,9 +772,7 @@ class depgraph: pkgsettings.setcpv(mykey, mydb=portdb) forced_flags = set() forced_flags.update(pkgsettings.useforce) - forced_flags.update(pkgsettings.puseforce) forced_flags.update(pkgsettings.usemask) - forced_flags.update(pkgsettings.pusemask) old_use = vardbapi.aux_get(mykey, ["USE"])[0].split() iuses = set(filter_iuse_defaults( mydbapi.aux_get(mykey, ["IUSE"])[0].split())) @@ -1182,9 +1180,7 @@ class depgraph: now_use = pkgsettings["USE"].split() forced_flags = set() forced_flags.update(pkgsettings.useforce) - forced_flags.update(pkgsettings.puseforce) forced_flags.update(pkgsettings.usemask) - forced_flags.update(pkgsettings.pusemask) cur_iuse = iuses if "--usepkgonly" not in self.myopts and myeb: cur_iuse = set(filter_iuse_defaults( @@ -1383,9 +1379,18 @@ class depgraph: invalidated after nodes referencing it have been merged.""" self.blocker_digraph.addnode(node, blocker) self.digraph.remove(blocker) + # Validate blockers that depend on merge order. + self.altlist() def altlist(self, reversed=False): mygraph=self.digraph.copy() + for node in self.blocker_digraph.root_nodes(): + """These nodes aren't needed for the merge order calculation. They + may be in the digraph for --tree support, but they need to be + removed prior to the merge order calculation so that they don't + trigger any false circular dependency issues.""" + if mygraph.contains(node): + mygraph.remove(node) myblockers = self.blocker_digraph.copy() retlist=[] circular_blocks = False @@ -1458,10 +1463,21 @@ class depgraph: if not myblockers.child_nodes(blocker): myblockers.remove(blocker) - for node in myblockers.root_nodes(): - retlist.append(node.split()) - for parent in self.blocker_parents[node]: - self.digraph.add(node, parent) + if reversed: + for node in self.blocker_digraph.root_nodes(): + if self.digraph.contains(node): + """This blocker has been validated by a previous merge + order calculation.""" + retlist.append(node.split()) + else: + """ Valid blockers need to be in the digraph for --tree + support. Blocker validation does not work with reverse mode, + so self.altlist() should first be called with reverse disabled + so that blockers are properly validated.""" + for node in myblockers.root_nodes(): + retlist.append(node.split()) + for parent in self.blocker_parents[node]: + self.digraph.add(node, parent, priority=digraph.SOFT) return retlist @@ -1780,9 +1796,7 @@ class depgraph: forced_flags = set() if not binary_package: forced_flags.update(pkgsettings.useforce) - forced_flags.update(pkgsettings.puseforce) forced_flags.update(pkgsettings.usemask) - forced_flags.update(pkgsettings.pusemask) cur_iuse = portage.unique_array(cur_iuse) cur_iuse.sort() @@ -3628,19 +3642,19 @@ def action_depclean(settings, trees, ldpath_mtimes, if "--quiet" not in myopts: print "\nCalculating dependencies ", - remaining_atoms = [(atom, 'world') for atom in worldlist if vardb.match(atom)] - remaining_atoms += [(atom, 'system') for atom in syslist if vardb.match(atom)] + soft = 0 + hard = 1 + remaining_atoms = [(atom, 'world', hard) for atom in worldlist if vardb.match(atom)] + remaining_atoms += [(atom, 'system', hard) for atom in syslist if vardb.match(atom)] unresolveable = {} + aux_keys = ["DEPEND", "RDEPEND", "PDEPEND"] while remaining_atoms: - atom, parent = remaining_atoms.pop() + atom, parent, priority = remaining_atoms.pop() pkgs = vardb.match(atom) if not pkgs: - if not atom.startswith("!"): - if atom in unresolveable: - unresolveable[atom].append(parent) - else: - unresolveable[atom] = [parent] + if not atom.startswith("!") and priority == hard: + unresolveable.setdefault(atom, []).append(parent) continue # Could put slot checking here to ensure that there aren't two # packages with the same slot... @@ -3650,22 +3664,23 @@ def action_depclean(settings, trees, ldpath_mtimes, spinner.update() fakedb.cpv_inject(pkg) mydb = portdb.xmatch("match-visible", "="+pkg) and portdb or vardb + myaux = dict(zip(aux_keys, mydb.aux_get(pkg, aux_keys))) + mydeps = [] if myopts.get("--with-bdeps", "y") == "y": - depstr = " ".join(mydb.aux_get(pkg, - ["DEPEND", "RDEPEND", "PDEPEND"])) - else: - depstr = " ".join(mydb.aux_get(pkg, - ["RDEPEND", "PDEPEND"])) + mydeps.append((myaux["DEPEND"], soft)) + del myaux["DEPEND"] + mydeps.append((" ".join(myaux.values()), hard)) usedef = vardb.aux_get(pkg, ["USE"])[0].split() - success, atoms = portage.dep_check(depstr, emptydb, settings, - myuse=usedef, trees=trees, myroot=settings["ROOT"]) - if not success: - print "\n\n" - print "Error occurred while processing",pkg - print str(atoms) - return - for atom in atoms: - remaining_atoms.append((atom, pkg)) + for depstr, priority in mydeps: + success, atoms = portage.dep_check(depstr, emptydb, settings, + myuse=usedef, trees=trees, myroot=settings["ROOT"]) + if not success: + print "\n\n" + print "Error occurred while processing",pkg + print str(atoms) + return + for atom in atoms: + remaining_atoms.append((atom, pkg, priority)) if "--quiet" not in myopts: print "\b\b... done!\n" diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index be5e7d0c5..7ba0a6ef6 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -380,10 +380,13 @@ dyn_package() { install_mask "${PORTAGE_BUILDDIR}/image" ${PKG_INSTALL_MASK} local pkg_dest="${PKGDIR}/All/${PF}.tbz2" local pkg_tmp="${PKGDIR}/All/${PF}.tbz2.$$" + local tar_options="" + [ "${PORTAGE_QUIET}" == "1" ] || tar_options="${tar_options} -v" # Sandbox is disabled in case the user wants to use a symlink # for $PKGDIR and/or $PKGDIR/All. export SANDBOX_ON="0" - tar -cf - . | bzip2 -f > "${pkg_tmp}" || die "Failed to create tarball" + tar ${tar_options} -cf - . | bzip2 -f > "${pkg_tmp}" || \ + die "Failed to create tarball" cd .. export PYTHONPATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} python -c "import xpak; t=xpak.tbz2('${pkg_tmp}'); t.recompose('${PORTAGE_BUILDDIR}/build-info')" diff --git a/cnf/make.globals b/cnf/make.globals index 6fc397d7a..c4ea2b572 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -60,7 +60,7 @@ PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --compress CONFIG_PROTECT="/etc" # Disable auto-use -USE_ORDER="env:pkg:conf:pkgprofile:defaults:pkginternal" +USE_ORDER="env:pkg:conf:defaults:pkginternal" # Default portage user/group PORTAGE_USER="@portageuser@" diff --git a/pym/portage.py b/pym/portage.py index b82ae32b8..fa6fbad37 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -853,9 +853,7 @@ class config: self.locked = 0 self.mycpv = None - self.puseforce = [] self.puse = [] - self.pusemask = [] self.modifiedkeys = [] self.uvlist = [] @@ -894,12 +892,13 @@ class config: self.use_defs = copy.deepcopy(clone.use_defs) self.usemask = copy.deepcopy(clone.usemask) - self.pusemaskdict = copy.deepcopy(clone.pusemaskdict) - self.pusemask = copy.deepcopy(clone.pusemask) + self.usemask_list = copy.deepcopy(clone.usemask_list) + self.pusemask_list = copy.deepcopy(clone.pusemask_list) self.useforce = copy.deepcopy(clone.useforce) - self.puseforcedict = copy.deepcopy(clone.puseforcedict) - self.puseforce = copy.deepcopy(clone.puseforce) + self.useforce_list = copy.deepcopy(clone.useforce_list) + self.puseforce_list = copy.deepcopy(clone.puseforce_list) self.puse = copy.deepcopy(clone.puse) + self.make_defaults_use = copy.deepcopy(clone.make_defaults_use) self.pkgprofileuse = copy.deepcopy(clone.pkgprofileuse) self.mycpv = copy.deepcopy(clone.mycpv) @@ -911,12 +910,11 @@ class config: "pkginternal": self.configlist[1], "globals": self.configlist[2], "defaults": self.configlist[3], - "pkgprofile": self.configlist[4], - "conf": self.configlist[5], - "pkg": self.configlist[6], - "auto": self.configlist[7], - "backupenv": self.configlist[8], - "env": self.configlist[9] } + "conf": self.configlist[4], + "pkg": self.configlist[5], + "auto": self.configlist[6], + "backupenv": self.configlist[7], + "env": self.configlist[8] } self.profiles = copy.deepcopy(clone.profiles) self.backupenv = self.configdict["backupenv"] self.pusedict = copy.deepcopy(clone.pusedict) @@ -1034,48 +1032,50 @@ class config: self.prevmaskdict[mycatpkg].append(x) # get profile-masked use flags -- INCREMENTAL Child over parent - usemask_lists = [grabfile(os.path.join(x, "use.mask")) for x in self.profiles] - self.usemask = stack_lists(usemask_lists, incremental=True) - del usemask_lists + self.usemask_list = [grabfile(os.path.join(x, "use.mask")) \ + for x in self.profiles] + self.usemask = set(stack_lists( + self.usemask_list, incremental=True)) use_defs_lists = [grabdict(os.path.join(x, "use.defaults")) for x in self.profiles] self.use_defs = stack_dictlist(use_defs_lists, incremental=True) del use_defs_lists - self.pusemaskdict = {} + self.pusemask_list = [] rawpusemask = [grabdict_package( os.path.join(x, "package.use.mask")) \ for x in self.profiles] - rawpusemask = stack_dictlist(rawpusemask, incremental=True) - for k, v in rawpusemask.iteritems(): - cp = dep_getkey(k) - self.pusemaskdict.setdefault(cp, {}) - self.pusemaskdict[cp][k] = v + for i in xrange(len(self.profiles)): + cpdict = {} + for k, v in rawpusemask[i].iteritems(): + cpdict.setdefault(dep_getkey(k), {})[k] = v + self.pusemask_list.append(cpdict) del rawpusemask - self.pkgprofileuse = {} + self.pkgprofileuse = [] rawprofileuse = [grabdict_package( os.path.join(x, "package.use"), juststrings=True) \ for x in self.profiles] - rawprofileuse = stack_dicts(rawprofileuse, incremental=True) - for k, v in rawprofileuse.iteritems(): - cp = dep_getkey(k) - self.pkgprofileuse.setdefault(cp, {}) - self.pkgprofileuse[cp][k] = v + for i in xrange(len(self.profiles)): + cpdict = {} + for k, v in rawprofileuse[i].iteritems(): + cpdict.setdefault(dep_getkey(k), {})[k] = v + self.pkgprofileuse.append(cpdict) del rawprofileuse - self.useforce = stack_lists( - [grabfile(os.path.join(x, "use.force")) \ - for x in self.profiles], incremental=True) + self.useforce_list = [grabfile(os.path.join(x, "use.force")) \ + for x in self.profiles] + self.useforce = set(stack_lists( + self.useforce_list, incremental=True)) - self.puseforcedict = {} + self.puseforce_list = [] rawpuseforce = [grabdict_package( os.path.join(x, "package.use.force")) \ for x in self.profiles] - rawpuseforce = stack_dictlist(rawpuseforce, incremental=True) - for k, v in rawpuseforce.iteritems(): - cp = dep_getkey(k) - self.puseforcedict.setdefault(cp, {}) - self.puseforcedict[cp][k] = v + for i in xrange(len(self.profiles)): + cpdict = {} + for k, v in rawpuseforce[i].iteritems(): + cpdict.setdefault(dep_getkey(k), {})[k] = v + self.puseforce_list.append(cpdict) del rawpuseforce try: @@ -1095,10 +1095,16 @@ class config: self.configlist.append(self.mygcfg) self.configdict["globals"]=self.configlist[-1] + self.make_defaults_use = [] self.mygcfg = {} if self.profiles: try: mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults")) for x in self.profiles] + for cfg in mygcfg_dlists: + if cfg: + self.make_defaults_use.append(cfg.get("USE", "")) + else: + self.make_defaults_use.append("") self.mygcfg = stack_dicts(mygcfg_dlists, incrementals=portage_const.INCREMENTALS, ignore_none=1) #self.mygcfg = grab_stacked("make.defaults", self.profiles, getconfig) if self.mygcfg is None: @@ -1117,9 +1123,6 @@ class config: self.configlist.append(self.mygcfg) self.configdict["defaults"]=self.configlist[-1] - self.configlist.append({}) - self.configdict["pkgprofile"] = self.configlist[-1] - try: self.mygcfg = getconfig( os.path.join(config_root, MAKE_CONF_FILE.lstrip(os.path.sep)), @@ -1301,7 +1304,7 @@ class config: # reasonable defaults; this is important as without USE_ORDER, # USE will always be "" (nothing set)! if "USE_ORDER" not in self: - self.backupenv["USE_ORDER"] = "env:pkg:conf:pkgprofile:defaults:pkginternal" + self.backupenv["USE_ORDER"] = "env:pkg:conf:defaults:pkginternal" self["PORTAGE_GID"] = str(portage_gid) self.backup_changes("PORTAGE_GID") @@ -1462,11 +1465,14 @@ class config: if not keeping_pkg: self.mycpv = None self.puse = "" - self.pusemask = [] - self.puseforce = [] self.configdict["pkg"].clear() self.configdict["pkginternal"].clear() - self.configdict["pkgprofile"].clear() + self.configdict["defaults"]["USE"] = \ + " ".join(self.make_defaults_use) + self.usemask = set(stack_lists( + self.usemask_list, incremental=True)) + self.useforce = set(stack_lists( + self.useforce_list, incremental=True)) self.regenerate(use_cache=use_cache) def load_infodir(self,infodir): @@ -1522,30 +1528,39 @@ class config: for x in mydb.aux_get(mycpv, ["IUSE"])[0].split() \ if x.startswith("+")]) self.configdict["pkginternal"]["USE"] = pkginternaluse - pkgprofileuse = "" - if cp in self.pkgprofileuse: - best_match = best_match_to_list( - self.mycpv, self.pkgprofileuse[cp].keys()) - if best_match: - pkgprofileuse = self.pkgprofileuse[cp][best_match] - self.configdict["pkgprofile"]["USE"] = pkgprofileuse + defaults = [] + for i in xrange(len(self.profiles)): + profile_use = self.make_defaults_use[i] + cpdict = self.pkgprofileuse[i].get(cp, None) + if cpdict: + best_match = best_match_to_list(self.mycpv, cpdict.keys()) + if best_match: + profile_use += " " + cpdict[best_match] + defaults.append(profile_use) + self.configdict["defaults"]["USE"] = " ".join(defaults) + useforce = [] + for i in xrange(len(self.profiles)): + useforce.append(self.useforce_list[i]) + cpdict = self.puseforce_list[i].get(cp, None) + if cpdict: + best_match = best_match_to_list(self.mycpv, cpdict.keys()) + if best_match: + useforce.append(cpdict[best_match]) + self.useforce = set(stack_lists(useforce, incremental=True)) + usemask = [] + for i in xrange(len(self.profiles)): + usemask.append(self.usemask_list[i]) + cpdict = self.pusemask_list[i].get(cp, None) + if cpdict: + best_match = best_match_to_list(self.mycpv, cpdict.keys()) + if best_match: + usemask.append(cpdict[best_match]) + self.usemask = set(stack_lists(usemask, incremental=True)) self.puse = "" if self.pusedict.has_key(cp): self.pusekey = best_match_to_list(self.mycpv, self.pusedict[cp].keys()) if self.pusekey: self.puse = " ".join(self.pusedict[cp][self.pusekey]) - self.pusemask = [] - if cp in self.pusemaskdict: - pusemaskkey = best_match_to_list(self.mycpv, - self.pusemaskdict[cp].keys()) - if pusemaskkey: - self.pusemask = set(self.pusemaskdict[cp][pusemaskkey]) - self.puseforce = [] - if cp in self.puseforcedict: - puseforcekey = best_match_to_list(self.mycpv, - self.puseforcedict[cp].keys()) - if puseforcekey: - self.puseforce = self.puseforcedict[cp][puseforcekey][:] self.configdict["pkg"]["PKGUSE"] = self.puse[:] # For saving to PUSE file self.configdict["pkg"]["USE"] = self.puse[:] # this gets appended to USE # CATEGORY is essential for doebuild calls @@ -1711,10 +1726,9 @@ class config: myflags = set(myflags) myflags.update(self.useforce) - myflags.update(self.puseforce) usesplit = [ x for x in myflags if \ - x not in self.usemask and x not in self.pusemask ] + x not in self.usemask] usesplit.sort() # Use the calculated USE flags to regenerate the USE_EXPAND flags so @@ -3598,7 +3612,6 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, # that is being merged to a $ROOT that is different from the one # that mysettings represents. mymasks.update(mysettings.usemask) - mymasks.update(mysettings.pusemask) mymasks.update(mysettings.archlist()) mymasks.discard(mysettings["ARCH"]) mysplit = portage_dep.use_reduce(mysplit,uselist=myusesplit,masklist=mymasks,matchall=(use=="all"),excludeall=[mysettings["ARCH"]]) diff --git a/pym/portage_util.py b/pym/portage_util.py index 32f673772..81f842044 100644 --- a/pym/portage_util.py +++ b/pym/portage_util.py @@ -137,9 +137,13 @@ def stack_lists(lists, incremental=1): new_list = {} for x in lists: for y in filter(None, x): - if incremental and y.startswith("-"): - if y[1:] in new_list: - del new_list[y[1:]] + if incremental: + if y == "-*": + new_list.clear() + elif y.startswith("-"): + new_list.pop(y[1:], None) + else: + new_list[y] = True else: new_list[y] = True return new_list.keys() -- 2.26.2