portage-2.1.5
==================================
+* The metadata-transfer feature is now disabled by default. This disables the
+ "Updating Portage cache" routine that used to run at the tail end of each
+ `emerge --sync` operation. If you use something like the sqlite module and
+ want to keep all metadata in that format alone (useful for querying), enable
+ FEATURES="metadata-transfer" in make.conf.
+
* The parallel-fetch feature is now enabled by default. It is optimized
to avoid doing redundant checksums for previously downloaded files that have
the correct size. Run `tail -f /var/log/emerge-fetch.log` in a
import os
-# This allows portageq to be exempt from sandbox,
-# for things like updating metadata cache.
-os.environ["SANDBOX_ON"] = "0"
-
import types
#-----------------------------------------------------------------------------
RESUMECOMMAND="wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
# Default user options
-FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
+FEATURES="distlocks parallel-fetch sandbox sfperms strict unmerge-orphans userfetch"
# Enable for global testing
FEATURES="${FEATURES} preserve-libs"
--- /dev/null
+<part id='dependency-resolution'>
+<title>Dependency Resolution</title>
+&dependency_resolution_package_modeling;
+&dependency_resolution_decision_making;
+&dependency_resolution_task_scheduling;
+</part>
--- /dev/null
+<chapter id='dependency-resolution-decision-making'>
+<title>Decision Making</title>
+<sect1 id='dependency-resolution-decision-making-dependency-expression-evaluation'>
+ <title>Dependency Expression Evaluation</title>
+ <para>
+ In terms of boolean logic, a dependency expression can
+ be expressed in disjunctive normal form (DNF), which is
+ a disjunction of conjunctive clauses. Each conjunctive clause
+ represents one possible alternative combination of dependency
+ atoms capable of satisfying the dependency expression.
+ </para>
+</sect1>
+<sect1 id='dependency-resolution-decision-making-look-ahead'>
+ <title>Look-Ahead</title>
+ <para>
+ When there are multiple combinations to choose from,
+ a look-ahead mechanism will choose an optimal combination
+ to satisfy constraints and minimize cost. The
+ following package states influence the cost calculation for
+ a given combination:
+ <itemizedlist>
+ <listitem>
+ installed
+ </listitem>
+ <listitem>
+ selected (for installation)
+ </listitem>
+ <listitem>
+ not selected (for installation)
+ </listitem>
+ </itemizedlist>
+ </para>
+ <para>
+ In cost calculations, virtual packages by themselves are
+ considered to cost nothing since they do not directly install anything.
+ It is the dependencies of a virtual package that contribute to it's cost.
+ </para>
+ <sect2 id='dependency-resolution-decision-making-look-ahead-constraint-propagation'>
+ <title>Constraint Propagation</title>
+ <para>
+ Combinations that include packages from the "installed" or
+ "selected" categories are less costly than those that
+ include packages from the "not selected" category.
+ When a package is chosen for installation, it transitions to the
+ "selected" state. This state change propagates
+ to the cost calculations of later decisions,
+ influencing later decisions to be consistent with earlier decisions.
+ This feedback mechanism serves to propagate constraints and can
+ influence the modeling process to
+ converge on a more optimal final state.
+ </para>
+ </sect2>
+ <sect2 id='dependency-resolution-decision-making-look-ahead-expanded-search-space'>
+ <title>Expanded Search Space</title>
+ <para>
+ When evaluating virtual atoms, an expanded search space is
+ considered which recursively traverses
+ the dependencies of virtual packages
+ from all slots matching a given virtual atom. All combinations in
+ this expanded search space are considered when choosing an optimal
+ combination to satisfy constraints with minimal cost.
+ </para>
+ </sect2>
+</sect1>
+</chapter>
--- /dev/null
+<chapter id='dependency-resolution-package-modeling'>
+<title>Package Modeling</title>
+
+<sect1 id='dependency-resolution-package-modeling-constraint-satisfaction'>
+ <title>Constraint Satisfaction</title>
+ <sect2 id='dedependency-resolution-package-modeling-constraint-satisfaction-constraint-types'>
+ <title>Constraint Types</title>
+ <para>
+ Dependency resolution involves satisfaction of
+ many constraints:
+ <itemizedlist>
+ <listitem>
+ Persistent configuration parameters, like those that come from
+ make.profile, make.conf, and the /etc/portage directory.
+ </listitem>
+ <listitem>
+ Current command parameters, which may include options, atoms, or sets.
+ </listitem>
+ <listitem>
+ <link linkend='dependency-resolution-package-modeling-constraint-satisfaction-package-dependencies'>
+ Package Dependencies</link>
+ </listitem>
+ </itemizedlist>
+ </para>
+ </sect2>
+
+ <sect2 id='dependency-resolution-package-modeling-constraint-satisfaction-package-dependencies'>
+ <title>Package Dependencies</title>
+ <para>
+ Common types of package dependencies:
+ <itemizedlist>
+ <listitem>
+ Files required for building or installing. Downloads may
+ be necessary to satisfy these.
+ </listitem>
+ <listitem>
+ Other packages required to be installed for
+ buildtime or runtime.
+ </listitem>
+ <listitem>
+ Blockers that prevent conflicting packages from being installed
+ simultaneously.
+ </listitem>
+ </itemizedlist>
+ </para>
+ </sect2>
+</sect1>
+
+<sect1 id='dependency-resolution-package-modeling-conflicts'>
+ <title>Conflicts</title>
+ <sect2 id='dependency-resolution-package-modeling-blocker-conflicts'>
+ <title>Blocker Conflicts</title>
+ <para>
+ If one package blocks another package, the two packages
+ conflict such that they cannot be installed simultaneously.
+ These conflicts are often due to file collisions.
+ </para>
+ </sect2>
+ <sect2 id='dependency-resolution-package-modeling-slot-conflicts'>
+ <title>Slot Conflicts</title>
+ <para>
+ If two different packages that occupy the same slot are chosen
+ to satisfy dependencies, a slot conflict occurs. The two packages
+ cannot be installed simultaneously and therefore the respective
+ dependencies will not be satisfied simultaneously.
+ </para>
+ </sect2>
+ <sect2 id='dependency-resolution-package-modeling-indirect-conflicts'>
+ <title>Indirect Conflicts</title>
+ <para>
+ If the dependencies of two parent packages cannot be installed
+ simultaneously, it creates an indirect conflict between the parent
+ packages since their respective dependencies cannot be satisfied
+ simultaneously.
+ </para>
+ </sect2>
+</sect1>
+
+<sect1 id='dependency-resolution-package-modeling-dependency-neglection'>
+ <title>Dependency Neglection</title>
+ <para>
+ In order to significantly reduce the resources consumed by
+ the modeling process, the dependencies of
+ installed packages may be neglected.
+ </para>
+ <para>
+ If a more complete dependency calculation is desired,
+ there is a --complete-graph option which will ensure that the
+ dependencies of installed packages are properly considered.
+ </para>
+</sect1>
+
+</chapter>
--- /dev/null
+<chapter id='dependency-resolution-task-scheduling'>
+<title>Task Scheduling</title>
+<sect1 id='dependency-resolution-task-scheduling-dependencies'>
+ <title>Task Dependencies</title>
+ <para>
+ All tasks are executed in an order such
+ that a task's dependencies are satisfied
+ when it is executed. Dependency relationships between tasks
+ form a directed graph.
+ </para>
+</sect1>
+<sect1 id='dependency-resolution-task-scheduling-conflict-avoidance'>
+ <title>Conflict Avoidance</title>
+ <para>
+ In some cases it is possible to adjust package installation order
+ to avoid having two conflicting packages installed simultaneously.
+ </para>
+ <para>
+ TODO: Automatically uninstall packages when necessary to avoid conflicts.
+ </para>
+</sect1>
+<sect1 id='dependency-resolution-task-scheduling-circular-dependencies'>
+ <title>Circular Dependencies</title>
+ <para>
+ TODO: Automatically solve circular dependencies by temporarily disabling
+ conditional dependencies and then rebuilding packages with the conditional
+ dependencies enabled.
+ </para>
+</sect1>
+<sect1 id='dependency-resolution-task-scheduling-parallel'>
+ <title>Parallel Scheduling</title>
+ <para>
+ TODO: Spawn an appropriate number of tasks in parallel when desired.
+ </para>
+</sect1>
+</chapter>
<!ENTITY project "portage">
+ <!ENTITY dependency_resolution SYSTEM "dependency_resolution.docbook">
+ <!ENTITY dependency_resolution_package_modeling SYSTEM "dependency_resolution/package_modeling.docbook">
+ <!ENTITY dependency_resolution_decision_making SYSTEM "dependency_resolution/decision_making.docbook">
+ <!ENTITY dependency_resolution_task_scheduling SYSTEM "dependency_resolution/task_scheduling.docbook">
<!ENTITY package SYSTEM "package.docbook">
<!ENTITY package_ebuild SYSTEM "package/ebuild.docbook">
<!ENTITY package_ebuild_phases SYSTEM "package/ebuild/phases.docbook">
</bookinfo>
&config;
+&dependency_resolution;
&package;
&qa;
populates the cache database that portage uses for pre-parsed lookups of
package data. It does not populate cache for the overlays listed in
PORTDIR_OVERLAY. In order to generate cache for overlays, use \fB\-\-regen\fR.
+In versions of portage >=2.1.5 the \-\-metadata action is totally unnecessary
+unless the user has enabled FEATURES="metadata-transfer" in \fBmake.conf\fR(5).
.TP
.BR "\-\-prune " (\fB\-P\fR)
\fBWARNING: This action can remove important packages!\fR Removes all but the
.TP
.B metadata\-transfer
Automatically perform a metadata transfer when `emerge \-\-sync` is run.
+In versions of portage >=2.1.5, this feature is disabled by
+default. When metadata\-transfer is disabled, metadata cache from the
+${PORTDIR}/metadata/cache/ directory will be used directly (if available)
+and eclasses in ${PORTDIR}/eclass/ must not be modified except by
+`emerge \-\-sync` operations since the cache validation mechanism
+will not recognize eclass modifications. Normally, this issue only
+pertains to users of the rsync tree since the cvs tree does not contain
+a metadata/cache/ directory. Users of the rsync tree who want to modify
+eclasses should use \fBPORTDIR_OVERLAY\fR in order for the cache
+validation mechanism to work correctly.
.TP
.B mirror
Fetch everything in \fBSRC_URI\fR regardless of \fBUSE\fR settings,
.I Example:
.nf
-portdbapi.auxdbmodule = portage.cache.metadata_overlay.database
-.fi
-
-The metadata_overlay cache module makes it possible to disable
-FEATURES="metadata-transfer" in \fBmake.conf\fR(5). When the user initially
-enables metadata_overlay in /etc/portage/modules, all of the cache files
-contained in /var/cache/edb/dep/${PORTDIR} must be manually removed in order
-to avoid unecessary cache regeneration. In addition, users of the
-metadata_overlay module must never modify eclasses in ${PORTDIR} because
-portage will not be able to detect that cache regeneration is necessary. If
-the user would like to modify eclasses, it is safe to use metadata_overlay
-together with PORTDIR_OVERLAY in \fBmake.conf\fR.
+portdbapi.auxdbmodule = portage.cache.sqlite.database
+.fi
+
+After changing the portdbapi.auxdbmodule setting, it may be necessary to
+transfer or regenerate metadata cache. Users of the rsync tree need to
+run `emerge \-\-metadata` if they have enabled FEATURES="metadata-transfer"
+in \fBmake.conf\fR(5). In order to regenerate metadata for repositories
+listed in \fBPORTDIR_OVERLAY\fR or a cvs tree, run `emerge \-\-regen`
+(see \fBemerge\fR(1)). If you use something like the sqlite module and want
+to keep all metadata in that format alone (useful for querying), enable
+FEATURES="metadata-transfer" in \fBmake.conf\fR(5).
.TP
.BR package.keywords
Per\-package KEYWORDS. Useful for mixing unstable packages in with a normally
self.edebug = 0
if settings.get("PORTAGE_DEBUG", "") == "1":
self.edebug = 1
+ if "--nodeps" in myopts:
+ spinner.update = spinner.update_quiet
self.spinner = spinner
self.pkgsettings = {}
# Maps slot atom to package for each Package added to the graph.
msg = []
msg.append("\n!!! Multiple versions within a single " + \
- "package slot have been \n")
- msg.append("!!! pulled into the dependency graph:\n\n")
+ "package slot have been pulled\n")
+ msg.append("!!! into the dependency graph, resulting" + \
+ " in a slot conflict:\n\n")
indent = " "
# Max number of parents shown, to avoid flooding the display.
max_parents = 3
# here, packages that have been masked since they
# were installed can be automatically downgraded
# to an unmasked version.
- if not visible(pkgsettings, pkg):
+ try:
+ if not visible(pkgsettings, pkg):
+ continue
+ except portage.exception.InvalidDependString:
continue
if not built and not calculated_use:
# This is avoided whenever possible because
arg = None
if arg:
return False
- if pkg.installed and \
- not visible(self._depgraph.pkgsettings[pkg.root], pkg):
- return False
+ if pkg.installed:
+ try:
+ if not visible(
+ self._depgraph.pkgsettings[pkg.root], pkg):
+ return False
+ except portage.exception.InvalidDependString:
+ pass
return True
def _dep_expand(self, atom):
ec = portage.eclass_cache.cache(portdb.porttree_root)
myportdir = os.path.realpath(settings["PORTDIR"])
cm = settings.load_best_module("portdbapi.metadbmodule")(
- myportdir, "metadata/cache", portage.auxdbkeys[:])
+ myportdir, "metadata/cache", portage.auxdbkeys[:], readonly=True)
from portage.cache import util
print " "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >"
print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]"
print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] "
- print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"] ["+green("--oneshot")+"] ["+green("--newuse")+"] ["+green("--noconfmem")+"]"
- print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]"
- print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] ["+green("--nospinner")+"]"
- print " [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ] [" + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]"
- print bold("Actions:")+" [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]"
+ print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]"
+ print " [ "+green("--columns")+" ] [ "+green("--deep")+" ] [ "+green("--newuse")+" ]"
+ print " [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+" ]"
+ print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--complete-graph")+" ]"
+ print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]"
+ print bold("Actions:")+" [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]"
def help(myaction,myopts,havecolor=1):
# TODO: Implement a wrap() that accounts for console color escape codes.
continue
if subdir_st.st_gid != portage_gid or \
((stat.S_ISDIR(subdir_st.st_mode) and \
- not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode)) or \
- (not stat.S_ISDIR(subdir_st.st_mode) and \
- not filemode == (stat.S_IMODE(subdir_st.st_mode) & filemode))):
+ not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode))):
droppriv_fix = True
break
if droppriv_fix:
# According to GLEP 37, RDEPEND is the only dependency type that is valid
# for new-style virtuals. Repoman should enforce this.
dep_keys = ["RDEPEND", "DEPEND", "PDEPEND"]
- def compare_pkgs(a, b):
- return pkgcmp(b[1], a[1])
portdb = trees[myroot]["porttree"].dbapi
if kwargs["use_binaries"]:
portdb = trees[myroot]["bintree"].dbapi
match_atom = x
if isblocker:
match_atom = x[1:]
- pkgs = {}
- for cpv in portdb.match(match_atom):
+ pkgs = []
+ matches = portdb.match(match_atom)
+ # Use descending order to prefer higher versions.
+ matches.reverse()
+ for cpv in matches:
# only use new-style matches
if cpv.startswith("virtual/"):
- pkgs[cpv] = (cpv, catpkgsplit(cpv)[1:], portdb)
- if kwargs["use_binaries"] and "vartree" in trees[myroot]:
- vardb = trees[myroot]["vartree"].dbapi
- for cpv in vardb.match(match_atom):
- # only use new-style matches
- if cpv.startswith("virtual/"):
- if cpv in pkgs:
- continue
- pkgs[cpv] = (cpv, catpkgsplit(cpv)[1:], vardb)
+ pkgs.append((cpv, catpkgsplit(cpv)[1:], portdb))
if not (pkgs or mychoices):
# This one couldn't be expanded as a new-style virtual. Old-style
# virtuals have already been expanded by dep_virtual, so this one
if not pkgs and len(mychoices) == 1:
newsplit.append(x.replace(mykey, mychoices[0]))
continue
- pkgs = pkgs.values()
- pkgs.sort(compare_pkgs) # Prefer higher versions.
if isblocker:
a = []
else:
#self.root=settings["PORTDIR"]
self.porttree_root = os.path.realpath(porttree_root)
- self.depcachedir = self.mysettings.depcachedir[:]
+ self.depcachedir = os.path.realpath(self.mysettings.depcachedir)
+
+ if os.environ.get("SANDBOX_ON") == "1":
+ # Make api consumers exempt from sandbox violations
+ # when doing metadata cache updates.
+ sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":")
+ if self.depcachedir not in sandbox_write:
+ sandbox_write.append(self.depcachedir)
+ os.environ["SANDBOX_WRITE"] = \
+ ":".join(filter(None, sandbox_write))
self.eclassdb = eclass_cache.cache(self.porttree_root,
overlays=self.mysettings["PORTDIR_OVERLAY"].split())
self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
self.auxdb = {}
+ self._pregen_auxdb = {}
self._init_cache_dirs()
# XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
# ~harring
# location, label, auxdbkeys
self.auxdb[x] = self.auxdbmodule(
self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid)
+ if "metadata-transfer" not in self.mysettings.features:
+ for x in self.porttrees:
+ if os.path.isdir(os.path.join(x, "metadata", "cache")):
+ self._pregen_auxdb[x] = self.metadbmodule(
+ x, "metadata/cache", filtered_auxdbkeys, readonly=True)
# Selectively cache metadata in order to optimize dep matching.
self._aux_cache_keys = set(
["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE",
noiselevel=-1)
raise KeyError(mycpv)
- try:
- mydata = self.auxdb[mylocation][mycpv]
- eapi = mydata.get("EAPI","").strip()
- if not eapi:
- eapi = "0"
- if eapi.startswith("-") and eapi_is_supported(eapi[1:]):
- doregen = True
- elif emtime != long(mydata.get("_mtime_", 0)):
- doregen = True
- elif len(mydata.get("_eclasses_", [])) > 0:
- doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
- else:
- doregen = False
-
- except KeyError:
- doregen = True
- except CacheError:
- doregen = True
+ # Pull pre-generated metadata from the metadata/cache/
+ # directory if it exists and is valid, otherwise fall
+ # back to the normal writable cache.
+ auxdbs = []
+ pregen_auxdb = self._pregen_auxdb.get(mylocation)
+ if pregen_auxdb is not None:
+ auxdbs.append(pregen_auxdb)
+ auxdbs.append(self.auxdb[mylocation])
+
+ doregen = True
+ for auxdb in auxdbs:
try:
- del self.auxdb[mylocation][mycpv]
+ mydata = auxdb[mycpv]
+ eapi = mydata.get("EAPI","").strip()
+ if not eapi:
+ eapi = "0"
+ if eapi.startswith("-") and eapi_is_supported(eapi[1:]):
+ pass
+ elif emtime != long(mydata.get("_mtime_", 0)):
+ pass
+ elif len(mydata.get("_eclasses_", [])) > 0:
+ if self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]):
+ doregen = False
+ else:
+ doregen = False
except KeyError:
pass
+ except CacheError:
+ if auxdb is not pregen_auxdb:
+ try:
+ del auxdb[mycpv]
+ except KeyError:
+ pass
+ if not doregen:
+ break
writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2)
def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None):
if mysettings is None:
- mysettings = self.mysettings
+ mysettings = self.doebuild_settings
try:
eapi, myuris = self.aux_get(mypkg,
["EAPI", "SRC_URI"], mytree=mytree)
"getfetchlist(): '%s' has unsupported EAPI: '%s'" % \
(mypkg, eapi.lstrip("-")))
- if useflags is None:
- useflags = mysettings["USE"].split()
+ if not all and useflags is None:
+ mysettings.setcpv(mypkg, mydb=self)
+ useflags = mysettings["PORTAGE_USE"].split()
myurilist = paren_reduce(myuris)
myurilist = use_reduce(myurilist, uselist=useflags, matchall=all)
unmerge_orphans = "unmerge-orphans" in self.settings.features
if pkgfiles:
+ self.updateprotect()
mykeys = pkgfiles.keys()
mykeys.sort()
mykeys.reverse()
projects / portage.git / commitdiff