From c65baf5e3f22cb8183565924ad233ce814fc24c8 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 24 Apr 2008 17:58:14 +0000 Subject: [PATCH] Merged from trunk 9909:9941 | 9910 | Inside dblink.unmerge(), call self.updateprotect() just in | | zmedico | case anything has changed since the dblink instance was | | | constructed. | | 9912 | Handle potential InvalidDependString exceptions from | | zmedico | visible(). | | 9914 | Bug #217905 - Fix portdbapi.getfetchlist() to correctly | | zmedico | calculate USE when the useflags parameter is not passed in. | | 9916 | If the mysettings parameter is not passed in to | | zmedico | portdbapi.getfetchlist() then substitue | | | self.doebuild_settings since that one is mutable which is | | | necessary in case setcpv() needs to be called. | | 9918 | Fix portdbapi.getfetchlist() so that it doesn't | | zmedico | unnecessarily calculate USE when the "all" parameter is | | | True. | | 9920 | Remove redundant version sorting in _expand_new_virtuals() | | zmedico | since the dbapi match results are already sorted. | | 9922 | Make the ccache userpriv permission check from bug #162542 | | zmedico | less sensitive by skipping permission checks on files and | | | just checking the directories. | | 9924 | Bug #218202 - Make sure the spinner is quiet in --nodep | | zmedico | mode. | | 9926 | Add a new part for "Dependency Resolution". | | zmedico | | | 9928 | Bug #218559 - Make portdbapi automatically add sandbox write | | zmedico | access to self.depcachedir when appropriate. This makes the | | | portageq blanket sandbox exemption unnecessary. | | 9930 | * Make portdbapi.aux_get() automatically pull pre-generated | | zmedico | metadata directly from the metadata/cache/ directory when | | | metadata-transfer is not enabled in FEATURES. This makes all | | | cache modules behave similar to existing metadata_overlay | | | module, except when FEATURES="metadata-transfer" has been | | | explicitly enabled. * Remove metadata-transfer from FEATURES | | | in make.globals so that it's no longer enabled by default. | | | Users will be happy about this since the "Updating Portage | | | cache" part of emerge --sync can be more time consuming than | | | the rsync run itself. It's also nicer when when using | | | PORTDIR over nfs or bind mounted in a chroot since there's | | | no need to run emerge --metadata. * Update | | | FEATURES="metadata-transfer" docs to warn users of the rsync | | | tree to modify eclases in PORTDIR_OVERLAY if necessary | | | instead of doing it directly in the rsync tree. * Relevant | | | updates to /etc/portage/modules docs. | | 9932 | Fix portage version, should be >=2.1.5. | | zmedico | | | 9934 | Document FEATURES="metadata-transfer" changes some more. | | zmedico | | | 9936 | In action_metadata(), properly put the source cache in | | zmedico | readonly mode. | | 9938 | In show_slot_collision_notice(), refer to the problem as a | | zmedico | "slot conflict" since that's how I'm refering to it in the | | | docs. | | 9940 | Bug #218439 - Improve alignment in short emerge --help | | zmedico | options output. | | 9941 | Swap --nospinner and --oneshot for alphabetical ordering. | | zmedico | | svn path=/main/branches/prefix/; revision=9961 --- RELEASE-NOTES | 6 ++ bin/portageq | 4 - cnf/make.globals | 2 +- doc/dependency_resolution.docbook | 6 ++ .../decision_making.docbook | 65 +++++++++++++ .../package_modeling.docbook | 93 +++++++++++++++++++ .../task_scheduling.docbook | 36 +++++++ doc/portage.docbook | 5 + man/emerge.1 | 2 + man/make.conf.5 | 10 ++ man/portage.5 | 23 +++-- pym/_emerge/__init__.py | 24 +++-- pym/_emerge/help.py | 11 ++- pym/portage/__init__.py | 25 ++--- pym/portage/dbapi/porttree.py | 75 ++++++++++----- pym/portage/dbapi/vartree.py | 1 + 16 files changed, 318 insertions(+), 70 deletions(-) create mode 100644 doc/dependency_resolution.docbook create mode 100644 doc/dependency_resolution/decision_making.docbook create mode 100644 doc/dependency_resolution/package_modeling.docbook create mode 100644 doc/dependency_resolution/task_scheduling.docbook diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 4b57835f1..ead0d7f0f 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -35,6 +35,12 @@ portage-2.2 portage-2.1.5 ================================== +* The metadata-transfer feature is now disabled by default. This disables the + "Updating Portage cache" routine that used to run at the tail end of each + `emerge --sync` operation. If you use something like the sqlite module and + want to keep all metadata in that format alone (useful for querying), enable + FEATURES="metadata-transfer" in make.conf. + * The parallel-fetch feature is now enabled by default. It is optimized to avoid doing redundant checksums for previously downloaded files that have the correct size. Run `tail -f /var/log/emerge-fetch.log` in a diff --git a/bin/portageq b/bin/portageq index 920d04cad..5ea82dbe2 100755 --- a/bin/portageq +++ b/bin/portageq @@ -21,10 +21,6 @@ except KeyboardInterrupt: import os -# This allows portageq to be exempt from sandbox, -# for things like updating metadata cache. -os.environ["SANDBOX_ON"] = "0" - import types #----------------------------------------------------------------------------- diff --git a/cnf/make.globals b/cnf/make.globals index 0851d53a0..6da603f55 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -38,7 +38,7 @@ FETCHCOMMAND="wget -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI RESUMECOMMAND="wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\"" # Default user options -FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch" +FEATURES="distlocks parallel-fetch sandbox sfperms strict unmerge-orphans userfetch" # Enable for global testing FEATURES="${FEATURES} preserve-libs" diff --git a/doc/dependency_resolution.docbook b/doc/dependency_resolution.docbook new file mode 100644 index 000000000..7dd18e84b --- /dev/null +++ b/doc/dependency_resolution.docbook @@ -0,0 +1,6 @@ + +Dependency Resolution +&dependency_resolution_package_modeling; +&dependency_resolution_decision_making; +&dependency_resolution_task_scheduling; + diff --git a/doc/dependency_resolution/decision_making.docbook b/doc/dependency_resolution/decision_making.docbook new file mode 100644 index 000000000..5bcf84388 --- /dev/null +++ b/doc/dependency_resolution/decision_making.docbook @@ -0,0 +1,65 @@ + +Decision Making + + Dependency Expression Evaluation + + In terms of boolean logic, a dependency expression can + be expressed in disjunctive normal form (DNF), which is + a disjunction of conjunctive clauses. Each conjunctive clause + represents one possible alternative combination of dependency + atoms capable of satisfying the dependency expression. + + + + Look-Ahead + + When there are multiple combinations to choose from, + a look-ahead mechanism will choose an optimal combination + to satisfy constraints and minimize cost. The + following package states influence the cost calculation for + a given combination: + + + installed + + + selected (for installation) + + + not selected (for installation) + + + + + In cost calculations, virtual packages by themselves are + considered to cost nothing since they do not directly install anything. + It is the dependencies of a virtual package that contribute to it's cost. + + + Constraint Propagation + + Combinations that include packages from the "installed" or + "selected" categories are less costly than those that + include packages from the "not selected" category. + When a package is chosen for installation, it transitions to the + "selected" state. This state change propagates + to the cost calculations of later decisions, + influencing later decisions to be consistent with earlier decisions. + This feedback mechanism serves to propagate constraints and can + influence the modeling process to + converge on a more optimal final state. + + + + Expanded Search Space + + When evaluating virtual atoms, an expanded search space is + considered which recursively traverses + the dependencies of virtual packages + from all slots matching a given virtual atom. All combinations in + this expanded search space are considered when choosing an optimal + combination to satisfy constraints with minimal cost. + + + + diff --git a/doc/dependency_resolution/package_modeling.docbook b/doc/dependency_resolution/package_modeling.docbook new file mode 100644 index 000000000..84d01a9f2 --- /dev/null +++ b/doc/dependency_resolution/package_modeling.docbook @@ -0,0 +1,93 @@ + +Package Modeling + + + Constraint Satisfaction + + Constraint Types + + Dependency resolution involves satisfaction of + many constraints: + + + Persistent configuration parameters, like those that come from + make.profile, make.conf, and the /etc/portage directory. + + + Current command parameters, which may include options, atoms, or sets. + + + + Package Dependencies + + + + + + + Package Dependencies + + Common types of package dependencies: + + + Files required for building or installing. Downloads may + be necessary to satisfy these. + + + Other packages required to be installed for + buildtime or runtime. + + + Blockers that prevent conflicting packages from being installed + simultaneously. + + + + + + + + Conflicts + + Blocker Conflicts + + If one package blocks another package, the two packages + conflict such that they cannot be installed simultaneously. + These conflicts are often due to file collisions. + + + + Slot Conflicts + + If two different packages that occupy the same slot are chosen + to satisfy dependencies, a slot conflict occurs. The two packages + cannot be installed simultaneously and therefore the respective + dependencies will not be satisfied simultaneously. + + + + Indirect Conflicts + + If the dependencies of two parent packages cannot be installed + simultaneously, it creates an indirect conflict between the parent + packages since their respective dependencies cannot be satisfied + simultaneously. + + + + + + Dependency Neglection + + In order to significantly reduce the resources consumed by + the modeling process, the dependencies of + installed packages may be neglected. + + + If a more complete dependency calculation is desired, + there is a --complete-graph option which will ensure that the + dependencies of installed packages are properly considered. + + + + diff --git a/doc/dependency_resolution/task_scheduling.docbook b/doc/dependency_resolution/task_scheduling.docbook new file mode 100644 index 000000000..8979a9f62 --- /dev/null +++ b/doc/dependency_resolution/task_scheduling.docbook @@ -0,0 +1,36 @@ + +Task Scheduling + + Task Dependencies + + All tasks are executed in an order such + that a task's dependencies are satisfied + when it is executed. Dependency relationships between tasks + form a directed graph. + + + + Conflict Avoidance + + In some cases it is possible to adjust package installation order + to avoid having two conflicting packages installed simultaneously. + + + TODO: Automatically uninstall packages when necessary to avoid conflicts. + + + + Circular Dependencies + + TODO: Automatically solve circular dependencies by temporarily disabling + conditional dependencies and then rebuilding packages with the conditional + dependencies enabled. + + + + Parallel Scheduling + + TODO: Spawn an appropriate number of tasks in parallel when desired. + + + diff --git a/doc/portage.docbook b/doc/portage.docbook index 5417bb73e..d158c1a21 100644 --- a/doc/portage.docbook +++ b/doc/portage.docbook @@ -7,6 +7,10 @@ + + + + @@ -34,6 +38,7 @@ &config; +&dependency_resolution; &package; &qa; diff --git a/man/emerge.1 b/man/emerge.1 index 90786156c..04efc73a9 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -133,6 +133,8 @@ tail end of an rsync update using \fBemerge \-\-sync\fR. This process populates the cache database that portage uses for pre-parsed lookups of package data. It does not populate cache for the overlays listed in PORTDIR_OVERLAY. In order to generate cache for overlays, use \fB\-\-regen\fR. +In versions of portage >=2.1.5 the \-\-metadata action is totally unnecessary +unless the user has enabled FEATURES="metadata-transfer" in \fBmake.conf\fR(5). .TP .BR "\-\-prune " (\fB\-P\fR) \fBWARNING: This action can remove important packages!\fR Removes all but the diff --git a/man/make.conf.5 b/man/make.conf.5 index 4976ebe4d..839fd6de7 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -215,6 +215,16 @@ when \fImirror\fR is also in the \fBebuild\fR(5) \fBRESTRICT\fR variable. .TP .B metadata\-transfer Automatically perform a metadata transfer when `emerge \-\-sync` is run. +In versions of portage >=2.1.5, this feature is disabled by +default. When metadata\-transfer is disabled, metadata cache from the +${PORTDIR}/metadata/cache/ directory will be used directly (if available) +and eclasses in ${PORTDIR}/eclass/ must not be modified except by +`emerge \-\-sync` operations since the cache validation mechanism +will not recognize eclass modifications. Normally, this issue only +pertains to users of the rsync tree since the cvs tree does not contain +a metadata/cache/ directory. Users of the rsync tree who want to modify +eclasses should use \fBPORTDIR_OVERLAY\fR in order for the cache +validation mechanism to work correctly. .TP .B mirror Fetch everything in \fBSRC_URI\fR regardless of \fBUSE\fR settings, diff --git a/man/portage.5 b/man/portage.5 index 0599c5e09..9164ada88 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -384,18 +384,17 @@ override. .I Example: .nf -portdbapi.auxdbmodule = portage.cache.metadata_overlay.database -.fi - -The metadata_overlay cache module makes it possible to disable -FEATURES="metadata-transfer" in \fBmake.conf\fR(5). When the user initially -enables metadata_overlay in /etc/portage/modules, all of the cache files -contained in /var/cache/edb/dep/${PORTDIR} must be manually removed in order -to avoid unecessary cache regeneration. In addition, users of the -metadata_overlay module must never modify eclasses in ${PORTDIR} because -portage will not be able to detect that cache regeneration is necessary. If -the user would like to modify eclasses, it is safe to use metadata_overlay -together with PORTDIR_OVERLAY in \fBmake.conf\fR. +portdbapi.auxdbmodule = portage.cache.sqlite.database +.fi + +After changing the portdbapi.auxdbmodule setting, it may be necessary to +transfer or regenerate metadata cache. Users of the rsync tree need to +run `emerge \-\-metadata` if they have enabled FEATURES="metadata-transfer" +in \fBmake.conf\fR(5). In order to regenerate metadata for repositories +listed in \fBPORTDIR_OVERLAY\fR or a cvs tree, run `emerge \-\-regen` +(see \fBemerge\fR(1)). If you use something like the sqlite module and want +to keep all metadata in that format alone (useful for querying), enable +FEATURES="metadata-transfer" in \fBmake.conf\fR(5). .TP .BR package.keywords Per\-package KEYWORDS. Useful for mixing unstable packages in with a normally diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index c8c8d0168..4e231ac7b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1566,6 +1566,8 @@ class depgraph(object): self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 + if "--nodeps" in myopts: + spinner.update = spinner.update_quiet self.spinner = spinner self.pkgsettings = {} # Maps slot atom to package for each Package added to the graph. @@ -1713,8 +1715,9 @@ class depgraph(object): msg = [] msg.append("\n!!! Multiple versions within a single " + \ - "package slot have been \n") - msg.append("!!! pulled into the dependency graph:\n\n") + "package slot have been pulled\n") + msg.append("!!! into the dependency graph, resulting" + \ + " in a slot conflict:\n\n") indent = " " # Max number of parents shown, to avoid flooding the display. max_parents = 3 @@ -2753,7 +2756,10 @@ class depgraph(object): # here, packages that have been masked since they # were installed can be automatically downgraded # to an unmasked version. - if not visible(pkgsettings, pkg): + try: + if not visible(pkgsettings, pkg): + continue + except portage.exception.InvalidDependString: continue if not built and not calculated_use: # This is avoided whenever possible because @@ -4465,9 +4471,13 @@ class depgraph(object): arg = None if arg: return False - if pkg.installed and \ - not visible(self._depgraph.pkgsettings[pkg.root], pkg): - return False + if pkg.installed: + try: + if not visible( + self._depgraph.pkgsettings[pkg.root], pkg): + return False + except portage.exception.InvalidDependString: + pass return True def _dep_expand(self, atom): @@ -6339,7 +6349,7 @@ def action_metadata(settings, portdb, myopts): ec = portage.eclass_cache.cache(portdb.porttree_root) myportdir = os.path.realpath(settings["PORTDIR"]) cm = settings.load_best_module("portdbapi.metadbmodule")( - myportdir, "metadata/cache", portage.auxdbkeys[:]) + myportdir, "metadata/cache", portage.auxdbkeys[:], readonly=True) from portage.cache import util diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index b6bc541cd..1e3f54aff 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -14,11 +14,12 @@ def shorthelp(): print " "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >" print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]" print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] " - print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"] ["+green("--oneshot")+"] ["+green("--newuse")+"] ["+green("--noconfmem")+"]" - print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]" - print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] ["+green("--nospinner")+"]" - print " [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ] [" + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]" - print bold("Actions:")+" [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]" + print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]" + print " [ "+green("--columns")+" ] [ "+green("--deep")+" ] [ "+green("--newuse")+" ]" + print " [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+" ]" + print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--complete-graph")+" ]" + print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]" + print bold("Actions:")+" [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]" def help(myaction,myopts,havecolor=1): # TODO: Implement a wrap() that accounts for console color escape codes. diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 7cc2e9963..264a66183 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4314,9 +4314,7 @@ def prepare_build_dirs(myroot, mysettings, cleanup): continue if subdir_st.st_gid != portage_gid or \ ((stat.S_ISDIR(subdir_st.st_mode) and \ - not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode)) or \ - (not stat.S_ISDIR(subdir_st.st_mode) and \ - not filemode == (stat.S_IMODE(subdir_st.st_mode) & filemode))): + not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode))): droppriv_fix = True break if droppriv_fix: @@ -5360,8 +5358,6 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", # According to GLEP 37, RDEPEND is the only dependency type that is valid # for new-style virtuals. Repoman should enforce this. dep_keys = ["RDEPEND", "DEPEND", "PDEPEND"] - def compare_pkgs(a, b): - return pkgcmp(b[1], a[1]) portdb = trees[myroot]["porttree"].dbapi if kwargs["use_binaries"]: portdb = trees[myroot]["bintree"].dbapi @@ -5387,19 +5383,14 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", match_atom = x if isblocker: match_atom = x[1:] - pkgs = {} - for cpv in portdb.match(match_atom): + pkgs = [] + matches = portdb.match(match_atom) + # Use descending order to prefer higher versions. + matches.reverse() + for cpv in matches: # only use new-style matches if cpv.startswith("virtual/"): - pkgs[cpv] = (cpv, catpkgsplit(cpv)[1:], portdb) - if kwargs["use_binaries"] and "vartree" in trees[myroot]: - vardb = trees[myroot]["vartree"].dbapi - for cpv in vardb.match(match_atom): - # only use new-style matches - if cpv.startswith("virtual/"): - if cpv in pkgs: - continue - pkgs[cpv] = (cpv, catpkgsplit(cpv)[1:], vardb) + pkgs.append((cpv, catpkgsplit(cpv)[1:], portdb)) if not (pkgs or mychoices): # This one couldn't be expanded as a new-style virtual. Old-style # virtuals have already been expanded by dep_virtual, so this one @@ -5411,8 +5402,6 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", if not pkgs and len(mychoices) == 1: newsplit.append(x.replace(mykey, mychoices[0])) continue - pkgs = pkgs.values() - pkgs.sort(compare_pkgs) # Prefer higher versions. if isblocker: a = [] else: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 532530131..fdf0f1168 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -62,7 +62,16 @@ class portdbapi(dbapi): #self.root=settings["PORTDIR"] self.porttree_root = os.path.realpath(porttree_root) - self.depcachedir = self.mysettings.depcachedir[:] + self.depcachedir = os.path.realpath(self.mysettings.depcachedir) + + if os.environ.get("SANDBOX_ON") == "1": + # Make api consumers exempt from sandbox violations + # when doing metadata cache updates. + sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":") + if self.depcachedir not in sandbox_write: + sandbox_write.append(self.depcachedir) + os.environ["SANDBOX_WRITE"] = \ + ":".join(filter(None, sandbox_write)) self.eclassdb = eclass_cache.cache(self.porttree_root, overlays=self.mysettings["PORTDIR_OVERLAY"].split()) @@ -98,6 +107,7 @@ class portdbapi(dbapi): self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") self.auxdb = {} + self._pregen_auxdb = {} self._init_cache_dirs() # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys # ~harring @@ -116,6 +126,11 @@ class portdbapi(dbapi): # location, label, auxdbkeys self.auxdb[x] = self.auxdbmodule( self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid) + if "metadata-transfer" not in self.mysettings.features: + for x in self.porttrees: + if os.path.isdir(os.path.join(x, "metadata", "cache")): + self._pregen_auxdb[x] = self.metadbmodule( + x, "metadata/cache", filtered_auxdbkeys, readonly=True) # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( ["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", @@ -274,28 +289,41 @@ class portdbapi(dbapi): noiselevel=-1) raise KeyError(mycpv) - try: - mydata = self.auxdb[mylocation][mycpv] - eapi = mydata.get("EAPI","").strip() - if not eapi: - eapi = "0" - if eapi.startswith("-") and eapi_is_supported(eapi[1:]): - doregen = True - elif emtime != long(mydata.get("_mtime_", 0)): - doregen = True - elif len(mydata.get("_eclasses_", [])) > 0: - doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]) - else: - doregen = False - - except KeyError: - doregen = True - except CacheError: - doregen = True + # Pull pre-generated metadata from the metadata/cache/ + # directory if it exists and is valid, otherwise fall + # back to the normal writable cache. + auxdbs = [] + pregen_auxdb = self._pregen_auxdb.get(mylocation) + if pregen_auxdb is not None: + auxdbs.append(pregen_auxdb) + auxdbs.append(self.auxdb[mylocation]) + + doregen = True + for auxdb in auxdbs: try: - del self.auxdb[mylocation][mycpv] + mydata = auxdb[mycpv] + eapi = mydata.get("EAPI","").strip() + if not eapi: + eapi = "0" + if eapi.startswith("-") and eapi_is_supported(eapi[1:]): + pass + elif emtime != long(mydata.get("_mtime_", 0)): + pass + elif len(mydata.get("_eclasses_", [])) > 0: + if self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]): + doregen = False + else: + doregen = False except KeyError: pass + except CacheError: + if auxdb is not pregen_auxdb: + try: + del auxdb[mycpv] + except KeyError: + pass + if not doregen: + break writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2) @@ -362,7 +390,7 @@ class portdbapi(dbapi): def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None): if mysettings is None: - mysettings = self.mysettings + mysettings = self.doebuild_settings try: eapi, myuris = self.aux_get(mypkg, ["EAPI", "SRC_URI"], mytree=mytree) @@ -379,8 +407,9 @@ class portdbapi(dbapi): "getfetchlist(): '%s' has unsupported EAPI: '%s'" % \ (mypkg, eapi.lstrip("-"))) - if useflags is None: - useflags = mysettings["USE"].split() + if not all and useflags is None: + mysettings.setcpv(mypkg, mydb=self) + useflags = mysettings["PORTAGE_USE"].split() myurilist = paren_reduce(myuris) myurilist = use_reduce(myurilist, uselist=useflags, matchall=all) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index ec88a921a..090546b2e 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1348,6 +1348,7 @@ class dblink(object): unmerge_orphans = "unmerge-orphans" in self.settings.features if pkgfiles: + self.updateprotect() mykeys = pkgfiles.keys() mykeys.sort() mykeys.reverse() -- 2.26.2