From: Zac Medico Date: Thu, 30 Apr 2009 06:58:28 +0000 (-0000) Subject: Add portdbapi support for a metadata/layout.conf file which X-Git-Tag: v2.1.6.12~111 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=805ceb16fdc2e9feb24b398d87064dbf94a19fd6;p=portage.git Add portdbapi support for a metadata/layout.conf file which specifies information about the repository layout. Currently, only a single "masters" attribute is supported, which is used to specify names of repositories which satisfy dependencies on eclasses and/or ebuilds. Each repository name should correspond the value of a repo_name entry from one of the repositories that is configured via the PORTDIR or PORTDIR_OVERLAY variables. Since layout.conf is now used to control eclass inheritance, it is now safer to use overlays which contain forked eclasses have names identical to those from the main tree. Such eclasses will only apply to their containing repository and any other repositories which reference their containing repository via layout.conf. This solves bug #124041 by containing eclass overrides so that they don't necessarily apply to all ebuilds. Thanks to Alistair Bush for his initial patch for layout.conf support in repoman (will be merged later). See the "QA Overlay Layout support" thread on the gentoo-dev mailing list for more information: http://archives.gentoo.org/gentoo-dev/msg_33c61550b4ed2b7b25dd5a4110e1ec81.xml (trunk r13291) svn path=/main/branches/2.1.6/; revision=13462 --- diff --git a/man/portage.5 b/man/portage.5 index 765aa08d1..22affb401 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -59,6 +59,11 @@ categories .BR /etc/portage/profile/ site-specific overrides of \fB/etc/make.profile/\fR .TP +.BR /usr/portage/metadata/ +.nf +layout.conf +.fi +.TP .BR /usr/portage/profiles/ .nf arch.list @@ -542,6 +547,23 @@ media\-other .fi .RE .TP +.BR /usr/portage/metadata/ +.RS +.TP +.BR layout.conf +Specifies information about the repository layout. Currently, only a single +"masters" attribute is supported, which is used to specify names of +repositories which satisfy dependencies on eclasses and/or ebuilds. Each +repository name should correspond the value of a \fBrepo_name\fR entry +from one of the repositories that is configured via the \fBPORTDIR\fR or +\fBPORTDIR_OVERLAY\fR variables (see \fBmake.conf\fR(5)). + +.I Example: +.nf +masters = gentoo java-overlay +.fi +.RE +.TP .BR /usr/portage/profiles/ Global Gentoo settings that are controlled by the developers. To override these settings, you can use the files in \fB/etc/portage/\fR. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 187a117b0..e8f75c2a9 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -14977,41 +14977,12 @@ def emerge_main(): #repo_name_check(trees) config_protect_check(trees) - eclasses_overridden = {} for mytrees in trees.itervalues(): mydb = mytrees["porttree"].dbapi # Freeze the portdbapi for performance (memoize all xmatch results). mydb.freeze() - eclasses_overridden.update(mydb.eclassdb._master_eclasses_overridden) del mytrees, mydb - if eclasses_overridden and \ - settings.get("PORTAGE_ECLASS_WARNING_ENABLE") != "0": - prefix = bad(" * ") - if len(eclasses_overridden) == 1: - writemsg(prefix + "Overlay eclass overrides " + \ - "eclass from PORTDIR:\n", noiselevel=-1) - else: - writemsg(prefix + "Overlay eclasses override " + \ - "eclasses from PORTDIR:\n", noiselevel=-1) - writemsg(prefix + "\n", noiselevel=-1) - for eclass_name in sorted(eclasses_overridden): - writemsg(prefix + " '%s/%s.eclass'\n" % \ - (eclasses_overridden[eclass_name], eclass_name), - noiselevel=-1) - writemsg(prefix + "\n", noiselevel=-1) - msg = "It is best to avoid overriding eclasses from PORTDIR " + \ - "because it will trigger invalidation of cached ebuild metadata " + \ - "that is distributed with the portage tree. If you must " + \ - "override eclasses from PORTDIR then you are advised to add " + \ - "FEATURES=\"metadata-transfer\" to /etc/make.conf and to run " + \ - "`emerge --regen` after each time that you run `emerge --sync`. " + \ - "Set PORTAGE_ECLASS_WARNING_ENABLE=\"0\" in /etc/make.conf if " + \ - "you would like to disable this warning." - from textwrap import wrap - for line in wrap(msg, 72): - writemsg("%s%s\n" % (prefix, line), noiselevel=-1) - if "moo" in myfiles: print """ diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 72a08a229..27dd62787 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1105,7 +1105,7 @@ class config(object): "PORTAGE_BACKGROUND", "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER", "PORTAGE_COUNTER_HASH", - "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES", + "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE", @@ -5150,6 +5150,12 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m mysettings["FILESDIR"] = pkg_dir+"/files" mysettings["PF"] = mypv + if hasattr(mydbapi, '_repo_info'): + mytree = os.path.dirname(os.path.dirname(pkg_dir)) + repo_info = mydbapi._repo_info[mytree] + mysettings['PORTDIR'] = repo_info.portdir + mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay + mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"]) diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index a8be01095..e3f6fc571 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -29,7 +29,7 @@ class database(flat_hash.database): loc = location super(database, self).__init__(location, *args, **config) self.location = os.path.join(loc, "metadata","cache") - self.ec = portage.eclass_cache.cache(loc) + self.ec = None self.raise_stat_collision = False def _parse_data(self, data, cpv): @@ -53,9 +53,11 @@ class database(flat_hash.database): if "_eclasses_" not in d: if "INHERITED" in d: + if self.ec is None: + self.ec = portage.eclass_cache.cache(self.location) try: d["_eclasses_"] = self.ec.get_eclass_data( - d["INHERITED"].split(), from_master_only=True) + d["INHERITED"].split()) except KeyError, e: # INHERITED contains a non-existent eclass. raise cache_errors.CacheCorruption(cpv, e) diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py index 0f2685c0a..bdee52ddc 100644 --- a/pym/portage/cache/util.py +++ b/pym/portage/cache/util.py @@ -102,8 +102,7 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, # Even if _eclasses_ already exists, replace it with data from # eclass_cache, in order to insert local eclass paths. try: - eclasses = eclass_cache.get_eclass_data(inherited, - from_master_only=True) + eclasses = eclass_cache.get_eclass_data(inherited) except KeyError: # INHERITED contains a non-existent eclass. noise.eclass_stale(x) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 9d4b1523d..1a896b754 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -8,7 +8,8 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.dep:dep_getkey,match_from_list,paren_reduce,use_reduce', - 'portage.util:ensure_dirs,writemsg', + 'portage.env.loaders:KeyValuePairFileLoader', + 'portage.util:ensure_dirs,writemsg,writemsg_level', 'portage.versions:best,catpkgsplit,pkgsplit,ver_regexp', ) @@ -25,7 +26,7 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \ _eapi_is_deprecated -import codecs, os, stat +import codecs, logging, os, stat from itertools import izip def _src_uri_validate(cpv, eapi, src_uri): @@ -95,6 +96,15 @@ def _src_uri_validate(cpv, eapi, src_uri): "getFetchMap(): '%s' SRC_URI arrow missing right operand" % \ (cpv,)) +class _repo_info(object): + __slots__ = ('name', 'path', 'eclass_db', 'portdir', 'portdir_overlay') + def __init__(self, name, path, eclass_db): + self.name = name + self.path = path + self.eclass_db = eclass_db + self.portdir = eclass_db.porttrees[0] + self.portdir_overlay = ' '.join(eclass_db.porttrees[1:]) + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] @@ -116,10 +126,8 @@ class portdbapi(dbapi): # instance that is passed in. self.doebuild_settings = config(clone=self.mysettings) - #self.root=settings["PORTDIR"] + porttree_root = os.path.realpath(porttree_root) self.porttree_root = porttree_root - if porttree_root: - self.porttree_root = os.path.realpath(porttree_root) self.depcachedir = os.path.realpath(self.mysettings.depcachedir) @@ -132,8 +140,7 @@ class portdbapi(dbapi): os.environ["SANDBOX_WRITE"] = \ ":".join(filter(None, sandbox_write)) - self.eclassdb = eclass_cache.cache(self.porttree_root, - overlays=self.mysettings["PORTDIR_OVERLAY"].split()) + self.eclassdb = eclass_cache.cache(porttree_root) # This is used as sanity check for aux_get(). If there is no # root eclass dir, we assume that PORTDIR is invalid or @@ -163,7 +170,43 @@ class portdbapi(dbapi): # don't want to see a warning every time the portage module is # imported. pass - + + self._repo_info = {} + eclass_dbs = {porttree_root : self.eclassdb} + for path in self.porttrees: + if path in self._repo_info: + continue + + layout_filename = os.path.join(path, "metadata/layout.conf") + layout_file = KeyValuePairFileLoader(layout_filename, None, None) + layout_data, layout_errors = layout_file.load() + porttrees = [] + for master_name in layout_data.get('masters', '').split(): + master_path = self.treemap.get(master_name) + if master_path is None: + writemsg_level(("Unavailable repository '%s' " + \ + "referenced by masters entry in '%s'\n") % \ + (master_name, layout_filename), + level=logging.ERROR, noiselevel=-1) + else: + porttrees.append(master_path) + if not porttrees: + porttrees.append(porttree_root) + porttrees.append(path) + + eclass_db = None + for porttree in porttrees: + tree_db = eclass_dbs.get(porttree) + if tree_db is None: + tree_db = eclass_cache.cache(porttree) + if eclass_db is None: + eclass_db = tree_db.copy() + else: + eclass_db.append(tree_db) + + self._repo_info[path] = _repo_info(self._repository_map.get(path), + path, eclass_db) + self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") self.auxdb = {} self._pregen_auxdb = {} @@ -191,6 +234,10 @@ class portdbapi(dbapi): if os.path.isdir(os.path.join(x, "metadata", "cache")): self._pregen_auxdb[x] = self.metadbmodule( x, "metadata/cache", filtered_auxdbkeys, readonly=True) + try: + self._pregen_auxdb[x].ec = self._repo_info[x].eclass_db + except AttributeError: + pass # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( ["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", diff --git a/pym/portage/eclass_cache.py b/pym/portage/eclass_cache.py index 670e9fb24..15602fc02 100644 --- a/pym/portage/eclass_cache.py +++ b/pym/portage/eclass_cache.py @@ -5,6 +5,7 @@ __all__ = ["cache"] +import warnings from portage.util import normalize_path, writemsg import errno, os, sys from portage.data import portage_gid @@ -15,18 +16,47 @@ class cache(object): Maintains the cache information about eclasses used in ebuild. """ def __init__(self, porttree_root, overlays=[]): - self.porttree_root = porttree_root self.eclasses = {} # {"Name": ("location","_mtime_")} self._eclass_locations = {} # screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you. # ~harring - self.porttrees = [self.porttree_root]+overlays - self.porttrees = tuple(map(normalize_path, self.porttrees)) - self._master_eclass_root = os.path.join(self.porttrees[0],"eclass") - self._master_eclasses_overridden = {} - self.update_eclasses() + if porttree_root: + self.porttree_root = porttree_root + self.porttrees = [self.porttree_root] + overlays + self.porttrees = tuple(map(normalize_path, self.porttrees)) + self._master_eclass_root = os.path.join(self.porttrees[0], "eclass") + self.update_eclasses() + else: + self.porttree_root = None + self.porttrees = () + self._master_eclass_root = None + + def copy(self): + return self.__copy__() + + def __copy__(self): + result = self.__class__(None) + result.eclasses = self.eclasses.copy() + result._eclass_locations = self._eclass_locations.copy() + result.porttree_root = self.porttree_root + result.porttrees = self.porttrees + result._master_eclass_root = self._master_eclass_root + return result + + def append(self, other): + """ + Append another instance to this instance. This will cause eclasses + from the other instance to override and eclases from this instance + that have the same name. + """ + if not isinstance(other, self.__class__): + raise TypeError( + "expected type %s, got %s" % (self.__class__, type(other))) + self.porttrees = self.porttrees + other.porttrees + self.eclasses.update(other.eclasses) + self._eclass_locations.update(other._eclass_locations) def close_caches(self): import traceback @@ -77,8 +107,7 @@ class cache(object): # It appears to be identical to the master, # so prefer the master entry. continue - else: - self._master_eclasses_overridden[ys] = x + self.eclasses[ys] = (x, mtime) self._eclass_locations[ys] = x @@ -100,8 +129,10 @@ class cache(object): ec_dict = {} for x in inherits: ec_dict[x] = self.eclasses[x] - if from_master_only and \ - self._eclass_locations[x] != self._master_eclass_root: - return None + + if from_master_only is not False: + warnings.warn("portage.eclass_cache.cache.get_eclass_data(): " + \ + "ignoring deprecated 'from_master_only' parameter", + DeprecationWarning) return ec_dict