From: Fabian Groffen Date: Sun, 12 Apr 2009 17:47:15 +0000 (-0000) Subject: Merged from trunk -r13322:13331 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=6e9b796e1e596ef535b74de37f206539d6f02fc5;p=portage.git Merged from trunk -r13322:13331 | 13323 | Set local_config=False in config constructor calls, since | | zmedico | egencache is similar to repoman in the way that it's used to | | | distribute something to other users and thus site-specific | | | local configuration is irrelevant. | | 13324 | Move the /etc/portage categories and mirrors docs into | | zmedico | alphabetical order. | | 13325 | Bug #265747 - Add a new /etc/portage/repos.conf config file | | zmedico | which can be used to configure site-specific eclass override | | | behavior. Note that configuration settings which are | | | specified here do not apply to tools such as repoman(1) and | | | egencache(1) since their operations are inherently not | | | site-specific. Beware that use of eclass-overrides is | | | generally not recommended and that it may trigger | | | performance issues under some circumstances (see bug | | | #124041). Example: # make all repositories inherit eclasses | | | from the java-overlay and # java-experimental repositories, | | | with eclasses from java-experimental # taking precedence | | | over those from java-overlay [DEFAULT] eclass-overrides = | | | java-overlay java-experimental # disable all eclass | | | overrides for the gentoo repository [gentoo] | | | eclass-overrides = | | 13326 | Pass the correct PORTDIR_OVERLAY value into the | | zmedico | profile-specific config constructor calls. | | 13327 | Inside the portdbapi constructor, handle repositories with | | zmedico | the same repo_name by discarding the ones that were | | | encountered earlier. | | 13328 | Bug #265768 - When initializing the eclass cache, use the | | zmedico | correct location. | | 13329 | Bug #265768 - If the user happens to be using the | | zmedico | metadata_overlay module then initialize it's eclass cache | | | with an appropriate instance inside the portdbapi | | | constructor. | | 13330 | Add a note about egencache to the --regen docs. | | zmedico | | | 13331 | Bug #265768 - Handle CacheError when deleting currupt cache | | zmedico | inside portdbapi._pull_valid_cache(). | svn path=/main/branches/prefix/; revision=13337 --- diff --git a/bin/egencache b/bin/egencache index 4103651d5..a9dc767b4 100755 --- a/bin/egencache +++ b/bin/egencache @@ -277,7 +277,7 @@ def egencache_main(args): env['PORTAGE_DEPCACHEDIR'] = options.cache_dir settings = portage.config(config_root=config_root, - target_root='/', env=env) + target_root='/', local_config=False, env=env) default_opts = None if not options.ignore_default_opts: @@ -293,7 +293,7 @@ def egencache_main(args): env['PORTAGE_DEPCACHEDIR'] = options.cache_dir settings = portage.config(config_root=config_root, - target_root='/', env=env) + target_root='/', local_config=False, env=env) if not options.update: parser.error('No action specified (--update ' + \ diff --git a/bin/repoman b/bin/repoman index b943cd27b..5bc12b788 100755 --- a/bin/repoman +++ b/bin/repoman @@ -498,9 +498,14 @@ repo_info = portdb._repo_info[portdir_overlay] portdb.porttrees = list(repo_info.eclass_db.porttrees) portdir = portdb.porttrees[0] +# Generate an appropriate PORTDIR_OVERLAY value for passing into the +# profile-specific config constructor calls. +env = os.environ.copy() +env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:]) + logging.info('Setting paths:') logging.info('PORTDIR = "' + portdir + '"') -logging.info('PORTDIR_OVERLAY = "%s"' % ' '.join(portdb.porttrees[1:])) +logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY']) portdb.mysettings = repoman_settings root_config = RootConfig(repoman_settings, trees[root], None) @@ -1529,7 +1534,8 @@ for x in scanlist: dep_settings = portage.config( config_profile_path=profdir, config_incrementals=portage.const.INCREMENTALS, - local_config=False) + local_config=False, + env=env) if options.without_mask: dep_settings.pmaskdict.clear() arch_caches[prof[0]] = dep_settings diff --git a/man/emerge.1 b/man/emerge.1 index 3e6063e6e..9dc53d269 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -172,7 +172,8 @@ user' :). Rsync users should simply run \fBemerge \-\-sync\fR to regenerate the cache. After a portage update, rsync users may find it convenient to run \fBemerge \-\-metadata\fR to rebuild the cache as portage does at the end of a sync operation. In order to specify parallel \fB\-\-regen\fR behavior, use -the \fB\-\-jobs\fR and \fB\-\-load\-average\fR options. +the \fB\-\-jobs\fR and \fB\-\-load\-average\fR options. If you would like to +generate and distribute cache for use by others, use \fBegencache\fR(1). .TP .BR "\-\-resume" Resumes the most recent merge list that has been aborted due to an error. diff --git a/man/portage.5 b/man/portage.5 index 0ab4ec344..593410265 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -46,14 +46,15 @@ virtuals .BR /etc/portage/ .nf bashrc +categories color.map +mirrors modules package.keywords package.mask package.unmask package.use -mirrors -categories +repos.conf .fi .TP .BR /etc/portage/profile/ @@ -394,9 +395,51 @@ If needed, this file can be used to set up a special environment for ebuilds, different from the standard root environment. The syntax is the same as for any other bash script. .TP +.BR categories +A simple list of valid categories that may be used in /usr/portage, +PORTDIR_OVERLAY, and PKGDIR (see \fBmake.conf\fR(5)). This allows for custom +categories to be created. + +.I Format: +.nf +\- one category per line +.fi + +.I Example: +.nf +app\-hackers +media\-other +.fi +.TP .BR color.map Contains variables customizing colors. See \fBcolor.map\fR(5). .TP +.BR mirrors +Whenever portage encounters a mirror:// style URL it will look up the actual +hosts here. If the mirror set is not found here, it will check the global +mirrors file at /usr/portage/profiles/thirdpartymirrors. You may also set a +special mirror type called "local". This list of mirrors will be checked +before GENTOO_MIRRORS and will be used even if the package has +RESTRICT="mirror" or RESTRICT="fetch". + +.I Format: +.nf +\- comments begin with # (no inline comments) +\- mirror type followed by a list of hosts +.fi + +.I Example: +.nf +# local private mirrors used only by my company +local ftp://192.168.0.3/mirrors/gentoo http://192.168.0.4/distfiles + +# people in japan would want to use the japanese mirror first +sourceforge http://keihanna.dl.sourceforge.net/sourceforge + +# people in tawain would want to use the local gnu mirror first +gnu ftp://ftp.nctu.edu.tw/UNIX/gnu/ +.fi +.TP .BR modules This file can be used to override the metadata cache implementation. In practice, portdbapi.auxdbmodule is the only variable that the user will want to @@ -505,46 +548,25 @@ documentation for QT. Easy as pie my friend! x11\-libs/qt \-mysql .fi .TP -.BR mirrors -Whenever portage encounters a mirror:// style URL it will look up the actual -hosts here. If the mirror set is not found here, it will check the global -mirrors file at /usr/portage/profiles/thirdpartymirrors. You may also set a -special mirror type called "local". This list of mirrors will be checked -before GENTOO_MIRRORS and will be used even if the package has -RESTRICT="mirror" or RESTRICT="fetch". - -.I Format: -.nf -\- comments begin with # (no inline comments) -\- mirror type followed by a list of hosts -.fi +.BR repos.conf +Specifies \fIsite\-specific\fR repository configuration information. Note that +configuration settings which are specified here do not apply to tools +such as \fBrepoman\fR(1) and \fBegencache\fR(1) since their operations +are inherently \fBnot\fR \fIsite\-specific\fR. Beware that use of +\fBeclass\-overrides\fR is generally not recommended and that it may trigger +performance issues under some circumstances (see \fBbug #124041\fR). .I Example: .nf -# local private mirrors used only by my company -local ftp://192.168.0.3/mirrors/gentoo http://192.168.0.4/distfiles +# make all repositories inherit eclasses from the java\-overlay and +# java\-experimental repositories, with eclasses from java\-experimental +# taking precedence over those from java\-overlay +[DEFAULT] +eclass\-overrides = java\-overlay java\-experimental -# people in japan would want to use the japanese mirror first -sourceforge http://keihanna.dl.sourceforge.net/sourceforge - -# people in tawain would want to use the local gnu mirror first -gnu ftp://ftp.nctu.edu.tw/UNIX/gnu/ -.fi -.TP -.BR categories -A simple list of valid categories that may be used in /usr/portage, -PORTDIR_OVERLAY, and PKGDIR (see \fBmake.conf\fR(5)). This allows for custom -categories to be created. - -.I Format: -.nf -\- one category per line -.fi - -.I Example: -.nf -app\-hackers -media\-other +# disable all eclass overrides for ebuilds from the gentoo repository +[gentoo] +eclass\-overrides = .fi .RE .TP diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 4e379429e..fd57cc3be 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -144,7 +144,8 @@ def help(myaction,myopts,havecolor=1): print " users as rsync updates the cache using server-side caches." print " Rsync users should simply 'emerge --sync' to regenerate." desc = "In order to specify parallel --regen behavior, use "+ \ - "the ---jobs and --load-average options." + "the ---jobs and --load-average options. If you would like to " + \ + "generate and distribute cache for use by others, use egencache(1)." for line in wrap(desc, desc_width): print desc_indent + line print diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 145a0656a..da1e5c2b7 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -988,6 +988,13 @@ def _lazy_iuse_regex(iuse_implicit): regex = regex.replace("\\.\\*", ".*") return regex +class _local_repo_config(object): + __slots__ = ('eclass_overrides', 'name',) + def __init__(self, name, repo_opts): + self.name = name + self.eclass_overrides = \ + tuple(repo_opts.get('eclass-overrides', '').split()) + class config(object): """ This class encompasses the main portage configuration. Data is pulled from @@ -1187,12 +1194,18 @@ class config(object): self.user_profile_dir = None self.local_config = local_config + self._local_repo_configs = None + self._local_repo_conf_path = None if clone: self.incrementals = copy.deepcopy(clone.incrementals) self.profile_path = copy.deepcopy(clone.profile_path) self.user_profile_dir = copy.deepcopy(clone.user_profile_dir) self.local_config = copy.deepcopy(clone.local_config) + self._local_repo_configs = \ + copy.deepcopy(clone._local_repo_configs) + self._local_repo_conf_path = \ + copy.deepcopy(clone._local_repo_conf_path) self.module_priority = copy.deepcopy(clone.module_priority) self.modules = copy.deepcopy(clone.modules) @@ -1639,6 +1652,38 @@ class config(object): self._plicensedict[cp] = cp_dict cp_dict[k] = self.expandLicenseTokens(v) + self._local_repo_configs = {} + self._local_repo_conf_path = \ + os.path.join(abs_user_config, 'repos.conf') + from ConfigParser import SafeConfigParser, ParsingError + repo_conf_parser = SafeConfigParser() + try: + repo_conf_parser.readfp( + codecs.open(self._local_repo_conf_path, + mode='r', errors='replace')) + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise + del e + except ParsingError, e: + portage.util.writemsg_level( + "!!! Error parsing '%s': %s\n" % \ + (self._local_repo_conf_path, e), + level=logging.ERROR, noiselevel=-1) + del e + else: + repo_defaults = repo_conf_parser.defaults() + if repo_defaults: + self._local_repo_configs['DEFAULT'] = \ + _local_repo_config('DEFAULT', repo_defaults) + for repo_name in repo_conf_parser.sections(): + repo_opts = repo_defaults.copy() + for opt_name in repo_conf_parser.options(repo_name): + repo_opts[opt_name] = \ + repo_conf_parser.get(repo_name, opt_name) + self._local_repo_configs[repo_name] = \ + _local_repo_config(repo_name, repo_opts) + #getting categories from an external file now categories = [grabfile(os.path.join(x, "categories")) for x in locations] self.categories = tuple(sorted( diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index e3f6fc571..e988451f4 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -54,7 +54,7 @@ class database(flat_hash.database): if "_eclasses_" not in d: if "INHERITED" in d: if self.ec is None: - self.ec = portage.eclass_cache.cache(self.location) + self.ec = portage.eclass_cache.cache(self.location[:-15]) try: d["_eclasses_"] = self.ec.get_eclass_data( d["INHERITED"].split()) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 3cd8b2acb..ed9ed99a4 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -125,10 +125,6 @@ class portdbapi(dbapi): # this purpose because doebuild makes many changes to the config # instance that is passed in. self.doebuild_settings = config(clone=self.mysettings) - - porttree_root = os.path.realpath(porttree_root) - self.porttree_root = porttree_root - self.depcachedir = os.path.realpath(self.mysettings.depcachedir) if os.environ.get("SANDBOX_ON") == "1": @@ -140,6 +136,46 @@ class portdbapi(dbapi): os.environ["SANDBOX_WRITE"] = \ ":".join(filter(None, sandbox_write)) + porttrees = [os.path.realpath(porttree_root)] + porttrees.extend(os.path.realpath(x) for x in \ + self.mysettings.get('PORTDIR_OVERLAY', '').split()) + treemap = {} + repository_map = {} + self.treemap = treemap + self._repository_map = repository_map + identically_named_paths = set() + for path in porttrees: + if path in repository_map: + continue + repo_name_path = os.path.join(path, REPO_NAME_LOC) + try: + repo_name = open(repo_name_path, 'r').readline().strip() + except EnvironmentError: + # warn about missing repo_name at some other time, since we + # don't want to see a warning every time the portage module is + # imported. + pass + else: + identically_named_path = treemap.get(repo_name) + if identically_named_path is not None: + # The earlier one is discarded. + del repository_map[identically_named_path] + identically_named_paths.add(identically_named_path) + if identically_named_path == porttrees[0]: + # Found another repo with the same name as + # $PORTDIR, so update porttrees[0] to match. + porttrees[0] = path + treemap[repo_name] = path + repository_map[path] = repo_name + + # Ensure that each repo_name is unique. Later paths override + # earlier ones that correspond to the same name. + porttrees = [x for x in porttrees if x not in identically_named_paths] + + self.porttrees = porttrees + porttree_root = porttrees[0] + self.porttree_root = porttree_root + self.eclassdb = eclass_cache.cache(porttree_root) # This is used as sanity check for aux_get(). If there is no @@ -155,28 +191,17 @@ class portdbapi(dbapi): self.xcache = {} self.frozen = 0 - self.porttrees = [self.porttree_root] + \ - [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()] - self.treemap = {} - self._repository_map = {} - for path in self.porttrees: - repo_name_path = os.path.join(path, REPO_NAME_LOC) - try: - repo_name = open(repo_name_path, 'r').readline().strip() - self.treemap[repo_name] = path - self._repository_map[path] = repo_name - except (OSError,IOError): - # warn about missing repo_name at some other time, since we - # don't want to see a warning every time the portage module is - # imported. - pass - self._repo_info = {} eclass_dbs = {porttree_root : self.eclassdb} + local_repo_configs = self.mysettings._local_repo_configs + default_loc_repo_config = None + if local_repo_configs is not None: + default_loc_repo_config = local_repo_configs.get('DEFAULT') for path in self.porttrees: if path in self._repo_info: continue + repo_name = self._repository_map.get(path) layout_filename = os.path.join(path, "metadata/layout.conf") layout_file = KeyValuePairFileLoader(layout_filename, None, None) layout_data, layout_errors = layout_file.load() @@ -200,6 +225,24 @@ class portdbapi(dbapi): porttrees.append(path) + if local_repo_configs is not None: + loc_repo_conf = None + if repo_name is not None: + loc_repo_conf = local_repo_configs.get(repo_name) + if loc_repo_conf is None: + loc_repo_conf = default_loc_repo_config + if loc_repo_conf is not None: + for other_name in loc_repo_conf.eclass_overrides: + other_path = self.treemap.get(other_name) + if other_path is None: + writemsg_level(("Unavailable repository '%s' " + \ + "referenced by eclass-overrides entry in " + \ + "'%s'\n") % (other_name, + self.mysettings._local_repo_conf_path), + level=logging.ERROR, noiselevel=-1) + continue + porttrees.append(other_path) + eclass_db = None for porttree in porttrees: tree_db = eclass_dbs.get(porttree) @@ -211,8 +254,7 @@ class portdbapi(dbapi): else: eclass_db.append(tree_db) - self._repo_info[path] = _repo_info(self._repository_map.get(path), - path, eclass_db) + self._repo_info[path] = _repo_info(repo_name, path, eclass_db) self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") self.auxdb = {} @@ -222,8 +264,8 @@ class portdbapi(dbapi): # ~harring filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys) filtered_auxdbkeys.sort() + from portage.cache import metadata_overlay, volatile if secpass < 1: - from portage.cache import metadata_overlay, volatile for x in self.porttrees: db_ro = self.auxdbmodule(self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid, readonly=True) @@ -238,6 +280,8 @@ class portdbapi(dbapi): # location, label, auxdbkeys self.auxdb[x] = self.auxdbmodule( self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid) + if self.auxdbmodule is metadata_overlay.database: + self.auxdb[x].db_ro.ec = self._repo_info[x].eclass_db if "metadata-transfer" not in self.mysettings.features: for x in self.porttrees: if x in self._pregen_auxdb: @@ -442,6 +486,8 @@ class portdbapi(dbapi): del auxdb[cpv] except KeyError: pass + except CacheError: + pass else: eapi = metadata.get('EAPI', '').strip() if not eapi: