pym/portage/repository/config.py

   1 # Copyright 2010-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 import io
   7 import logging
   8 import warnings
   9 import sys
  10 import re
  11
  12 try:
  13         from configparser import Error as ConfigParserError
  14         if sys.hexversion >= 0x3020000:
  15                 from configparser import ConfigParser as SafeConfigParser
  16         else:
  17                 from configparser import SafeConfigParser
  18 except ImportError:
  19         from ConfigParser import SafeConfigParser, Error as ConfigParserError
  20 import portage
  21 from portage import eclass_cache, os
  22 from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH,
  23         PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH)
  24 from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level
  25 from portage.env.loaders import KeyValuePairFileLoader
  26 from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split,
  27         stack_lists, writemsg, writemsg_level, _recursive_file_list)
  28 from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess
  29 from portage.localization import _
  30 from portage import _unicode_decode
  31 from portage import _unicode_encode
  32 from portage import _encodings
  33 from portage import manifest
  34
  35 if sys.hexversion >= 0x3000000:
  36         basestring = str
  37
  38 # Characters prohibited by repoman's file.name check.
  39 _invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]')
  40
  41 _valid_profile_formats = frozenset(
  42         ['pms', 'portage-1', 'portage-2'])
  43
  44 _portage1_profiles_allow_directories = frozenset(
  45         ["portage-1-compat", "portage-1", 'portage-2'])
  46
  47 _repo_name_sub_re = re.compile(r'[^\w-]')
  48
  49 def _gen_valid_repo(name):
  50         """
  51         Substitute hyphen in place of characters that don't conform to PMS 3.1.5,
  52         and strip hyphen from left side if necessary. This returns None if the
  53         given name contains no valid characters.
  54         """
  55         name = _repo_name_sub_re.sub(' ', name.strip())
  56         name = '-'.join(name.split())
  57         name = name.lstrip('-')
  58         if not name:
  59                 name = None
  60         return name
  61
  62 def _find_invalid_path_char(path, pos=0, endpos=None):
  63         """
  64         Returns the position of the first invalid character found in basename,
  65         or -1 if no invalid characters are found.
  66         """
  67         if endpos is None:
  68                 endpos = len(path)
  69
  70         m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos)
  71         if m is not None:
  72                 return m.start()
  73
  74         return -1
  75
  76 class RepoConfig(object):
  77         """Stores config of one repository"""
  78
  79         __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual',
  80                 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi',
  81                 'eclass_db', 'eclass_locations', 'eclass_overrides',
  82                 'find_invalid_path_char', 'force', 'format', 'local_config', 'location',
  83                 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
  84                 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
  85                 'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo',
  86                 'sync_type', 'sync_uri', 'thin_manifest', 'update_changelog',
  87                 'user_location', '_eapis_banned', '_eapis_deprecated', '_masters_orig')
  88
  89         def __init__(self, name, repo_opts, local_config=True):
  90                 """Build a RepoConfig with options in repo_opts
  91                    Try to read repo_name in repository location, but if
  92                    it is not found use variable name as repository name"""
  93
  94                 force = repo_opts.get('force')
  95                 if force is not None:
  96                         force = tuple(force.split())
  97                 self.force = force
  98                 if force is None:
  99                         force = ()
 100
 101                 self.local_config = local_config
 102
 103                 if local_config or 'aliases' in force:
 104                         aliases = repo_opts.get('aliases')
 105                         if aliases is not None:
 106                                 aliases = tuple(aliases.split())
 107                 else:
 108                         aliases = None
 109
 110                 self.aliases = aliases
 111
 112                 if local_config or 'eclass-overrides' in force:
 113                         eclass_overrides = repo_opts.get('eclass-overrides')
 114                         if eclass_overrides is not None:
 115                                 eclass_overrides = tuple(eclass_overrides.split())
 116                 else:
 117                         eclass_overrides = None
 118
 119                 self.eclass_overrides = eclass_overrides
 120                 # Eclass databases and locations are computed later.
 121                 self.eclass_db = None
 122                 self.eclass_locations = None
 123
 124                 if local_config or 'masters' in force:
 125                         # Masters from repos.conf override layout.conf.
 126                         masters = repo_opts.get('masters')
 127                         if masters is not None:
 128                                 masters = tuple(masters.split())
 129                 else:
 130                         masters = None
 131
 132                 self.masters = masters
 133
 134                 #The main-repo key makes only sense for the 'DEFAULT' section.
 135                 self.main_repo = repo_opts.get('main-repo')
 136
 137                 priority = repo_opts.get('priority')
 138                 if priority is not None:
 139                         try:
 140                                 priority = int(priority)
 141                         except ValueError:
 142                                 priority = None
 143                 self.priority = priority
 144
 145                 sync_cvs_repo = repo_opts.get('sync-cvs-repo')
 146                 if sync_cvs_repo is not None:
 147                         sync_cvs_repo = sync_cvs_repo.strip()
 148                 self.sync_cvs_repo = sync_cvs_repo or None
 149
 150                 sync_type = repo_opts.get('sync-type')
 151                 if sync_type is not None:
 152                         sync_type = sync_type.strip()
 153                 self.sync_type = sync_type or None
 154
 155                 sync_uri = repo_opts.get('sync-uri')
 156                 if sync_uri is not None:
 157                         sync_uri = sync_uri.strip()
 158                 self.sync_uri = sync_uri or None
 159
 160                 # Not implemented.
 161                 format = repo_opts.get('format')
 162                 if format is not None:
 163                         format = format.strip()
 164                 self.format = format
 165
 166                 location = repo_opts.get('location')
 167                 self.user_location = location
 168                 if location is not None and location.strip():
 169                         if os.path.isdir(location) or portage._sync_disabled_warnings:
 170                                 location = os.path.realpath(location)
 171                 else:
 172                         location = None
 173                 self.location = location
 174
 175                 eapi = None
 176                 missing = True
 177                 self.name = name
 178                 if self.location is not None:
 179                         if os.path.isdir(location):
 180                                 eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC))
 181                                 self.name, missing = self._read_valid_repo_name(self.location)
 182                         else:
 183                                 missing = not portage._sync_disabled_warnings
 184                 elif name == "DEFAULT":
 185                         missing = False
 186
 187                 self.eapi = eapi
 188                 self.missing_repo_name = missing
 189                 # sign_commit is disabled by default, since it requires Git >=1.7.9,
 190                 # and key_id configured by `git config user.signingkey key_id`
 191                 self.sign_commit = False
 192                 self.sign_manifest = True
 193                 self.thin_manifest = False
 194                 self.allow_missing_manifest = False
 195                 self.allow_provide_virtual = False
 196                 self.create_manifest = True
 197                 self.disable_manifest = False
 198                 self.manifest_hashes = None
 199                 self.update_changelog = False
 200                 self.cache_formats = None
 201                 self.portage1_profiles = True
 202                 self.portage1_profiles_compat = False
 203                 self.find_invalid_path_char = _find_invalid_path_char
 204                 self._masters_orig = None
 205
 206                 # Parse layout.conf.
 207                 if self.location:
 208                         layout_data = parse_layout_conf(self.location, self.name)[0]
 209                         self._masters_orig = layout_data['masters']
 210
 211                         # layout.conf masters may be overridden here if we have a masters
 212                         # setting from the user's repos.conf
 213                         if self.masters is None:
 214                                 self.masters = layout_data['masters']
 215
 216                         if (local_config or 'aliases' in force) and layout_data['aliases']:
 217                                 aliases = self.aliases
 218                                 if aliases is None:
 219                                         aliases = ()
 220                                 # repos.conf aliases come after layout.conf aliases, giving
 221                                 # them the ability to do incremental overrides
 222                                 self.aliases = layout_data['aliases'] + tuple(aliases)
 223
 224                         if layout_data['repo-name']:
 225                                 # allow layout.conf to override repository name
 226                                 # useful when having two copies of the same repo enabled
 227                                 # to avoid modifying profiles/repo_name in one of them
 228                                 self.name = layout_data['repo-name']
 229
 230                         for value in ('allow-missing-manifest',
 231                                 'allow-provide-virtual', 'cache-formats',
 232                                 'create-manifest', 'disable-manifest', 'manifest-hashes',
 233                                 'profile-formats',
 234                                 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'):
 235                                 setattr(self, value.lower().replace("-", "_"), layout_data[value])
 236
 237                         self.portage1_profiles = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \
 238                                 any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats'])
 239                         self.portage1_profiles_compat = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \
 240                                 layout_data['profile-formats'] == ('portage-1-compat',)
 241
 242                         self._eapis_banned = frozenset(layout_data['eapis-banned'])
 243                         self._eapis_deprecated = frozenset(layout_data['eapis-deprecated'])
 244
 245         def eapi_is_banned(self, eapi):
 246                 return eapi in self._eapis_banned
 247
 248         def eapi_is_deprecated(self, eapi):
 249                 return eapi in self._eapis_deprecated
 250
 251         def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False):
 252                 """
 253                 Reads layout.conf cache-formats from left to right and yields cache
 254                 instances for each supported type that's found. If no cache-formats
 255                 are specified in layout.conf, 'pms' type is assumed if the
 256                 metadata/cache directory exists or force is True.
 257                 """
 258                 formats = self.cache_formats
 259                 if not formats:
 260                         if not force:
 261                                 return
 262                         # The default egencache format was 'pms' prior to portage-2.1.11.32
 263                         # (portage versions prior to portage-2.1.11.14 will NOT
 264                         # recognize md5-dict format unless it is explicitly listed in
 265                         # layout.conf).
 266                         formats = ('md5-dict',)
 267
 268                 for fmt in formats:
 269                         name = None
 270                         if fmt == 'pms':
 271                                 from portage.cache.metadata import database
 272                                 name = 'metadata/cache'
 273                         elif fmt == 'md5-dict':
 274                                 from portage.cache.flat_hash import md5_database as database
 275                                 name = 'metadata/md5-cache'
 276
 277                         if name is not None:
 278                                 yield database(self.location, name,
 279                                         auxdbkeys, readonly=readonly)
 280
 281         def get_pregenerated_cache(self, auxdbkeys, readonly=True, force=False):
 282                 """
 283                 Returns the first cache instance yielded from
 284                 iter_pregenerated_caches(), or None if no cache is available or none
 285                 of the available formats are supported.
 286                 """
 287                 return next(self.iter_pregenerated_caches(
 288                         auxdbkeys, readonly=readonly, force=force), None)
 289
 290         def load_manifest(self, *args, **kwds):
 291                 kwds['thin'] = self.thin_manifest
 292                 kwds['allow_missing'] = self.allow_missing_manifest
 293                 kwds['allow_create'] = self.create_manifest
 294                 kwds['hashes'] = self.manifest_hashes
 295                 if self.disable_manifest:
 296                         kwds['from_scratch'] = True
 297                 kwds['find_invalid_path_char'] = self.find_invalid_path_char
 298                 return manifest.Manifest(*args, **portage._native_kwargs(kwds))
 299
 300         def update(self, new_repo):
 301                 """Update repository with options in another RepoConfig"""
 302
 303                 keys = set(self.__slots__)
 304                 keys.discard("missing_repo_name")
 305                 for k in keys:
 306                         v = getattr(new_repo, k, None)
 307                         if v is not None:
 308                                 setattr(self, k, v)
 309
 310                 if new_repo.name is not None:
 311                         self.missing_repo_name = new_repo.missing_repo_name
 312
 313         @staticmethod
 314         def _read_valid_repo_name(repo_path):
 315                 name, missing = RepoConfig._read_repo_name(repo_path)
 316                 # We must ensure that the name conforms to PMS 3.1.5
 317                 # in order to avoid InvalidAtom exceptions when we
 318                 # use it to generate atoms.
 319                 name = _gen_valid_repo(name)
 320                 if not name:
 321                         # name only contains invalid characters
 322                         name = "x-" + os.path.basename(repo_path)
 323                         name = _gen_valid_repo(name)
 324                         # If basename only contains whitespace then the
 325                         # end result is name = 'x-'.
 326                 return name, missing
 327
 328         @staticmethod
 329         def _read_repo_name(repo_path):
 330                 """
 331                 Read repo_name from repo_path.
 332                 Returns repo_name, missing.
 333                 """
 334                 repo_name_path = os.path.join(repo_path, REPO_NAME_LOC)
 335                 f = None
 336                 try:
 337                         f = io.open(
 338                                 _unicode_encode(repo_name_path,
 339                                 encoding=_encodings['fs'], errors='strict'),
 340                                 mode='r', encoding=_encodings['repo.content'],
 341                                 errors='replace')
 342                         return f.readline().strip(), False
 343                 except EnvironmentError:
 344                         return "x-" + os.path.basename(repo_path), True
 345                 finally:
 346                         if f is not None:
 347                                 f.close()
 348
 349         def info_string(self):
 350                 """
 351                 Returns a formatted string containing informations about the repository.
 352                 Used by emerge --info.
 353                 """
 354                 indent = " " * 4
 355                 repo_msg = []
 356                 repo_msg.append(self.name)
 357                 if self.format:
 358                         repo_msg.append(indent + "format: " + self.format)
 359                 if self.user_location:
 360                         repo_msg.append(indent + "location: " + self.user_location)
 361                 if self.sync_cvs_repo:
 362                         repo_msg.append(indent + "sync-cvs-repo: " + self.sync_cvs_repo)
 363                 if self.sync_type:
 364                         repo_msg.append(indent + "sync-type: " + self.sync_type)
 365                 if self.sync_uri:
 366                         repo_msg.append(indent + "sync-uri: " + self.sync_uri)
 367                 if self.masters:
 368                         repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters))
 369                 if self.priority is not None:
 370                         repo_msg.append(indent + "priority: " + str(self.priority))
 371                 if self.aliases:
 372                         repo_msg.append(indent + "aliases: " + " ".join(self.aliases))
 373                 if self.eclass_overrides:
 374                         repo_msg.append(indent + "eclass-overrides: " + \
 375                                 " ".join(self.eclass_overrides))
 376                 repo_msg.append("")
 377                 return "\n".join(repo_msg)
 378
 379         def __repr__(self):
 380                 return "<portage.repository.config.RepoConfig(name=%r, location=%r)>" % (self.name, _unicode_decode(self.location))
 381
 382         def __str__(self):
 383                 d = {}
 384                 for k in self.__slots__:
 385                         d[k] = getattr(self, k, None)
 386                 return "%s" % (d,)
 387
 388         if sys.hexversion < 0x3000000:
 389
 390                 __unicode__ = __str__
 391
 392                 def __str__(self):
 393                         return _unicode_encode(self.__unicode__())
 394
 395 class RepoConfigLoader(object):
 396         """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf"""
 397
 398         @staticmethod
 399         def _add_repositories(portdir, portdir_overlay, prepos,
 400                 ignored_map, ignored_location_map, local_config, default_portdir):
 401                 """Add overlays in PORTDIR_OVERLAY as repositories"""
 402                 overlays = []
 403                 portdir_orig = None
 404                 if portdir:
 405                         portdir = normalize_path(portdir)
 406                         portdir_orig = portdir
 407                         overlays.append(portdir)
 408                 try:
 409                         port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
 410                 except ValueError as e:
 411                         #File "/usr/lib/python3.2/shlex.py", line 168, in read_token
 412                         #       raise ValueError("No closing quotation")
 413                         writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
 414                                 " %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
 415                         port_ov = []
 416                 overlays.extend(port_ov)
 417                 default_repo_opts = {}
 418                 if prepos['DEFAULT'].aliases is not None:
 419                         default_repo_opts['aliases'] = \
 420                                 ' '.join(prepos['DEFAULT'].aliases)
 421                 if prepos['DEFAULT'].eclass_overrides is not None:
 422                         default_repo_opts['eclass-overrides'] = \
 423                                 ' '.join(prepos['DEFAULT'].eclass_overrides)
 424                 if prepos['DEFAULT'].masters is not None:
 425                         default_repo_opts['masters'] = \
 426                                 ' '.join(prepos['DEFAULT'].masters)
 427
 428                 if overlays:
 429                         # We need a copy of the original repos.conf data, since we're
 430                         # going to modify the prepos dict and some of the RepoConfig
 431                         # objects that we put in prepos may have to be discarded if
 432                         # they get overridden by a repository with the same name but
 433                         # a different location. This is common with repoman, for example,
 434                         # when temporarily overriding an rsync repo with another copy
 435                         # of the same repo from CVS.
 436                         repos_conf = prepos.copy()
 437                         #overlay priority is negative because we want them to be looked before any other repo
 438                         base_priority = 0
 439                         for ov in overlays:
 440                                 if isdir_raise_eaccess(ov):
 441                                         repo_opts = default_repo_opts.copy()
 442                                         repo_opts['location'] = ov
 443                                         repo = RepoConfig(None, repo_opts, local_config=local_config)
 444                                         # repos_conf_opts contains options from repos.conf
 445                                         repos_conf_opts = repos_conf.get(repo.name)
 446                                         if repos_conf_opts is not None:
 447                                                 # Selectively copy only the attributes which
 448                                                 # repos.conf is allowed to override.
 449                                                 for k in ('aliases', 'eclass_overrides', 'force', 'masters',
 450                                                         'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'):
 451                                                         v = getattr(repos_conf_opts, k, None)
 452                                                         if v is not None:
 453                                                                 setattr(repo, k, v)
 454
 455                                         if repo.name in prepos:
 456                                                 old_location = prepos[repo.name].location
 457                                                 if old_location is not None and \
 458                                                         old_location != repo.location and \
 459                                                         not (old_location == default_portdir and
 460                                                         not exists_raise_eaccess(old_location)):
 461                                                         ignored_map.setdefault(repo.name, []).append(old_location)
 462                                                         ignored_location_map[old_location] = repo.name
 463                                                         if old_location == portdir:
 464                                                                 portdir = repo.user_location
 465
 466                                         if repo.priority is None:
 467                                                 if base_priority == 0 and ov == portdir_orig:
 468                                                         # If it's the original PORTDIR setting and it's not
 469                                                         # in PORTDIR_OVERLAY, then it will be assigned a
 470                                                         # special priority setting later.
 471                                                         pass
 472                                                 else:
 473                                                         repo.priority = base_priority
 474                                                         base_priority += 1
 475
 476                                         prepos[repo.name] = repo
 477                                 else:
 478
 479                                         if not portage._sync_disabled_warnings:
 480                                                 writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1)
 481
 482                 return portdir
 483
 484         @staticmethod
 485         def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir):
 486                 """Parse files in paths to load config"""
 487                 parser = SafeConfigParser()
 488
 489                 # use read_file/readfp in order to control decoding of unicode
 490                 try:
 491                         # Python >=3.2
 492                         read_file = parser.read_file
 493                         source_kwarg = 'source'
 494                 except AttributeError:
 495                         read_file = parser.readfp
 496                         source_kwarg = 'filename'
 497
 498                 recursive_paths = []
 499                 for p in paths:
 500                         if isinstance(p, basestring):
 501                                 recursive_paths.extend(_recursive_file_list(p))
 502                         else:
 503                                 recursive_paths.append(p)
 504
 505                 for p in recursive_paths:
 506                         if isinstance(p, basestring):
 507                                 f = None
 508                                 try:
 509                                         f = io.open(_unicode_encode(p,
 510                                                 encoding=_encodings['fs'], errors='strict'),
 511                                                 mode='r', encoding=_encodings['repo.content'],
 512                                                 errors='replace')
 513                                 except EnvironmentError:
 514                                         pass
 515                                 else:
 516                                         # The 'source' keyword argument is needed since otherwise
 517                                         # ConfigParser in Python <3.3.3 may throw a TypeError
 518                                         # because it assumes that f.name is a native string rather
 519                                         # than binary when constructing error messages.
 520                                         kwargs = {source_kwarg: p}
 521                                         read_file(f, **portage._native_kwargs(kwargs))
 522                                 finally:
 523                                         if f is not None:
 524                                                 f.close()
 525                         elif isinstance(p, io.StringIO):
 526                                 kwargs = {source_kwarg: "<io.StringIO>"}
 527                                 read_file(p, **portage._native_kwargs(kwargs))
 528                         else:
 529                                 raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p))
 530
 531                 prepos['DEFAULT'] = RepoConfig("DEFAULT",
 532                         parser.defaults(), local_config=local_config)
 533
 534                 for sname in parser.sections():
 535                         optdict = {}
 536                         for oname in parser.options(sname):
 537                                 optdict[oname] = parser.get(sname, oname)
 538
 539                         repo = RepoConfig(sname, optdict, local_config=local_config)
 540
 541                         if repo.sync_type is not None and repo.sync_uri is None:
 542                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute") %
 543                                         sname, level=logging.ERROR, noiselevel=-1)
 544                                 continue
 545
 546                         if repo.sync_uri is not None and repo.sync_type is None:
 547                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute") %
 548                                         sname, level=logging.ERROR, noiselevel=-1)
 549                                 continue
 550
 551                         if repo.sync_type not in (None, "cvs", "git", "rsync"):
 552                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute set to unsupported value: '%s'") %
 553                                         (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1)
 554                                 continue
 555
 556                         if repo.sync_type == "cvs" and repo.sync_cvs_repo is None:
 557                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute") %
 558                                         sname, level=logging.ERROR, noiselevel=-1)
 559                                 continue
 560
 561                         # For backward compatibility with locations set via PORTDIR and
 562                         # PORTDIR_OVERLAY, delay validation of the location and repo.name
 563                         # until after PORTDIR and PORTDIR_OVERLAY have been processed.
 564                         prepos[sname] = repo
 565
 566         def __init__(self, paths, settings):
 567                 """Load config from files in paths"""
 568
 569                 prepos = {}
 570                 location_map = {}
 571                 treemap = {}
 572                 ignored_map = {}
 573                 ignored_location_map = {}
 574
 575                 if "PORTAGE_REPOSITORIES" in settings:
 576                         portdir = ""
 577                         portdir_overlay = ""
 578                         portdir_sync = ""
 579                 else:
 580                         portdir = settings.get("PORTDIR", "")
 581                         portdir_overlay = settings.get("PORTDIR_OVERLAY", "")
 582                         portdir_sync = settings.get("SYNC", "")
 583
 584                 try:
 585                         self._parse(paths, prepos, ignored_map,
 586                                 ignored_location_map, settings.local_config,
 587                                 portdir)
 588                 except ConfigParserError as e:
 589                         writemsg(
 590                                 _("!!! Error while reading repo config file: %s\n") % e,
 591                                 noiselevel=-1)
 592                         # The configparser state is unreliable (prone to quirky
 593                         # exceptions) after it has thrown an error, so use empty
 594                         # config and try to fall back to PORTDIR{,_OVERLAY}.
 595                         prepos.clear()
 596                         prepos['DEFAULT'] = RepoConfig('DEFAULT',
 597                                 {}, local_config=settings.local_config)
 598                         location_map.clear()
 599                         treemap.clear()
 600                         ignored_map.clear()
 601                         ignored_location_map.clear()
 602
 603                 default_portdir = os.path.join(os.sep,
 604                         settings['EPREFIX'].lstrip(os.sep), 'usr', 'portage')
 605
 606                 # If PORTDIR_OVERLAY contains a repo with the same repo_name as
 607                 # PORTDIR, then PORTDIR is overridden.
 608                 portdir = self._add_repositories(portdir, portdir_overlay, prepos,
 609                         ignored_map, ignored_location_map, settings.local_config,
 610                         default_portdir)
 611                 if portdir and portdir.strip():
 612                         portdir = os.path.realpath(portdir)
 613
 614                 ignored_repos = tuple((repo_name, tuple(paths)) \
 615                         for repo_name, paths in ignored_map.items())
 616
 617                 self.missing_repo_names = frozenset(repo.location
 618                         for repo in prepos.values()
 619                         if repo.location is not None and repo.missing_repo_name)
 620
 621                 # Do this before expanding aliases, so that location_map and
 622                 # treemap consistently map unaliased names whenever available.
 623                 for repo_name, repo in list(prepos.items()):
 624                         if repo.location is None:
 625                                 if repo_name != 'DEFAULT':
 626                                         # Skip this warning for repoman (bug #474578).
 627                                         if settings.local_config and paths:
 628                                                 writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf is missing location attribute") %
 629                                                         repo.name, level=logging.ERROR, noiselevel=-1)
 630                                         del prepos[repo_name]
 631                                         continue
 632                         else:
 633                                 if not portage._sync_disabled_warnings:
 634                                         if not isdir_raise_eaccess(repo.location):
 635                                                 writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has location attribute set "
 636                                                         "to nonexistent directory: '%s'") %
 637                                                         (repo_name, repo.location), level=logging.ERROR, noiselevel=-1)
 638                                                 del prepos[repo_name]
 639                                                 continue
 640
 641                                         # After removing support for PORTDIR_OVERLAY, the following check can be:
 642                                         # if repo.missing_repo_name:
 643                                         if repo.missing_repo_name and repo.name != repo_name:
 644                                                 writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf refers to repository "
 645                                                         "without repository name set in '%s'") %
 646                                                         (repo_name, os.path.join(repo.location, REPO_NAME_LOC)), level=logging.ERROR, noiselevel=-1)
 647                                                 del prepos[repo_name]
 648                                                 continue
 649
 650                                         if repo.name != repo_name:
 651                                                 writemsg_level("!!! %s\n" % _("Section '%s' in repos.conf has name different "
 652                                                         "from repository name '%s' set inside repository") %
 653                                                         (repo_name, repo.name), level=logging.ERROR, noiselevel=-1)
 654                                                 del prepos[repo_name]
 655                                                 continue
 656
 657                                 location_map[repo.location] = repo_name
 658                                 treemap[repo_name] = repo.location
 659
 660                 # Add alias mappings, but never replace unaliased mappings.
 661                 for repo_name, repo in list(prepos.items()):
 662                         names = set()
 663                         names.add(repo_name)
 664                         if repo.aliases:
 665                                 aliases = stack_lists([repo.aliases], incremental=True)
 666                                 names.update(aliases)
 667
 668                         for name in names:
 669                                 if name in prepos and prepos[name].location is not None:
 670                                         if name == repo_name:
 671                                                 # unaliased names already handled earlier
 672                                                 continue
 673                                         writemsg_level(_("!!! Repository name or alias '%s', " + \
 674                                                 "defined for repository '%s', overrides " + \
 675                                                 "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1)
 676                                         # Never replace an unaliased mapping with
 677                                         # an aliased mapping.
 678                                         continue
 679                                 prepos[name] = repo
 680                                 if repo.location is not None:
 681                                         if repo.location not in location_map:
 682                                                 # Never replace an unaliased mapping with
 683                                                 # an aliased mapping.
 684                                                 location_map[repo.location] = name
 685                                         treemap[name] = repo.location
 686
 687                 main_repo = prepos['DEFAULT'].main_repo
 688                 if main_repo is None or main_repo not in prepos:
 689                         #setting main_repo if it was not set in repos.conf
 690                         main_repo = location_map.get(portdir)
 691                         if main_repo is not None:
 692                                 prepos['DEFAULT'].main_repo = main_repo
 693                         else:
 694                                 prepos['DEFAULT'].main_repo = None
 695                                 if not portage._sync_disabled_warnings:
 696                                         writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty.\n"), noiselevel=-1)
 697
 698                 if main_repo is not None and prepos[main_repo].priority is None:
 699                         # This happens if main-repo has been set in repos.conf.
 700                         prepos[main_repo].priority = -1000
 701
 702                 # Backward compatible SYNC support for mirrorselect.
 703                 if portdir_sync and main_repo is not None:
 704                         if portdir_sync.startswith("rsync://"):
 705                                 prepos[main_repo].sync_uri = portdir_sync
 706                                 prepos[main_repo].sync_type = "rsync"
 707
 708                 # Include repo.name in sort key, for predictable sorting
 709                 # even when priorities are equal.
 710                 prepos_order = sorted(prepos.items(),
 711                         key=lambda r:(r[1].priority or 0, r[1].name))
 712
 713                 # filter duplicates from aliases, by only including
 714                 # items where repo.name == key
 715                 prepos_order = [repo.name for (key, repo) in prepos_order
 716                         if repo.name == key and key != 'DEFAULT' and
 717                         repo.location is not None]
 718
 719                 self.prepos = prepos
 720                 self.prepos_order = prepos_order
 721                 self.ignored_repos = ignored_repos
 722                 self.location_map = location_map
 723                 self.treemap = treemap
 724                 self._prepos_changed = True
 725                 self._repo_location_list = []
 726
 727                 #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig.
 728                 for repo_name, repo in prepos.items():
 729                         if repo_name == "DEFAULT":
 730                                 continue
 731                         if repo.masters is None:
 732                                 if self.mainRepo() and repo_name != self.mainRepo().name:
 733                                         repo.masters = self.mainRepo(),
 734                                 else:
 735                                         repo.masters = ()
 736                         else:
 737                                 if repo.masters and isinstance(repo.masters[0], RepoConfig):
 738                                         # This one has already been processed
 739                                         # because it has an alias.
 740                                         continue
 741                                 master_repos = []
 742                                 for master_name in repo.masters:
 743                                         if master_name not in prepos:
 744                                                 layout_filename = os.path.join(repo.user_location,
 745                                                         "metadata", "layout.conf")
 746                                                 writemsg_level(_("Unavailable repository '%s' " \
 747                                                         "referenced by masters entry in '%s'\n") % \
 748                                                         (master_name, layout_filename),
 749                                                         level=logging.ERROR, noiselevel=-1)
 750                                         else:
 751                                                 master_repos.append(prepos[master_name])
 752                                 repo.masters = tuple(master_repos)
 753
 754                 #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths.
 755                 for repo_name, repo in prepos.items():
 756                         if repo_name == "DEFAULT":
 757                                 continue
 758
 759                         eclass_locations = []
 760                         eclass_locations.extend(master_repo.location for master_repo in repo.masters)
 761                         # Only append the current repo to eclass_locations if it's not
 762                         # there already. This allows masters to have more control over
 763                         # eclass override order, which may be useful for scenarios in
 764                         # which there is a plan to migrate eclasses to a master repo.
 765                         if repo.location not in eclass_locations:
 766                                 eclass_locations.append(repo.location)
 767
 768                         if repo.eclass_overrides:
 769                                 for other_repo_name in repo.eclass_overrides:
 770                                         if other_repo_name in self.treemap:
 771                                                 eclass_locations.append(self.get_location_for_name(other_repo_name))
 772                                         else:
 773                                                 writemsg_level(_("Unavailable repository '%s' " \
 774                                                         "referenced by eclass-overrides entry for " \
 775                                                         "'%s'\n") % (other_repo_name, repo_name), \
 776                                                         level=logging.ERROR, noiselevel=-1)
 777                         repo.eclass_locations = tuple(eclass_locations)
 778
 779                 eclass_dbs = {}
 780                 for repo_name, repo in prepos.items():
 781                         if repo_name == "DEFAULT":
 782                                 continue
 783
 784                         eclass_db = None
 785                         for eclass_location in repo.eclass_locations:
 786                                 tree_db = eclass_dbs.get(eclass_location)
 787                                 if tree_db is None:
 788                                         tree_db = eclass_cache.cache(eclass_location)
 789                                         eclass_dbs[eclass_location] = tree_db
 790                                 if eclass_db is None:
 791                                         eclass_db = tree_db.copy()
 792                                 else:
 793                                         eclass_db.append(tree_db)
 794                         repo.eclass_db = eclass_db
 795
 796                 for repo_name, repo in prepos.items():
 797                         if repo_name == "DEFAULT":
 798                                 continue
 799
 800                         if repo._masters_orig is None and self.mainRepo() and \
 801                                 repo_name != self.mainRepo().name and not portage._sync_disabled_warnings:
 802                                 writemsg_level("!!! %s\n" % _("Repository '%s' is missing masters attribute in '%s'") %
 803                                         (repo_name, os.path.join(repo.location, "metadata", "layout.conf")) +
 804                                         "!!! %s\n" % _("Set 'masters = %s' in this file for future compatibility") %
 805                                         self.mainRepo().name, level=logging.WARNING, noiselevel=-1)
 806
 807                 self._prepos_changed = True
 808                 self._repo_location_list = []
 809
 810                 self._check_locations()
 811
 812         def repoLocationList(self):
 813                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
 814                 if self._prepos_changed:
 815                         _repo_location_list = []
 816                         for repo in self.prepos_order:
 817                                 if self.prepos[repo].location is not None:
 818                                         _repo_location_list.append(self.prepos[repo].location)
 819                         self._repo_location_list = tuple(_repo_location_list)
 820
 821                         self._prepos_changed = False
 822                 return self._repo_location_list
 823
 824         def repoUserLocationList(self):
 825                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
 826                 user_location_list = []
 827                 for repo in self.prepos_order:
 828                         if self.prepos[repo].location is not None:
 829                                 user_location_list.append(self.prepos[repo].user_location)
 830                 return tuple(user_location_list)
 831
 832         def mainRepoLocation(self):
 833                 """Returns the location of main repo"""
 834                 main_repo = self.prepos['DEFAULT'].main_repo
 835                 if main_repo is not None and main_repo in self.prepos:
 836                         return self.prepos[main_repo].location
 837                 else:
 838                         return ''
 839
 840         def mainRepo(self):
 841                 """Returns the main repo"""
 842                 main_repo = self.prepos['DEFAULT'].main_repo
 843                 if main_repo is None:
 844                         return None
 845                 return self.prepos[main_repo]
 846
 847         def _check_locations(self):
 848                 """Check if repositories location are correct and show a warning message if not"""
 849                 for (name, r) in self.prepos.items():
 850                         if name != 'DEFAULT':
 851                                 if r.location is None:
 852                                         writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1)
 853                                 else:
 854                                         if not isdir_raise_eaccess(r.location) and not portage._sync_disabled_warnings:
 855                                                 self.prepos_order.remove(name)
 856                                                 writemsg(_("!!! Invalid Repository Location"
 857                                                         " (not a dir): '%s'\n") % r.location, noiselevel=-1)
 858
 859         def repos_with_profiles(self):
 860                 for repo_name in self.prepos_order:
 861                         repo = self.prepos[repo_name]
 862                         if repo.format != "unavailable":
 863                                 yield repo
 864
 865         def get_name_for_location(self, location):
 866                 return self.location_map[location]
 867
 868         def get_location_for_name(self, repo_name):
 869                 if repo_name is None:
 870                         # This simplifies code in places where
 871                         # we want to be able to pass in Atom.repo
 872                         # even if it is None.
 873                         return None
 874                 return self.treemap[repo_name]
 875
 876         def get_repo_for_location(self, location):
 877                 return self.prepos[self.get_name_for_location(location)]
 878
 879         def __setitem__(self, repo_name, repo):
 880                 # self.prepos[repo_name] = repo
 881                 raise NotImplementedError
 882
 883         def __getitem__(self, repo_name):
 884                 return self.prepos[repo_name]
 885
 886         def __delitem__(self, repo_name):
 887                 location = self.prepos[repo_name].location
 888                 del self.prepos[repo_name]
 889                 if repo_name in self.prepos_order:
 890                         self.prepos_order.remove(repo_name)
 891                 for k, v in self.location_map.copy().items():
 892                         if v == repo_name:
 893                                 del self.location_map[k]
 894                 if repo_name in self.treemap:
 895                         del self.treemap[repo_name]
 896                 self._repo_location_list = tuple(x for x in self._repo_location_list if x != location)
 897
 898         def __iter__(self):
 899                 for repo_name in self.prepos_order:
 900                         yield self.prepos[repo_name]
 901
 902         def __contains__(self, repo_name):
 903                 return repo_name in self.prepos
 904
 905         def config_string(self):
 906                 str_or_int_keys = ("format", "location", "main_repo", "priority", "sync_cvs_repo", "sync_type", "sync_uri")
 907                 str_tuple_keys = ("aliases", "eclass_overrides", "force")
 908                 repo_config_tuple_keys = ("masters",)
 909                 keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys
 910                 config_string = ""
 911                 for repo_name, repo in sorted(self.prepos.items()):
 912                         config_string += "\n[%s]\n" % repo_name
 913                         for key in sorted(keys):
 914                                 if key == "main_repo" and repo_name != "DEFAULT":
 915                                         continue
 916                                 if getattr(repo, key) is not None:
 917                                         if key in str_or_int_keys:
 918                                                 config_string += "%s = %s\n" % (key.replace("_", "-"), getattr(repo, key))
 919                                         elif key in str_tuple_keys:
 920                                                 config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(getattr(repo, key)))
 921                                         elif key in repo_config_tuple_keys:
 922                                                 config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(x.name for x in getattr(repo, key)))
 923                 return config_string.lstrip("\n")
 924
 925 def load_repository_config(settings, extra_files=None):
 926         repoconfigpaths = []
 927         if "PORTAGE_REPOSITORIES" in settings:
 928                 repoconfigpaths.append(io.StringIO(settings["PORTAGE_REPOSITORIES"]))
 929         else:
 930                 if portage._working_copy:
 931                         repoconfigpaths.append(os.path.join(PORTAGE_BASE_PATH, "cnf", "repos.conf"))
 932                 else:
 933                         repoconfigpaths.append(os.path.join(settings.global_config_path, "repos.conf"))
 934                 repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "repos.conf"))
 935         if extra_files:
 936                 repoconfigpaths.extend(extra_files)
 937         return RepoConfigLoader(repoconfigpaths, settings)
 938
 939 def _get_repo_name(repo_location, cached=None):
 940         if cached is not None:
 941                 return cached
 942         name, missing = RepoConfig._read_repo_name(repo_location)
 943         if missing:
 944                 return None
 945         return name
 946
 947 def parse_layout_conf(repo_location, repo_name=None):
 948         eapi = read_corresponding_eapi_file(os.path.join(repo_location, REPO_NAME_LOC))
 949
 950         layout_filename = os.path.join(repo_location, "metadata", "layout.conf")
 951         layout_file = KeyValuePairFileLoader(layout_filename, None, None)
 952         layout_data, layout_errors = layout_file.load()
 953
 954         data = {}
 955
 956         # None indicates abscence of a masters setting, which later code uses
 957         # to trigger a backward compatibility fallback that sets an implicit
 958         # master. In order to avoid this fallback behavior, layout.conf can
 959         # explicitly set masters to an empty value, which will result in an
 960         # empty tuple here instead of None.
 961         masters = layout_data.get('masters')
 962         if masters is not None:
 963                 masters = tuple(masters.split())
 964         data['masters'] = masters
 965         data['aliases'] = tuple(layout_data.get('aliases', '').split())
 966
 967         data['allow-provide-virtual'] = \
 968                 layout_data.get('allow-provide-virtuals', 'false').lower() == 'true'
 969
 970         data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split())
 971         data['eapis-deprecated'] = tuple(layout_data.get('eapis-deprecated', '').split())
 972
 973         data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \
 974                 == 'true'
 975
 976         data['sign-manifest'] = layout_data.get('sign-manifests', 'true').lower() \
 977                 == 'true'
 978
 979         data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \
 980                 == 'true'
 981
 982         data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', ''))
 983
 984         manifest_policy = layout_data.get('use-manifests', 'strict').lower()
 985         data['allow-missing-manifest'] = manifest_policy != 'strict'
 986         data['create-manifest'] = manifest_policy != 'false'
 987         data['disable-manifest'] = manifest_policy == 'false'
 988
 989         # for compatibility w/ PMS, fallback to pms; but also check if the
 990         # cache exists or not.
 991         cache_formats = layout_data.get('cache-formats', '').lower().split()
 992         if not cache_formats:
 993                 # Auto-detect cache formats, and prefer md5-cache if available.
 994                 # This behavior was deployed in portage-2.1.11.14, so that the
 995                 # default egencache format could eventually be changed to md5-dict
 996                 # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14
 997                 # will NOT recognize md5-dict format unless it is explicitly
 998                 # listed in layout.conf.
 999                 cache_formats = []
1000                 if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')):
1001                         cache_formats.append('md5-dict')
1002                 if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')):
1003                         cache_formats.append('pms')
1004         data['cache-formats'] = tuple(cache_formats)
1005
1006         manifest_hashes = layout_data.get('manifest-hashes')
1007         if manifest_hashes is not None:
1008                 manifest_hashes = frozenset(manifest_hashes.upper().split())
1009                 if MANIFEST2_REQUIRED_HASH not in manifest_hashes:
1010                         repo_name = _get_repo_name(repo_location, cached=repo_name)
1011                         warnings.warn((_("Repository named '%(repo_name)s' has a "
1012                                 "'manifest-hashes' setting that does not contain "
1013                                 "the '%(hash)s' hash which is required by this "
1014                                 "portage version. You will have to upgrade portage "
1015                                 "if you want to generate valid manifests for this "
1016                                 "repository: %(layout_filename)s") %
1017                                 {"repo_name": repo_name or 'unspecified',
1018                                 "hash":MANIFEST2_REQUIRED_HASH,
1019                                 "layout_filename":layout_filename}),
1020                                 DeprecationWarning)
1021                 unsupported_hashes = manifest_hashes.difference(
1022                         MANIFEST2_HASH_FUNCTIONS)
1023                 if unsupported_hashes:
1024                         repo_name = _get_repo_name(repo_location, cached=repo_name)
1025                         warnings.warn((_("Repository named '%(repo_name)s' has a "
1026                                 "'manifest-hashes' setting that contains one "
1027                                 "or more hash types '%(hashes)s' which are not supported by "
1028                                 "this portage version. You will have to upgrade "
1029                                 "portage if you want to generate valid manifests for "
1030                                 "this repository: %(layout_filename)s") %
1031                                 {"repo_name": repo_name or 'unspecified',
1032                                 "hashes":" ".join(sorted(unsupported_hashes)),
1033                                 "layout_filename":layout_filename}),
1034                                 DeprecationWarning)
1035         data['manifest-hashes'] = manifest_hashes
1036
1037         data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \
1038                 == 'true'
1039
1040         raw_formats = layout_data.get('profile-formats')
1041         if raw_formats is None:
1042                 if eapi_allows_directories_on_profile_level_and_repository_level(eapi):
1043                         raw_formats = ('portage-1',)
1044                 else:
1045                         raw_formats = ('portage-1-compat',)
1046         else:
1047                 raw_formats = set(raw_formats.split())
1048                 unknown = raw_formats.difference(_valid_profile_formats)
1049                 if unknown:
1050                         repo_name = _get_repo_name(repo_location, cached=repo_name)
1051                         warnings.warn((_("Repository named '%(repo_name)s' has unsupported "
1052                                 "profiles in use ('profile-formats = %(unknown_fmts)s' setting in "
1053                                 "'%(layout_filename)s; please upgrade portage.") %
1054                                 dict(repo_name=repo_name or 'unspecified',
1055                                 layout_filename=layout_filename,
1056                                 unknown_fmts=" ".join(unknown))),
1057                                 DeprecationWarning)
1058                 raw_formats = tuple(raw_formats.intersection(_valid_profile_formats))
1059         data['profile-formats'] = raw_formats
1060
1061         return data, layout_errors